Line data Source code
1 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 : /* This Source Code Form is subject to the terms of the Mozilla Public
3 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
5 :
6 : #include "MediaStreamGraphImpl.h"
7 : #include "mozilla/MathAlgorithms.h"
8 : #include "mozilla/SizePrintfMacros.h"
9 : #include "mozilla/Unused.h"
10 :
11 : #include "AudioSegment.h"
12 : #include "VideoSegment.h"
13 : #include "nsContentUtils.h"
14 : #include "nsIObserver.h"
15 : #include "nsPrintfCString.h"
16 : #include "nsServiceManagerUtils.h"
17 : #include "prerror.h"
18 : #include "mozilla/Logging.h"
19 : #include "mozilla/Attributes.h"
20 : #include "TrackUnionStream.h"
21 : #include "ImageContainer.h"
22 : #include "AudioCaptureStream.h"
23 : #include "AudioChannelService.h"
24 : #include "AudioNodeStream.h"
25 : #include "AudioNodeExternalInputStream.h"
26 : #include "MediaStreamListener.h"
27 : #include "MediaStreamVideoSink.h"
28 : #include "mozilla/dom/BaseAudioContextBinding.h"
29 : #include "mozilla/media/MediaUtils.h"
30 : #include <algorithm>
31 : #include "GeckoProfiler.h"
32 : #include "VideoFrameContainer.h"
33 : #include "mozilla/AbstractThread.h"
34 : #include "mozilla/Unused.h"
35 : #ifdef MOZ_WEBRTC
36 : #include "AudioOutputObserver.h"
37 : #endif
38 : #include "mtransport/runnable_utils.h"
39 :
40 : #include "webaudio/blink/DenormalDisabler.h"
41 : #include "webaudio/blink/HRTFDatabaseLoader.h"
42 :
43 : using namespace mozilla::layers;
44 : using namespace mozilla::dom;
45 : using namespace mozilla::gfx;
46 : using namespace mozilla::media;
47 :
48 : namespace mozilla {
49 :
50 : LazyLogModule gMediaStreamGraphLog("MediaStreamGraph");
51 : #ifdef LOG
52 : #undef LOG
53 : #endif // LOG
54 : #define LOG(type, msg) MOZ_LOG(gMediaStreamGraphLog, type, msg)
55 :
56 : enum SourceMediaStream::TrackCommands : uint32_t {
57 : TRACK_CREATE = TrackEventCommand::TRACK_EVENT_CREATED,
58 : TRACK_END = TrackEventCommand::TRACK_EVENT_ENDED,
59 : TRACK_UNUSED = TrackEventCommand::TRACK_EVENT_UNUSED,
60 : };
61 :
62 : /**
63 : * A hash table containing the graph instances, one per AudioChannel.
64 : */
65 3 : static nsDataHashtable<nsUint32HashKey, MediaStreamGraphImpl*> gGraphs;
66 :
67 0 : MediaStreamGraphImpl::~MediaStreamGraphImpl()
68 : {
69 0 : NS_ASSERTION(IsEmpty(),
70 : "All streams should have been destroyed by messages from the main thread");
71 0 : LOG(LogLevel::Debug, ("MediaStreamGraph %p destroyed", this));
72 0 : LOG(LogLevel::Debug, ("MediaStreamGraphImpl::~MediaStreamGraphImpl"));
73 0 : }
74 :
75 : void
76 0 : MediaStreamGraphImpl::FinishStream(MediaStream* aStream)
77 : {
78 0 : if (aStream->mFinished)
79 0 : return;
80 0 : LOG(LogLevel::Debug, ("MediaStream %p will finish", aStream));
81 : #ifdef DEBUG
82 0 : for (StreamTracks::TrackIter track(aStream->mTracks);
83 0 : !track.IsEnded(); track.Next()) {
84 0 : if (!track->IsEnded()) {
85 0 : LOG(LogLevel::Error,
86 : ("MediaStream %p will finish, but track %d has not ended.",
87 : aStream,
88 : track->GetID()));
89 0 : NS_ASSERTION(false, "Finished stream cannot contain live track");
90 : }
91 : }
92 : #endif
93 0 : aStream->mFinished = true;
94 0 : aStream->mTracks.AdvanceKnownTracksTime(STREAM_TIME_MAX);
95 :
96 0 : SetStreamOrderDirty();
97 : }
98 :
99 : void
100 0 : MediaStreamGraphImpl::AddStreamGraphThread(MediaStream* aStream)
101 : {
102 0 : aStream->mTracksStartTime = mProcessedTime;
103 :
104 0 : if (aStream->AsSourceStream()) {
105 0 : SourceMediaStream* source = aStream->AsSourceStream();
106 0 : TimeStamp currentTimeStamp = CurrentDriver()->GetCurrentTimeStamp();
107 : TimeStamp processedTimeStamp = currentTimeStamp +
108 0 : TimeDuration::FromSeconds(MediaTimeToSeconds(mProcessedTime - IterationEnd()));
109 0 : source->SetStreamTracksStartTimeStamp(processedTimeStamp);
110 : }
111 :
112 0 : if (aStream->IsSuspended()) {
113 0 : mSuspendedStreams.AppendElement(aStream);
114 0 : LOG(LogLevel::Debug,
115 : ("Adding media stream %p to the graph, in the suspended stream array",
116 : aStream));
117 : } else {
118 0 : mStreams.AppendElement(aStream);
119 0 : LOG(LogLevel::Debug,
120 : ("Adding media stream %p to graph %p, count %" PRIuSIZE,
121 : aStream,
122 : this,
123 : mStreams.Length()));
124 0 : LOG(LogLevel::Debug,
125 : ("Adding media stream %p to graph %p, count %" PRIuSIZE,
126 : aStream,
127 : this,
128 : mStreams.Length()));
129 : }
130 :
131 0 : SetStreamOrderDirty();
132 0 : }
133 :
134 : void
135 0 : MediaStreamGraphImpl::RemoveStreamGraphThread(MediaStream* aStream)
136 : {
137 : // Remove references in mStreamUpdates before we allow aStream to die.
138 : // Pending updates are not needed (since the main thread has already given
139 : // up the stream) so we will just drop them.
140 : {
141 0 : MonitorAutoLock lock(mMonitor);
142 0 : for (uint32_t i = 0; i < mStreamUpdates.Length(); ++i) {
143 0 : if (mStreamUpdates[i].mStream == aStream) {
144 0 : mStreamUpdates[i].mStream = nullptr;
145 : }
146 : }
147 : }
148 :
149 : // Ensure that mFirstCycleBreaker and mMixer are updated when necessary.
150 0 : SetStreamOrderDirty();
151 :
152 0 : if (aStream->IsSuspended()) {
153 0 : mSuspendedStreams.RemoveElement(aStream);
154 : } else {
155 0 : mStreams.RemoveElement(aStream);
156 : }
157 :
158 0 : LOG(LogLevel::Debug,
159 : ("Removed media stream %p from graph %p, count %" PRIuSIZE,
160 : aStream,
161 : this,
162 : mStreams.Length()));
163 0 : LOG(LogLevel::Debug,
164 : ("Removed media stream %p from graph %p, count %" PRIuSIZE,
165 : aStream,
166 : this,
167 : mStreams.Length()));
168 :
169 0 : NS_RELEASE(aStream); // probably destroying it
170 0 : }
171 :
172 : void
173 0 : MediaStreamGraphImpl::ExtractPendingInput(SourceMediaStream* aStream,
174 : GraphTime aDesiredUpToTime,
175 : bool* aEnsureNextIteration)
176 : {
177 : bool finished;
178 : {
179 0 : MutexAutoLock lock(aStream->mMutex);
180 0 : if (aStream->mPullEnabled && !aStream->mFinished &&
181 0 : !aStream->mListeners.IsEmpty()) {
182 : // Compute how much stream time we'll need assuming we don't block
183 : // the stream at all.
184 0 : StreamTime t = aStream->GraphTimeToStreamTime(aDesiredUpToTime);
185 0 : LOG(LogLevel::Verbose,
186 : ("Calling NotifyPull aStream=%p t=%f current end=%f",
187 : aStream,
188 : MediaTimeToSeconds(t),
189 : MediaTimeToSeconds(aStream->mTracks.GetEnd())));
190 0 : if (t > aStream->mTracks.GetEnd()) {
191 0 : *aEnsureNextIteration = true;
192 : #ifdef DEBUG
193 0 : if (aStream->mListeners.Length() == 0) {
194 0 : LOG(
195 : LogLevel::Error,
196 : ("No listeners in NotifyPull aStream=%p desired=%f current end=%f",
197 : aStream,
198 : MediaTimeToSeconds(t),
199 : MediaTimeToSeconds(aStream->mTracks.GetEnd())));
200 0 : aStream->DumpTrackInfo();
201 : }
202 : #endif
203 0 : for (uint32_t j = 0; j < aStream->mListeners.Length(); ++j) {
204 0 : MediaStreamListener* l = aStream->mListeners[j];
205 : {
206 0 : MutexAutoUnlock unlock(aStream->mMutex);
207 0 : l->NotifyPull(this, t);
208 : }
209 : }
210 : }
211 : }
212 0 : finished = aStream->mUpdateFinished;
213 0 : bool shouldNotifyTrackCreated = false;
214 0 : for (int32_t i = aStream->mUpdateTracks.Length() - 1; i >= 0; --i) {
215 0 : SourceMediaStream::TrackData* data = &aStream->mUpdateTracks[i];
216 0 : aStream->ApplyTrackDisabling(data->mID, data->mData);
217 : // Dealing with NotifyQueuedTrackChanges and NotifyQueuedAudioData part.
218 :
219 : // The logic is different from the manipulating of aStream->mTracks part.
220 : // So it is not combined with the manipulating of aStream->mTracks part.
221 : StreamTime offset =
222 0 : (data->mCommands & SourceMediaStream::TRACK_CREATE)
223 0 : ? data->mStart
224 0 : : aStream->mTracks.FindTrack(data->mID)->GetSegment()->GetDuration();
225 :
226 : // Audio case.
227 0 : if (data->mData->GetType() == MediaSegment::AUDIO) {
228 0 : if (data->mCommands) {
229 0 : MOZ_ASSERT(!(data->mCommands & SourceMediaStream::TRACK_UNUSED));
230 0 : for (MediaStreamListener* l : aStream->mListeners) {
231 0 : if (data->mCommands & SourceMediaStream::TRACK_END) {
232 0 : l->NotifyQueuedAudioData(this, data->mID,
233 0 : offset, *(static_cast<AudioSegment*>(data->mData.get())));
234 : }
235 0 : l->NotifyQueuedTrackChanges(this, data->mID,
236 0 : offset, static_cast<TrackEventCommand>(data->mCommands), *data->mData);
237 0 : if (data->mCommands & SourceMediaStream::TRACK_CREATE) {
238 0 : l->NotifyQueuedAudioData(this, data->mID,
239 0 : offset, *(static_cast<AudioSegment*>(data->mData.get())));
240 : }
241 : }
242 : } else {
243 0 : for (MediaStreamListener* l : aStream->mListeners) {
244 0 : l->NotifyQueuedAudioData(this, data->mID,
245 0 : offset, *(static_cast<AudioSegment*>(data->mData.get())));
246 : }
247 : }
248 : }
249 :
250 : // Video case.
251 0 : if (data->mData->GetType() == MediaSegment::VIDEO) {
252 0 : if (data->mCommands) {
253 0 : MOZ_ASSERT(!(data->mCommands & SourceMediaStream::TRACK_UNUSED));
254 0 : for (MediaStreamListener* l : aStream->mListeners) {
255 0 : l->NotifyQueuedTrackChanges(this, data->mID,
256 0 : offset, static_cast<TrackEventCommand>(data->mCommands), *data->mData);
257 : }
258 : }
259 : }
260 :
261 0 : for (TrackBound<MediaStreamTrackListener>& b : aStream->mTrackListeners) {
262 0 : if (b.mTrackID != data->mID) {
263 0 : continue;
264 : }
265 0 : b.mListener->NotifyQueuedChanges(this, offset, *data->mData);
266 0 : if (data->mCommands & SourceMediaStream::TRACK_END) {
267 0 : b.mListener->NotifyEnded();
268 : }
269 : }
270 0 : if (data->mCommands & SourceMediaStream::TRACK_CREATE) {
271 0 : MediaSegment* segment = data->mData.forget();
272 0 : LOG(LogLevel::Debug,
273 : ("SourceMediaStream %p creating track %d, start %" PRId64
274 : ", initial end %" PRId64,
275 : aStream,
276 : data->mID,
277 : int64_t(data->mStart),
278 : int64_t(segment->GetDuration())));
279 :
280 0 : data->mEndOfFlushedData += segment->GetDuration();
281 0 : aStream->mTracks.AddTrack(data->mID, data->mStart, segment);
282 : // The track has taken ownership of data->mData, so let's replace
283 : // data->mData with an empty clone.
284 0 : data->mData = segment->CreateEmptyClone();
285 0 : data->mCommands &= ~SourceMediaStream::TRACK_CREATE;
286 0 : shouldNotifyTrackCreated = true;
287 0 : } else if (data->mData->GetDuration() > 0) {
288 0 : MediaSegment* dest = aStream->mTracks.FindTrack(data->mID)->GetSegment();
289 0 : LOG(LogLevel::Verbose,
290 : ("SourceMediaStream %p track %d, advancing end from %" PRId64
291 : " to %" PRId64,
292 : aStream,
293 : data->mID,
294 : int64_t(dest->GetDuration()),
295 : int64_t(dest->GetDuration() + data->mData->GetDuration())));
296 0 : data->mEndOfFlushedData += data->mData->GetDuration();
297 0 : dest->AppendFrom(data->mData);
298 : }
299 0 : if (data->mCommands & SourceMediaStream::TRACK_END) {
300 0 : aStream->mTracks.FindTrack(data->mID)->SetEnded();
301 0 : aStream->mUpdateTracks.RemoveElementAt(i);
302 : }
303 : }
304 0 : if (shouldNotifyTrackCreated) {
305 0 : for (MediaStreamListener* l : aStream->mListeners) {
306 0 : l->NotifyFinishedTrackCreation(this);
307 : }
308 : }
309 0 : if (!aStream->mFinished) {
310 0 : aStream->mTracks.AdvanceKnownTracksTime(aStream->mUpdateKnownTracksTime);
311 : }
312 : }
313 0 : if (aStream->mTracks.GetEnd() > 0) {
314 0 : aStream->mHasCurrentData = true;
315 : }
316 0 : if (finished) {
317 0 : FinishStream(aStream);
318 : }
319 0 : }
320 :
321 : StreamTime
322 0 : MediaStreamGraphImpl::GraphTimeToStreamTimeWithBlocking(MediaStream* aStream,
323 : GraphTime aTime)
324 : {
325 0 : MOZ_ASSERT(aTime <= mStateComputedTime,
326 : "Don't ask about times where we haven't made blocking decisions yet");
327 : return std::max<StreamTime>(0,
328 0 : std::min(aTime, aStream->mStartBlocking) - aStream->mTracksStartTime);
329 : }
330 :
331 : GraphTime
332 0 : MediaStreamGraphImpl::IterationEnd() const
333 : {
334 0 : return CurrentDriver()->IterationEnd();
335 : }
336 :
337 : void
338 0 : MediaStreamGraphImpl::UpdateCurrentTimeForStreams(GraphTime aPrevCurrentTime)
339 : {
340 0 : for (MediaStream* stream : AllStreams()) {
341 0 : bool isAnyBlocked = stream->mStartBlocking < mStateComputedTime;
342 0 : bool isAnyUnblocked = stream->mStartBlocking > aPrevCurrentTime;
343 :
344 : // Calculate blocked time and fire Blocked/Unblocked events
345 0 : GraphTime blockedTime = mStateComputedTime - stream->mStartBlocking;
346 0 : NS_ASSERTION(blockedTime >= 0, "Error in blocking time");
347 0 : stream->AdvanceTimeVaryingValuesToCurrentTime(mStateComputedTime,
348 0 : blockedTime);
349 0 : LOG(LogLevel::Verbose,
350 : ("MediaStream %p bufferStartTime=%f blockedTime=%f",
351 : stream,
352 : MediaTimeToSeconds(stream->mTracksStartTime),
353 : MediaTimeToSeconds(blockedTime)));
354 0 : stream->mStartBlocking = mStateComputedTime;
355 :
356 0 : if (isAnyUnblocked && stream->mNotifiedBlocked) {
357 0 : for (uint32_t j = 0; j < stream->mListeners.Length(); ++j) {
358 0 : MediaStreamListener* l = stream->mListeners[j];
359 0 : l->NotifyBlockingChanged(this, MediaStreamListener::UNBLOCKED);
360 : }
361 0 : stream->mNotifiedBlocked = false;
362 : }
363 0 : if (isAnyBlocked && !stream->mNotifiedBlocked) {
364 0 : for (uint32_t j = 0; j < stream->mListeners.Length(); ++j) {
365 0 : MediaStreamListener* l = stream->mListeners[j];
366 0 : l->NotifyBlockingChanged(this, MediaStreamListener::BLOCKED);
367 : }
368 0 : stream->mNotifiedBlocked = true;
369 : }
370 :
371 0 : if (isAnyUnblocked) {
372 0 : NS_ASSERTION(!stream->mNotifiedFinished,
373 : "Shouldn't have already notified of finish *and* have output!");
374 0 : for (uint32_t j = 0; j < stream->mListeners.Length(); ++j) {
375 0 : MediaStreamListener* l = stream->mListeners[j];
376 0 : l->NotifyOutput(this, mProcessedTime);
377 : }
378 : }
379 :
380 : // The stream is fully finished when all of its track data has been played
381 : // out.
382 0 : if (stream->mFinished && !stream->mNotifiedFinished &&
383 0 : mProcessedTime >=
384 0 : stream->StreamTimeToGraphTime(stream->GetStreamTracks().GetAllTracksEnd())) {
385 0 : stream->mNotifiedFinished = true;
386 0 : SetStreamOrderDirty();
387 0 : for (uint32_t j = 0; j < stream->mListeners.Length(); ++j) {
388 0 : MediaStreamListener* l = stream->mListeners[j];
389 0 : l->NotifyEvent(this, MediaStreamGraphEvent::EVENT_FINISHED);
390 : }
391 : }
392 : }
393 0 : }
394 :
395 : template<typename C, typename Chunk>
396 : void
397 0 : MediaStreamGraphImpl::ProcessChunkMetadataForInterval(MediaStream* aStream,
398 : TrackID aTrackID,
399 : C& aSegment,
400 : StreamTime aStart,
401 : StreamTime aEnd)
402 : {
403 0 : MOZ_ASSERT(aStream);
404 0 : MOZ_ASSERT(IsTrackIDExplicit(aTrackID));
405 :
406 0 : StreamTime offset = 0;
407 0 : for (typename C::ConstChunkIterator chunk(aSegment);
408 0 : !chunk.IsEnded(); chunk.Next()) {
409 0 : if (offset >= aEnd) {
410 0 : break;
411 : }
412 0 : offset += chunk->GetDuration();
413 0 : if (chunk->IsNull() || offset < aStart) {
414 0 : continue;
415 : }
416 0 : PrincipalHandle principalHandle = chunk->GetPrincipalHandle();
417 0 : if (principalHandle != aSegment.GetLastPrincipalHandle()) {
418 0 : aSegment.SetLastPrincipalHandle(principalHandle);
419 0 : LOG(LogLevel::Debug,
420 : ("MediaStream %p track %d, principalHandle "
421 : "changed in %sChunk with duration %lld",
422 : aStream,
423 : aTrackID,
424 : aSegment.GetType() == MediaSegment::AUDIO ? "Audio" : "Video",
425 : (long long)chunk->GetDuration()));
426 0 : for (const TrackBound<MediaStreamTrackListener>& listener :
427 : aStream->mTrackListeners) {
428 0 : if (listener.mTrackID == aTrackID) {
429 0 : listener.mListener->NotifyPrincipalHandleChanged(this, principalHandle);
430 : }
431 : }
432 : }
433 : }
434 0 : }
435 :
436 : void
437 0 : MediaStreamGraphImpl::ProcessChunkMetadata(GraphTime aPrevCurrentTime)
438 : {
439 0 : for (MediaStream* stream : AllStreams()) {
440 0 : StreamTime iterationStart = stream->GraphTimeToStreamTime(aPrevCurrentTime);
441 0 : StreamTime iterationEnd = stream->GraphTimeToStreamTime(mProcessedTime);
442 0 : for (StreamTracks::TrackIter tracks(stream->mTracks);
443 0 : !tracks.IsEnded(); tracks.Next()) {
444 0 : MediaSegment* segment = tracks->GetSegment();
445 0 : if (!segment) {
446 0 : continue;
447 : }
448 0 : if (tracks->GetType() == MediaSegment::AUDIO) {
449 0 : AudioSegment* audio = static_cast<AudioSegment*>(segment);
450 0 : ProcessChunkMetadataForInterval<AudioSegment, AudioChunk>(
451 0 : stream, tracks->GetID(), *audio, iterationStart, iterationEnd);
452 0 : } else if (tracks->GetType() == MediaSegment::VIDEO) {
453 0 : VideoSegment* video = static_cast<VideoSegment*>(segment);
454 0 : ProcessChunkMetadataForInterval<VideoSegment, VideoChunk>(
455 0 : stream, tracks->GetID(), *video, iterationStart, iterationEnd);
456 : } else {
457 0 : MOZ_CRASH("Unknown track type");
458 : }
459 : }
460 : }
461 0 : }
462 :
463 : GraphTime
464 0 : MediaStreamGraphImpl::WillUnderrun(MediaStream* aStream,
465 : GraphTime aEndBlockingDecisions)
466 : {
467 : // Finished streams can't underrun. ProcessedMediaStreams also can't cause
468 : // underrun currently, since we'll always be able to produce data for them
469 : // unless they block on some other stream.
470 0 : if (aStream->mFinished || aStream->AsProcessedStream()) {
471 0 : return aEndBlockingDecisions;
472 : }
473 : // This stream isn't finished or suspended. We don't need to call
474 : // StreamTimeToGraphTime since an underrun is the only thing that can block
475 : // it.
476 0 : GraphTime bufferEnd = aStream->GetTracksEnd() + aStream->mTracksStartTime;
477 : #ifdef DEBUG
478 0 : if (bufferEnd < mProcessedTime) {
479 0 : LOG(LogLevel::Error,
480 : ("MediaStream %p underrun, "
481 : "bufferEnd %f < mProcessedTime %f (%" PRId64 " < %" PRId64
482 : "), Streamtime %" PRId64,
483 : aStream,
484 : MediaTimeToSeconds(bufferEnd),
485 : MediaTimeToSeconds(mProcessedTime),
486 : bufferEnd,
487 : mProcessedTime,
488 : aStream->GetTracksEnd()));
489 0 : aStream->DumpTrackInfo();
490 0 : NS_ASSERTION(bufferEnd >= mProcessedTime, "Buffer underran");
491 : }
492 : #endif
493 0 : return std::min(bufferEnd, aEndBlockingDecisions);
494 : }
495 :
496 : namespace {
497 : // Value of mCycleMarker for unvisited streams in cycle detection.
498 : const uint32_t NOT_VISITED = UINT32_MAX;
499 : // Value of mCycleMarker for ordered streams in muted cycles.
500 : const uint32_t IN_MUTED_CYCLE = 1;
501 : } // namespace
502 :
503 : bool
504 0 : MediaStreamGraphImpl::AudioTrackPresent(bool& aNeedsAEC)
505 : {
506 0 : AssertOnGraphThreadOrNotRunning();
507 :
508 0 : bool audioTrackPresent = false;
509 0 : for (uint32_t i = 0; i < mStreams.Length() && audioTrackPresent == false; ++i) {
510 0 : MediaStream* stream = mStreams[i];
511 0 : SourceMediaStream* source = stream->AsSourceStream();
512 : #ifdef MOZ_WEBRTC
513 0 : if (source && source->NeedsMixing()) {
514 0 : aNeedsAEC = true;
515 : }
516 : #endif
517 : // If this is a AudioNodeStream, force a AudioCallbackDriver.
518 0 : if (stream->AsAudioNodeStream()) {
519 0 : audioTrackPresent = true;
520 : } else {
521 0 : for (StreamTracks::TrackIter tracks(stream->GetStreamTracks(), MediaSegment::AUDIO);
522 0 : !tracks.IsEnded(); tracks.Next()) {
523 0 : audioTrackPresent = true;
524 : }
525 : }
526 0 : if (source) {
527 0 : audioTrackPresent = source->HasPendingAudioTrack();
528 : }
529 : }
530 :
531 : // XXX For some reason, there are race conditions when starting an audio input where
532 : // we find no active audio tracks. In any case, if we have an active audio input we
533 : // should not allow a switch back to a SystemClockDriver
534 0 : if (!audioTrackPresent && mInputDeviceUsers.Count() != 0) {
535 0 : NS_WARNING("No audio tracks, but full-duplex audio is enabled!!!!!");
536 0 : audioTrackPresent = true;
537 : #ifdef MOZ_WEBRTC
538 0 : aNeedsAEC = true;
539 : #endif
540 : }
541 :
542 0 : return audioTrackPresent;
543 : }
544 :
545 : void
546 0 : MediaStreamGraphImpl::UpdateStreamOrder()
547 : {
548 0 : bool shouldAEC = false;
549 0 : bool audioTrackPresent = AudioTrackPresent(shouldAEC);
550 :
551 : // Note that this looks for any audio streams, input or output, and switches to a
552 : // SystemClockDriver if there are none. However, if another is already pending, let that
553 : // switch happen.
554 :
555 0 : if (!audioTrackPresent && mRealtime &&
556 0 : CurrentDriver()->AsAudioCallbackDriver()) {
557 0 : MonitorAutoLock mon(mMonitor);
558 0 : if (CurrentDriver()->AsAudioCallbackDriver()->IsStarted() &&
559 0 : !(CurrentDriver()->Switching())) {
560 0 : if (mLifecycleState == LIFECYCLE_RUNNING) {
561 0 : SystemClockDriver* driver = new SystemClockDriver(this);
562 0 : CurrentDriver()->SwitchAtNextIteration(driver);
563 : }
564 : }
565 : }
566 :
567 0 : bool switching = false;
568 : {
569 0 : MonitorAutoLock mon(mMonitor);
570 0 : switching = CurrentDriver()->Switching();
571 : }
572 :
573 0 : if (audioTrackPresent && mRealtime &&
574 0 : !CurrentDriver()->AsAudioCallbackDriver() &&
575 0 : !switching) {
576 0 : MonitorAutoLock mon(mMonitor);
577 0 : if (mLifecycleState == LIFECYCLE_RUNNING) {
578 0 : AudioCallbackDriver* driver = new AudioCallbackDriver(this);
579 0 : CurrentDriver()->SwitchAtNextIteration(driver);
580 : }
581 : }
582 :
583 0 : if (!mStreamOrderDirty) {
584 0 : return;
585 : }
586 :
587 0 : mStreamOrderDirty = false;
588 :
589 : // The algorithm for finding cycles is based on Tim Leslie's iterative
590 : // implementation [1][2] of Pearce's variant [3] of Tarjan's strongly
591 : // connected components (SCC) algorithm. There are variations (a) to
592 : // distinguish whether streams in SCCs of size 1 are in a cycle and (b) to
593 : // re-run the algorithm over SCCs with breaks at DelayNodes.
594 : //
595 : // [1] http://www.timl.id.au/?p=327
596 : // [2] https://github.com/scipy/scipy/blob/e2c502fca/scipy/sparse/csgraph/_traversal.pyx#L582
597 : // [3] http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.102.1707
598 : //
599 : // There are two stacks. One for the depth-first search (DFS),
600 0 : mozilla::LinkedList<MediaStream> dfsStack;
601 : // and another for streams popped from the DFS stack, but still being
602 : // considered as part of SCCs involving streams on the stack.
603 0 : mozilla::LinkedList<MediaStream> sccStack;
604 :
605 : // An index into mStreams for the next stream found with no unsatisfied
606 : // upstream dependencies.
607 0 : uint32_t orderedStreamCount = 0;
608 :
609 0 : for (uint32_t i = 0; i < mStreams.Length(); ++i) {
610 0 : MediaStream* s = mStreams[i];
611 0 : ProcessedMediaStream* ps = s->AsProcessedStream();
612 0 : if (ps) {
613 : // The dfsStack initially contains a list of all processed streams in
614 : // unchanged order.
615 0 : dfsStack.insertBack(s);
616 0 : ps->mCycleMarker = NOT_VISITED;
617 : } else {
618 : // SourceMediaStreams have no inputs and so can be ordered now.
619 0 : mStreams[orderedStreamCount] = s;
620 0 : ++orderedStreamCount;
621 : }
622 : }
623 :
624 : // mNextStackMarker corresponds to "index" in Tarjan's algorithm. It is a
625 : // counter to label mCycleMarker on the next visited stream in the DFS
626 : // uniquely in the set of visited streams that are still being considered.
627 : //
628 : // In this implementation, the counter descends so that the values are
629 : // strictly greater than the values that mCycleMarker takes when the stream
630 : // has been ordered (0 or IN_MUTED_CYCLE).
631 : //
632 : // Each new stream labelled, as the DFS searches upstream, receives a value
633 : // less than those used for all other streams being considered.
634 0 : uint32_t nextStackMarker = NOT_VISITED - 1;
635 : // Reset list of DelayNodes in cycles stored at the tail of mStreams.
636 0 : mFirstCycleBreaker = mStreams.Length();
637 :
638 : // Rearrange dfsStack order as required to DFS upstream and pop streams
639 : // in processing order to place in mStreams.
640 0 : while (auto ps = static_cast<ProcessedMediaStream*>(dfsStack.getFirst())) {
641 0 : const auto& inputs = ps->mInputs;
642 0 : MOZ_ASSERT(ps->AsProcessedStream());
643 0 : if (ps->mCycleMarker == NOT_VISITED) {
644 : // Record the position on the visited stack, so that any searches
645 : // finding this stream again know how much of the stack is in the cycle.
646 0 : ps->mCycleMarker = nextStackMarker;
647 0 : --nextStackMarker;
648 : // Not-visited input streams should be processed first.
649 : // SourceMediaStreams have already been ordered.
650 0 : for (uint32_t i = inputs.Length(); i--; ) {
651 0 : if (inputs[i]->mSource->IsSuspended()) {
652 0 : continue;
653 : }
654 0 : auto input = inputs[i]->mSource->AsProcessedStream();
655 0 : if (input && input->mCycleMarker == NOT_VISITED) {
656 : // It can be that this stream has an input which is from a suspended
657 : // AudioContext.
658 0 : if (input->isInList()) {
659 0 : input->remove();
660 0 : dfsStack.insertFront(input);
661 : }
662 : }
663 : }
664 0 : continue;
665 : }
666 :
667 : // Returning from DFS. Pop from dfsStack.
668 0 : ps->remove();
669 :
670 : // cycleStackMarker keeps track of the highest marker value on any
671 : // upstream stream, if any, found receiving input, directly or indirectly,
672 : // from the visited stack (and so from |ps|, making a cycle). In a
673 : // variation from Tarjan's SCC algorithm, this does not include |ps|
674 : // unless it is part of the cycle.
675 0 : uint32_t cycleStackMarker = 0;
676 0 : for (uint32_t i = inputs.Length(); i--; ) {
677 0 : if (inputs[i]->mSource->IsSuspended()) {
678 0 : continue;
679 : }
680 0 : auto input = inputs[i]->mSource->AsProcessedStream();
681 0 : if (input) {
682 0 : cycleStackMarker = std::max(cycleStackMarker, input->mCycleMarker);
683 : }
684 : }
685 :
686 0 : if (cycleStackMarker <= IN_MUTED_CYCLE) {
687 : // All inputs have been ordered and their stack markers have been removed.
688 : // This stream is not part of a cycle. It can be processed next.
689 0 : ps->mCycleMarker = 0;
690 0 : mStreams[orderedStreamCount] = ps;
691 0 : ++orderedStreamCount;
692 0 : continue;
693 : }
694 :
695 : // A cycle has been found. Record this stream for ordering when all
696 : // streams in this SCC have been popped from the DFS stack.
697 0 : sccStack.insertFront(ps);
698 :
699 0 : if (cycleStackMarker > ps->mCycleMarker) {
700 : // Cycles have been found that involve streams that remain on the stack.
701 : // Leave mCycleMarker indicating the most downstream (last) stream on
702 : // the stack known to be part of this SCC. In this way, any searches on
703 : // other paths that find |ps| will know (without having to traverse from
704 : // this stream again) that they are part of this SCC (i.e. part of an
705 : // intersecting cycle).
706 0 : ps->mCycleMarker = cycleStackMarker;
707 0 : continue;
708 : }
709 :
710 : // |ps| is the root of an SCC involving no other streams on dfsStack, the
711 : // complete SCC has been recorded, and streams in this SCC are part of at
712 : // least one cycle.
713 0 : MOZ_ASSERT(cycleStackMarker == ps->mCycleMarker);
714 : // If there are DelayNodes in this SCC, then they may break the cycles.
715 0 : bool haveDelayNode = false;
716 0 : auto next = sccStack.getFirst();
717 : // Streams in this SCC are identified by mCycleMarker <= cycleStackMarker.
718 : // (There may be other streams later in sccStack from other incompletely
719 : // searched SCCs, involving streams still on dfsStack.)
720 : //
721 : // DelayNodes in cycles must behave differently from those not in cycles,
722 : // so all DelayNodes in the SCC must be identified.
723 0 : while (next && static_cast<ProcessedMediaStream*>(next)->
724 0 : mCycleMarker <= cycleStackMarker) {
725 0 : auto ns = next->AsAudioNodeStream();
726 : // Get next before perhaps removing from list below.
727 0 : next = next->getNext();
728 0 : if (ns && ns->Engine()->AsDelayNodeEngine()) {
729 0 : haveDelayNode = true;
730 : // DelayNodes break cycles by producing their output in a
731 : // preprocessing phase; they do not need to be ordered before their
732 : // consumers. Order them at the tail of mStreams so that they can be
733 : // handled specially. Do so now, so that DFS ignores them.
734 0 : ns->remove();
735 0 : ns->mCycleMarker = 0;
736 0 : --mFirstCycleBreaker;
737 0 : mStreams[mFirstCycleBreaker] = ns;
738 : }
739 : }
740 0 : auto after_scc = next;
741 0 : while ((next = sccStack.getFirst()) != after_scc) {
742 0 : next->remove();
743 0 : auto removed = static_cast<ProcessedMediaStream*>(next);
744 0 : if (haveDelayNode) {
745 : // Return streams to the DFS stack again (to order and detect cycles
746 : // without delayNodes). Any of these streams that are still inputs
747 : // for streams on the visited stack must be returned to the front of
748 : // the stack to be ordered before their dependents. We know that none
749 : // of these streams need input from streams on the visited stack, so
750 : // they can all be searched and ordered before the current stack head
751 : // is popped.
752 0 : removed->mCycleMarker = NOT_VISITED;
753 0 : dfsStack.insertFront(removed);
754 : } else {
755 : // Streams in cycles without any DelayNodes must be muted, and so do
756 : // not need input and can be ordered now. They must be ordered before
757 : // their consumers so that their muted output is available.
758 0 : removed->mCycleMarker = IN_MUTED_CYCLE;
759 0 : mStreams[orderedStreamCount] = removed;
760 0 : ++orderedStreamCount;
761 : }
762 : }
763 0 : }
764 :
765 0 : MOZ_ASSERT(orderedStreamCount == mFirstCycleBreaker);
766 : }
767 :
768 : void
769 0 : MediaStreamGraphImpl::NotifyHasCurrentData(MediaStream* aStream)
770 : {
771 0 : if (!aStream->mNotifiedHasCurrentData && aStream->mHasCurrentData) {
772 0 : for (uint32_t j = 0; j < aStream->mListeners.Length(); ++j) {
773 0 : MediaStreamListener* l = aStream->mListeners[j];
774 0 : l->NotifyHasCurrentData(this);
775 : }
776 0 : aStream->mNotifiedHasCurrentData = true;
777 : }
778 0 : }
779 :
780 : void
781 0 : MediaStreamGraphImpl::CreateOrDestroyAudioStreams(MediaStream* aStream)
782 : {
783 0 : MOZ_ASSERT(mRealtime, "Should only attempt to create audio streams in real-time mode");
784 :
785 0 : if (aStream->mAudioOutputs.IsEmpty()) {
786 0 : aStream->mAudioOutputStreams.Clear();
787 0 : return;
788 : }
789 :
790 0 : if (!aStream->GetStreamTracks().GetAndResetTracksDirty() &&
791 0 : !aStream->mAudioOutputStreams.IsEmpty()) {
792 0 : return;
793 : }
794 :
795 0 : LOG(LogLevel::Debug,
796 : ("Updating AudioOutputStreams for MediaStream %p", aStream));
797 :
798 0 : AutoTArray<bool,2> audioOutputStreamsFound;
799 0 : for (uint32_t i = 0; i < aStream->mAudioOutputStreams.Length(); ++i) {
800 0 : audioOutputStreamsFound.AppendElement(false);
801 : }
802 :
803 0 : for (StreamTracks::TrackIter tracks(aStream->GetStreamTracks(), MediaSegment::AUDIO);
804 0 : !tracks.IsEnded(); tracks.Next()) {
805 : uint32_t i;
806 0 : for (i = 0; i < audioOutputStreamsFound.Length(); ++i) {
807 0 : if (aStream->mAudioOutputStreams[i].mTrackID == tracks->GetID()) {
808 0 : break;
809 : }
810 : }
811 0 : if (i < audioOutputStreamsFound.Length()) {
812 0 : audioOutputStreamsFound[i] = true;
813 : } else {
814 : MediaStream::AudioOutputStream* audioOutputStream =
815 0 : aStream->mAudioOutputStreams.AppendElement();
816 0 : audioOutputStream->mAudioPlaybackStartTime = mProcessedTime;
817 0 : audioOutputStream->mBlockedAudioTime = 0;
818 0 : audioOutputStream->mLastTickWritten = 0;
819 0 : audioOutputStream->mTrackID = tracks->GetID();
820 :
821 0 : bool switching = false;
822 :
823 : {
824 0 : MonitorAutoLock lock(mMonitor);
825 0 : switching = CurrentDriver()->Switching();
826 : }
827 :
828 0 : if (!CurrentDriver()->AsAudioCallbackDriver() &&
829 0 : !switching) {
830 0 : MonitorAutoLock mon(mMonitor);
831 0 : if (mLifecycleState == LIFECYCLE_RUNNING) {
832 0 : AudioCallbackDriver* driver = new AudioCallbackDriver(this);
833 0 : CurrentDriver()->SwitchAtNextIteration(driver);
834 : }
835 : }
836 : }
837 : }
838 :
839 0 : for (int32_t i = audioOutputStreamsFound.Length() - 1; i >= 0; --i) {
840 0 : if (!audioOutputStreamsFound[i]) {
841 0 : aStream->mAudioOutputStreams.RemoveElementAt(i);
842 : }
843 : }
844 : }
845 :
846 : StreamTime
847 0 : MediaStreamGraphImpl::PlayAudio(MediaStream* aStream)
848 : {
849 0 : MOZ_ASSERT(mRealtime, "Should only attempt to play audio in realtime mode");
850 :
851 0 : float volume = 0.0f;
852 0 : for (uint32_t i = 0; i < aStream->mAudioOutputs.Length(); ++i) {
853 0 : volume += aStream->mAudioOutputs[i].mVolume;
854 : }
855 :
856 0 : StreamTime ticksWritten = 0;
857 :
858 0 : for (uint32_t i = 0; i < aStream->mAudioOutputStreams.Length(); ++i) {
859 0 : ticksWritten = 0;
860 :
861 0 : MediaStream::AudioOutputStream& audioOutput = aStream->mAudioOutputStreams[i];
862 0 : StreamTracks::Track* track = aStream->mTracks.FindTrack(audioOutput.mTrackID);
863 0 : AudioSegment* audio = track->Get<AudioSegment>();
864 0 : AudioSegment output;
865 :
866 0 : StreamTime offset = aStream->GraphTimeToStreamTime(mProcessedTime);
867 :
868 : // We don't update aStream->mTracksStartTime here to account for time spent
869 : // blocked. Instead, we'll update it in UpdateCurrentTimeForStreams after
870 : // the blocked period has completed. But we do need to make sure we play
871 : // from the right offsets in the stream buffer, even if we've already
872 : // written silence for some amount of blocked time after the current time.
873 0 : GraphTime t = mProcessedTime;
874 0 : while (t < mStateComputedTime) {
875 0 : bool blocked = t >= aStream->mStartBlocking;
876 0 : GraphTime end = blocked ? mStateComputedTime : aStream->mStartBlocking;
877 0 : NS_ASSERTION(end <= mStateComputedTime, "mStartBlocking is wrong!");
878 :
879 : // Check how many ticks of sound we can provide if we are blocked some
880 : // time in the middle of this cycle.
881 0 : StreamTime toWrite = end - t;
882 :
883 0 : if (blocked) {
884 0 : output.InsertNullDataAtStart(toWrite);
885 0 : ticksWritten += toWrite;
886 0 : LOG(LogLevel::Verbose,
887 : ("MediaStream %p writing %" PRId64 " blocking-silence samples for "
888 : "%f to %f (%" PRId64 " to %" PRId64 ")",
889 : aStream,
890 : toWrite,
891 : MediaTimeToSeconds(t),
892 : MediaTimeToSeconds(end),
893 : offset,
894 : offset + toWrite));
895 : } else {
896 0 : StreamTime endTicksNeeded = offset + toWrite;
897 0 : StreamTime endTicksAvailable = audio->GetDuration();
898 :
899 0 : if (endTicksNeeded <= endTicksAvailable) {
900 0 : LOG(LogLevel::Verbose,
901 : ("MediaStream %p writing %" PRId64 " samples for %f to %f "
902 : "(samples %" PRId64 " to %" PRId64 ")",
903 : aStream,
904 : toWrite,
905 : MediaTimeToSeconds(t),
906 : MediaTimeToSeconds(end),
907 : offset,
908 : endTicksNeeded));
909 0 : output.AppendSlice(*audio, offset, endTicksNeeded);
910 0 : ticksWritten += toWrite;
911 0 : offset = endTicksNeeded;
912 : } else {
913 : // MOZ_ASSERT(track->IsEnded(), "Not enough data, and track not ended.");
914 : // If we are at the end of the track, maybe write the remaining
915 : // samples, and pad with/output silence.
916 0 : if (endTicksNeeded > endTicksAvailable &&
917 : offset < endTicksAvailable) {
918 0 : output.AppendSlice(*audio, offset, endTicksAvailable);
919 0 : LOG(LogLevel::Verbose,
920 : ("MediaStream %p writing %" PRId64 " samples for %f to %f "
921 : "(samples %" PRId64 " to %" PRId64 ")",
922 : aStream,
923 : toWrite,
924 : MediaTimeToSeconds(t),
925 : MediaTimeToSeconds(end),
926 : offset,
927 : endTicksNeeded));
928 0 : uint32_t available = endTicksAvailable - offset;
929 0 : ticksWritten += available;
930 0 : toWrite -= available;
931 0 : offset = endTicksAvailable;
932 : }
933 0 : output.AppendNullData(toWrite);
934 0 : LOG(LogLevel::Verbose,
935 : ("MediaStream %p writing %" PRId64 " padding slsamples for %f to "
936 : "%f (samples %" PRId64 " to %" PRId64 ")",
937 : aStream,
938 : toWrite,
939 : MediaTimeToSeconds(t),
940 : MediaTimeToSeconds(end),
941 : offset,
942 : endTicksNeeded));
943 0 : ticksWritten += toWrite;
944 : }
945 0 : output.ApplyVolume(volume);
946 : }
947 0 : t = end;
948 : }
949 0 : audioOutput.mLastTickWritten = offset;
950 :
951 : // Need unique id for stream & track - and we want it to match the inserter
952 0 : output.WriteTo(LATENCY_STREAM_ID(aStream, track->GetID()),
953 : mMixer, AudioChannelCount(),
954 0 : mSampleRate);
955 : }
956 0 : return ticksWritten;
957 : }
958 :
959 : void
960 0 : MediaStreamGraphImpl::OpenAudioInputImpl(int aID,
961 : AudioDataListener *aListener)
962 : {
963 : // Bug 1238038 Need support for multiple mics at once
964 0 : if (mInputDeviceUsers.Count() > 0 &&
965 0 : !mInputDeviceUsers.Get(aListener, nullptr)) {
966 0 : NS_ASSERTION(false, "Input from multiple mics not yet supported; bug 1238038");
967 : // Need to support separate input-only AudioCallback drivers; they'll
968 : // call us back on "other" threads. We will need to echo-cancel them, though.
969 0 : return;
970 : }
971 0 : mInputWanted = true;
972 :
973 : // Add to count of users for this ID.
974 : // XXX Since we can't rely on IDs staying valid (ugh), use the listener as
975 : // a stand-in for the ID. Fix as part of support for multiple-captures
976 : // (Bug 1238038)
977 0 : uint32_t count = 0;
978 0 : mInputDeviceUsers.Get(aListener, &count); // ok if this fails
979 0 : count++;
980 0 : mInputDeviceUsers.Put(aListener, count); // creates a new entry in the hash if needed
981 :
982 0 : if (count == 1) { // first open for this listener
983 : // aID is a cubeb_devid, and we assume that opaque ptr is valid until
984 : // we close cubeb.
985 0 : mInputDeviceID = aID;
986 0 : mAudioInputs.AppendElement(aListener); // always monitor speaker data
987 :
988 : // Switch Drivers since we're adding input (to input-only or full-duplex)
989 0 : MonitorAutoLock mon(mMonitor);
990 0 : if (mLifecycleState == LIFECYCLE_RUNNING) {
991 0 : AudioCallbackDriver* driver = new AudioCallbackDriver(this);
992 0 : driver->SetMicrophoneActive(true);
993 0 : LOG(
994 : LogLevel::Debug,
995 : ("OpenAudioInput: starting new AudioCallbackDriver(input) %p", driver));
996 0 : LOG(
997 : LogLevel::Debug,
998 : ("OpenAudioInput: starting new AudioCallbackDriver(input) %p", driver));
999 0 : driver->SetInputListener(aListener);
1000 0 : CurrentDriver()->SwitchAtNextIteration(driver);
1001 : } else {
1002 0 : LOG(LogLevel::Error, ("OpenAudioInput in shutdown!"));
1003 0 : LOG(LogLevel::Debug, ("OpenAudioInput in shutdown!"));
1004 0 : NS_ASSERTION(false, "Can't open cubeb inputs in shutdown");
1005 : }
1006 : }
1007 : }
1008 :
1009 : nsresult
1010 0 : MediaStreamGraphImpl::OpenAudioInput(int aID,
1011 : AudioDataListener *aListener)
1012 : {
1013 : // So, so, so annoying. Can't AppendMessage except on Mainthread
1014 0 : if (!NS_IsMainThread()) {
1015 : RefPtr<nsIRunnable> runnable =
1016 0 : WrapRunnable(this,
1017 : &MediaStreamGraphImpl::OpenAudioInput,
1018 : aID,
1019 0 : RefPtr<AudioDataListener>(aListener));
1020 0 : mAbstractMainThread->Dispatch(runnable.forget());
1021 0 : return NS_OK;
1022 : }
1023 0 : class Message : public ControlMessage {
1024 : public:
1025 0 : Message(MediaStreamGraphImpl *aGraph, int aID,
1026 0 : AudioDataListener *aListener) :
1027 0 : ControlMessage(nullptr), mGraph(aGraph), mID(aID), mListener(aListener) {}
1028 0 : virtual void Run()
1029 : {
1030 0 : mGraph->OpenAudioInputImpl(mID, mListener);
1031 0 : }
1032 : MediaStreamGraphImpl *mGraph;
1033 : int mID;
1034 : RefPtr<AudioDataListener> mListener;
1035 : };
1036 : // XXX Check not destroyed!
1037 0 : this->AppendMessage(MakeUnique<Message>(this, aID, aListener));
1038 0 : return NS_OK;
1039 : }
1040 :
1041 : void
1042 0 : MediaStreamGraphImpl::CloseAudioInputImpl(AudioDataListener *aListener)
1043 : {
1044 : uint32_t count;
1045 0 : DebugOnly<bool> result = mInputDeviceUsers.Get(aListener, &count);
1046 0 : MOZ_ASSERT(result);
1047 0 : if (--count > 0) {
1048 0 : mInputDeviceUsers.Put(aListener, count);
1049 0 : return; // still in use
1050 : }
1051 0 : mInputDeviceUsers.Remove(aListener);
1052 0 : mInputDeviceID = -1;
1053 0 : mInputWanted = false;
1054 0 : AudioCallbackDriver *driver = CurrentDriver()->AsAudioCallbackDriver();
1055 0 : if (driver) {
1056 0 : driver->RemoveInputListener(aListener);
1057 : }
1058 0 : mAudioInputs.RemoveElement(aListener);
1059 :
1060 : // Switch Drivers since we're adding or removing an input (to nothing/system or output only)
1061 0 : bool shouldAEC = false;
1062 0 : bool audioTrackPresent = AudioTrackPresent(shouldAEC);
1063 :
1064 0 : MonitorAutoLock mon(mMonitor);
1065 0 : if (mLifecycleState == LIFECYCLE_RUNNING) {
1066 : GraphDriver* driver;
1067 0 : if (audioTrackPresent) {
1068 : // We still have audio output
1069 0 : LOG(LogLevel::Debug, ("CloseInput: output present (AudioCallback)"));
1070 :
1071 0 : driver = new AudioCallbackDriver(this);
1072 0 : CurrentDriver()->SwitchAtNextIteration(driver);
1073 0 : } else if (CurrentDriver()->AsAudioCallbackDriver()) {
1074 0 : LOG(LogLevel::Debug,
1075 : ("CloseInput: no output present (SystemClockCallback)"));
1076 :
1077 0 : driver = new SystemClockDriver(this);
1078 0 : CurrentDriver()->SwitchAtNextIteration(driver);
1079 : } // else SystemClockDriver->SystemClockDriver, no switch
1080 : }
1081 : }
1082 :
1083 : void
1084 0 : MediaStreamGraphImpl::CloseAudioInput(AudioDataListener *aListener)
1085 : {
1086 : // So, so, so annoying. Can't AppendMessage except on Mainthread
1087 0 : if (!NS_IsMainThread()) {
1088 : RefPtr<nsIRunnable> runnable =
1089 0 : WrapRunnable(this,
1090 : &MediaStreamGraphImpl::CloseAudioInput,
1091 0 : RefPtr<AudioDataListener>(aListener));
1092 0 : mAbstractMainThread->Dispatch(runnable.forget());
1093 0 : return;
1094 : }
1095 0 : class Message : public ControlMessage {
1096 : public:
1097 0 : Message(MediaStreamGraphImpl *aGraph, AudioDataListener *aListener) :
1098 0 : ControlMessage(nullptr), mGraph(aGraph), mListener(aListener) {}
1099 0 : virtual void Run()
1100 : {
1101 0 : mGraph->CloseAudioInputImpl(mListener);
1102 0 : }
1103 : MediaStreamGraphImpl *mGraph;
1104 : RefPtr<AudioDataListener> mListener;
1105 : };
1106 0 : this->AppendMessage(MakeUnique<Message>(this, aListener));
1107 : }
1108 :
1109 : // All AudioInput listeners get the same speaker data (at least for now).
1110 : void
1111 0 : MediaStreamGraph::NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames,
1112 : TrackRate aRate, uint32_t aChannels)
1113 : {
1114 0 : for (auto& listener : mAudioInputs) {
1115 0 : listener->NotifyOutputData(this, aBuffer, aFrames, aRate, aChannels);
1116 : }
1117 0 : }
1118 :
1119 : void
1120 0 : MediaStreamGraph::AssertOnGraphThreadOrNotRunning() const
1121 : {
1122 : // either we're on the right thread (and calling CurrentDriver() is safe),
1123 : // or we're going to assert anyways, so don't cross-check CurrentDriver
1124 : #ifdef DEBUG
1125 : MediaStreamGraphImpl const * graph =
1126 0 : static_cast<MediaStreamGraphImpl const *>(this);
1127 : // if all the safety checks fail, assert we own the monitor
1128 0 : if (!graph->mDriver->OnThread()) {
1129 0 : if (!(graph->mDetectedNotRunning &&
1130 0 : graph->mLifecycleState > MediaStreamGraphImpl::LIFECYCLE_RUNNING &&
1131 0 : NS_IsMainThread())) {
1132 0 : graph->mMonitor.AssertCurrentThreadOwns();
1133 : }
1134 : }
1135 : #endif
1136 0 : }
1137 :
1138 : bool
1139 0 : MediaStreamGraphImpl::ShouldUpdateMainThread()
1140 : {
1141 0 : if (mRealtime) {
1142 0 : return true;
1143 : }
1144 :
1145 0 : TimeStamp now = TimeStamp::Now();
1146 0 : if ((now - mLastMainThreadUpdate).ToMilliseconds() > CurrentDriver()->IterationDuration()) {
1147 0 : mLastMainThreadUpdate = now;
1148 0 : return true;
1149 : }
1150 0 : return false;
1151 : }
1152 :
1153 : void
1154 0 : MediaStreamGraphImpl::PrepareUpdatesToMainThreadState(bool aFinalUpdate)
1155 : {
1156 0 : mMonitor.AssertCurrentThreadOwns();
1157 :
1158 : // We don't want to frequently update the main thread about timing update
1159 : // when we are not running in realtime.
1160 0 : if (aFinalUpdate || ShouldUpdateMainThread()) {
1161 : // Strip updates that will be obsoleted below, so as to keep the length of
1162 : // mStreamUpdates sane.
1163 0 : size_t keptUpdateCount = 0;
1164 0 : for (size_t i = 0; i < mStreamUpdates.Length(); ++i) {
1165 0 : MediaStream* stream = mStreamUpdates[i].mStream;
1166 : // RemoveStreamGraphThread() clears mStream in updates for
1167 : // streams that are removed from the graph.
1168 0 : MOZ_ASSERT(!stream || stream->GraphImpl() == this);
1169 0 : if (!stream || stream->MainThreadNeedsUpdates()) {
1170 : // Discard this update as it has either been cleared when the stream
1171 : // was destroyed or there will be a newer update below.
1172 0 : continue;
1173 : }
1174 0 : if (keptUpdateCount != i) {
1175 0 : mStreamUpdates[keptUpdateCount] = Move(mStreamUpdates[i]);
1176 0 : MOZ_ASSERT(!mStreamUpdates[i].mStream);
1177 : }
1178 0 : ++keptUpdateCount;
1179 : }
1180 0 : mStreamUpdates.TruncateLength(keptUpdateCount);
1181 :
1182 0 : mStreamUpdates.SetCapacity(mStreamUpdates.Length() + mStreams.Length() +
1183 0 : mSuspendedStreams.Length());
1184 0 : for (MediaStream* stream : AllStreams()) {
1185 0 : if (!stream->MainThreadNeedsUpdates()) {
1186 0 : continue;
1187 : }
1188 0 : StreamUpdate* update = mStreamUpdates.AppendElement();
1189 0 : update->mStream = stream;
1190 : // No blocking to worry about here, since we've passed
1191 : // UpdateCurrentTimeForStreams.
1192 0 : update->mNextMainThreadCurrentTime =
1193 0 : stream->GraphTimeToStreamTime(mProcessedTime);
1194 0 : update->mNextMainThreadFinished = stream->mNotifiedFinished;
1195 : }
1196 0 : if (!mPendingUpdateRunnables.IsEmpty()) {
1197 0 : mUpdateRunnables.AppendElements(Move(mPendingUpdateRunnables));
1198 : }
1199 : }
1200 :
1201 : // Don't send the message to the main thread if it's not going to have
1202 : // any work to do.
1203 0 : if (aFinalUpdate ||
1204 0 : !mUpdateRunnables.IsEmpty() ||
1205 0 : !mStreamUpdates.IsEmpty()) {
1206 0 : EnsureStableStateEventPosted();
1207 : }
1208 0 : }
1209 :
1210 : GraphTime
1211 0 : MediaStreamGraphImpl::RoundUpToNextAudioBlock(GraphTime aTime)
1212 : {
1213 0 : StreamTime ticks = aTime;
1214 0 : uint64_t block = ticks >> WEBAUDIO_BLOCK_SIZE_BITS;
1215 0 : uint64_t nextBlock = block + 1;
1216 0 : StreamTime nextTicks = nextBlock << WEBAUDIO_BLOCK_SIZE_BITS;
1217 0 : return nextTicks;
1218 : }
1219 :
1220 : void
1221 0 : MediaStreamGraphImpl::ProduceDataForStreamsBlockByBlock(uint32_t aStreamIndex,
1222 : TrackRate aSampleRate)
1223 : {
1224 0 : MOZ_ASSERT(aStreamIndex <= mFirstCycleBreaker,
1225 : "Cycle breaker is not AudioNodeStream?");
1226 0 : GraphTime t = mProcessedTime;
1227 0 : while (t < mStateComputedTime) {
1228 0 : GraphTime next = RoundUpToNextAudioBlock(t);
1229 0 : for (uint32_t i = mFirstCycleBreaker; i < mStreams.Length(); ++i) {
1230 0 : auto ns = static_cast<AudioNodeStream*>(mStreams[i]);
1231 0 : MOZ_ASSERT(ns->AsAudioNodeStream());
1232 0 : ns->ProduceOutputBeforeInput(t);
1233 : }
1234 0 : for (uint32_t i = aStreamIndex; i < mStreams.Length(); ++i) {
1235 0 : ProcessedMediaStream* ps = mStreams[i]->AsProcessedStream();
1236 0 : if (ps) {
1237 0 : ps->ProcessInput(t, next,
1238 0 : (next == mStateComputedTime) ? ProcessedMediaStream::ALLOW_FINISH : 0);
1239 : }
1240 : }
1241 0 : t = next;
1242 : }
1243 0 : NS_ASSERTION(t == mStateComputedTime,
1244 : "Something went wrong with rounding to block boundaries");
1245 0 : }
1246 :
1247 : bool
1248 0 : MediaStreamGraphImpl::AllFinishedStreamsNotified()
1249 : {
1250 0 : for (MediaStream* stream : AllStreams()) {
1251 0 : if (stream->mFinished && !stream->mNotifiedFinished) {
1252 0 : return false;
1253 : }
1254 : }
1255 0 : return true;
1256 : }
1257 :
1258 : void
1259 0 : MediaStreamGraphImpl::RunMessageAfterProcessing(UniquePtr<ControlMessage> aMessage)
1260 : {
1261 0 : MOZ_ASSERT(CurrentDriver()->OnThread());
1262 :
1263 0 : if (mFrontMessageQueue.IsEmpty()) {
1264 0 : mFrontMessageQueue.AppendElement();
1265 : }
1266 :
1267 : // Only one block is used for messages from the graph thread.
1268 0 : MOZ_ASSERT(mFrontMessageQueue.Length() == 1);
1269 0 : mFrontMessageQueue[0].mMessages.AppendElement(Move(aMessage));
1270 0 : }
1271 :
1272 : void
1273 0 : MediaStreamGraphImpl::RunMessagesInQueue()
1274 : {
1275 : // Calculate independent action times for each batch of messages (each
1276 : // batch corresponding to an event loop task). This isolates the performance
1277 : // of different scripts to some extent.
1278 0 : for (uint32_t i = 0; i < mFrontMessageQueue.Length(); ++i) {
1279 0 : nsTArray<UniquePtr<ControlMessage>>& messages = mFrontMessageQueue[i].mMessages;
1280 :
1281 0 : for (uint32_t j = 0; j < messages.Length(); ++j) {
1282 0 : messages[j]->Run();
1283 : }
1284 : }
1285 0 : mFrontMessageQueue.Clear();
1286 0 : }
1287 :
1288 : void
1289 0 : MediaStreamGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions)
1290 : {
1291 0 : MOZ_ASSERT(aEndBlockingDecisions >= mProcessedTime);
1292 : // The next state computed time can be the same as the previous: it
1293 : // means the driver would be have been blocking indefinitly, but the graph has
1294 : // been woken up right after having been to sleep.
1295 0 : MOZ_ASSERT(aEndBlockingDecisions >= mStateComputedTime);
1296 :
1297 0 : UpdateStreamOrder();
1298 :
1299 0 : bool ensureNextIteration = false;
1300 :
1301 : // Grab pending stream input and compute blocking time
1302 0 : for (MediaStream* stream : mStreams) {
1303 0 : if (SourceMediaStream* is = stream->AsSourceStream()) {
1304 0 : ExtractPendingInput(is, aEndBlockingDecisions, &ensureNextIteration);
1305 : }
1306 :
1307 0 : if (stream->mFinished) {
1308 : // The stream's not suspended, and since it's finished, underruns won't
1309 : // stop it playing out. So there's no blocking other than what we impose
1310 : // here.
1311 0 : GraphTime endTime = stream->GetStreamTracks().GetAllTracksEnd() +
1312 0 : stream->mTracksStartTime;
1313 0 : if (endTime <= mStateComputedTime) {
1314 0 : LOG(LogLevel::Verbose,
1315 : ("MediaStream %p is blocked due to being finished", stream));
1316 0 : stream->mStartBlocking = mStateComputedTime;
1317 : } else {
1318 0 : LOG(LogLevel::Verbose,
1319 : ("MediaStream %p is finished, but not blocked yet (end at %f, with "
1320 : "blocking at %f)",
1321 : stream,
1322 : MediaTimeToSeconds(stream->GetTracksEnd()),
1323 : MediaTimeToSeconds(endTime)));
1324 : // Data can't be added to a finished stream, so underruns are irrelevant.
1325 0 : stream->mStartBlocking = std::min(endTime, aEndBlockingDecisions);
1326 : }
1327 : } else {
1328 0 : stream->mStartBlocking = WillUnderrun(stream, aEndBlockingDecisions);
1329 : }
1330 : }
1331 :
1332 0 : for (MediaStream* stream : mSuspendedStreams) {
1333 0 : stream->mStartBlocking = mStateComputedTime;
1334 : }
1335 :
1336 : // The loop is woken up so soon that IterationEnd() barely advances and we
1337 : // end up having aEndBlockingDecision == mStateComputedTime.
1338 : // Since stream blocking is computed in the interval of
1339 : // [mStateComputedTime, aEndBlockingDecision), it won't be computed at all.
1340 : // We should ensure next iteration so that pending blocking changes will be
1341 : // computed in next loop.
1342 0 : if (ensureNextIteration ||
1343 0 : aEndBlockingDecisions == mStateComputedTime) {
1344 0 : EnsureNextIteration();
1345 : }
1346 0 : }
1347 :
1348 : void
1349 0 : MediaStreamGraphImpl::Process()
1350 : {
1351 : // Play stream contents.
1352 0 : bool allBlockedForever = true;
1353 : // True when we've done ProcessInput for all processed streams.
1354 0 : bool doneAllProducing = false;
1355 : // This is the number of frame that are written to the AudioStreams, for
1356 : // this cycle.
1357 0 : StreamTime ticksPlayed = 0;
1358 :
1359 0 : mMixer.StartMixing();
1360 :
1361 : // Figure out what each stream wants to do
1362 0 : for (uint32_t i = 0; i < mStreams.Length(); ++i) {
1363 0 : MediaStream* stream = mStreams[i];
1364 0 : if (!doneAllProducing) {
1365 0 : ProcessedMediaStream* ps = stream->AsProcessedStream();
1366 0 : if (ps) {
1367 0 : AudioNodeStream* n = stream->AsAudioNodeStream();
1368 0 : if (n) {
1369 : #ifdef DEBUG
1370 : // Verify that the sampling rate for all of the following streams is the same
1371 0 : for (uint32_t j = i + 1; j < mStreams.Length(); ++j) {
1372 0 : AudioNodeStream* nextStream = mStreams[j]->AsAudioNodeStream();
1373 0 : if (nextStream) {
1374 0 : MOZ_ASSERT(n->SampleRate() == nextStream->SampleRate(),
1375 : "All AudioNodeStreams in the graph must have the same sampling rate");
1376 : }
1377 : }
1378 : #endif
1379 : // Since an AudioNodeStream is present, go ahead and
1380 : // produce audio block by block for all the rest of the streams.
1381 0 : ProduceDataForStreamsBlockByBlock(i, n->SampleRate());
1382 0 : doneAllProducing = true;
1383 : } else {
1384 0 : ps->ProcessInput(mProcessedTime, mStateComputedTime,
1385 0 : ProcessedMediaStream::ALLOW_FINISH);
1386 0 : NS_ASSERTION(stream->mTracks.GetEnd() >=
1387 : GraphTimeToStreamTimeWithBlocking(stream, mStateComputedTime),
1388 : "Stream did not produce enough data");
1389 : }
1390 : }
1391 : }
1392 0 : NotifyHasCurrentData(stream);
1393 : // Only playback audio and video in real-time mode
1394 0 : if (mRealtime) {
1395 0 : CreateOrDestroyAudioStreams(stream);
1396 0 : if (CurrentDriver()->AsAudioCallbackDriver()) {
1397 0 : StreamTime ticksPlayedForThisStream = PlayAudio(stream);
1398 0 : if (!ticksPlayed) {
1399 0 : ticksPlayed = ticksPlayedForThisStream;
1400 : } else {
1401 0 : MOZ_ASSERT(!ticksPlayedForThisStream || ticksPlayedForThisStream == ticksPlayed,
1402 : "Each stream should have the same number of frame.");
1403 : }
1404 : }
1405 : }
1406 0 : if (stream->mStartBlocking > mProcessedTime) {
1407 0 : allBlockedForever = false;
1408 : }
1409 : }
1410 :
1411 0 : if (CurrentDriver()->AsAudioCallbackDriver() && ticksPlayed) {
1412 0 : mMixer.FinishMixing();
1413 : }
1414 :
1415 0 : if (!allBlockedForever) {
1416 0 : EnsureNextIteration();
1417 : }
1418 0 : }
1419 :
1420 : bool
1421 0 : MediaStreamGraphImpl::UpdateMainThreadState()
1422 : {
1423 0 : MonitorAutoLock lock(mMonitor);
1424 0 : bool finalUpdate = mForceShutDown ||
1425 0 : (mProcessedTime >= mEndTime && AllFinishedStreamsNotified()) ||
1426 0 : (IsEmpty() && mBackMessageQueue.IsEmpty());
1427 0 : PrepareUpdatesToMainThreadState(finalUpdate);
1428 0 : if (finalUpdate) {
1429 : // Enter shutdown mode. The stable-state handler will detect this
1430 : // and complete shutdown. Destroy any streams immediately.
1431 0 : LOG(LogLevel::Debug,
1432 : ("MediaStreamGraph %p waiting for main thread cleanup", this));
1433 : // We'll shut down this graph object if it does not get restarted.
1434 0 : mLifecycleState = LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP;
1435 : // No need to Destroy streams here. The main-thread owner of each
1436 : // stream is responsible for calling Destroy on them.
1437 0 : return false;
1438 : }
1439 :
1440 0 : CurrentDriver()->WaitForNextIteration();
1441 :
1442 0 : SwapMessageQueues();
1443 0 : return true;
1444 : }
1445 :
1446 : bool
1447 0 : MediaStreamGraphImpl::OneIteration(GraphTime aStateEnd)
1448 : {
1449 0 : WebCore::DenormalDisabler disabler;
1450 :
1451 : // Process graph message from the main thread for this iteration.
1452 0 : RunMessagesInQueue();
1453 :
1454 0 : GraphTime stateEnd = std::min(aStateEnd, mEndTime);
1455 0 : UpdateGraph(stateEnd);
1456 :
1457 0 : mStateComputedTime = stateEnd;
1458 :
1459 0 : Process();
1460 :
1461 0 : GraphTime oldProcessedTime = mProcessedTime;
1462 0 : mProcessedTime = stateEnd;
1463 :
1464 0 : UpdateCurrentTimeForStreams(oldProcessedTime);
1465 :
1466 0 : ProcessChunkMetadata(oldProcessedTime);
1467 :
1468 : // Process graph messages queued from RunMessageAfterProcessing() on this
1469 : // thread during the iteration.
1470 0 : RunMessagesInQueue();
1471 :
1472 0 : return UpdateMainThreadState();
1473 : }
1474 :
1475 : void
1476 0 : MediaStreamGraphImpl::ApplyStreamUpdate(StreamUpdate* aUpdate)
1477 : {
1478 0 : mMonitor.AssertCurrentThreadOwns();
1479 :
1480 0 : MediaStream* stream = aUpdate->mStream;
1481 0 : if (!stream)
1482 0 : return;
1483 0 : stream->mMainThreadCurrentTime = aUpdate->mNextMainThreadCurrentTime;
1484 0 : stream->mMainThreadFinished = aUpdate->mNextMainThreadFinished;
1485 :
1486 0 : if (stream->ShouldNotifyStreamFinished()) {
1487 0 : stream->NotifyMainThreadListeners();
1488 : }
1489 : }
1490 :
1491 : void
1492 0 : MediaStreamGraphImpl::ForceShutDown(ShutdownTicket* aShutdownTicket)
1493 : {
1494 0 : NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread");
1495 0 : LOG(LogLevel::Debug, ("MediaStreamGraph %p ForceShutdown", this));
1496 :
1497 0 : MonitorAutoLock lock(mMonitor);
1498 0 : if (aShutdownTicket) {
1499 0 : MOZ_ASSERT(!mForceShutdownTicket);
1500 : // Avoid waiting forever for a graph to shut down
1501 : // synchronously. Reports are that some 3rd-party audio drivers
1502 : // occasionally hang in shutdown (both for us and Chrome).
1503 0 : mShutdownTimer = do_CreateInstance(NS_TIMER_CONTRACTID);
1504 0 : if (!mShutdownTimer) {
1505 0 : return;
1506 : }
1507 0 : mShutdownTimer->InitWithCallback(this,
1508 : MediaStreamGraph::AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT,
1509 0 : nsITimer::TYPE_ONE_SHOT);
1510 : }
1511 0 : mForceShutDown = true;
1512 0 : mForceShutdownTicket = aShutdownTicket;
1513 0 : if (mLifecycleState == LIFECYCLE_THREAD_NOT_STARTED) {
1514 : // We *could* have just sent this a message to start up, so don't
1515 : // yank the rug out from under it. Tell it to startup and let it
1516 : // shut down.
1517 0 : RefPtr<GraphDriver> driver = CurrentDriver();
1518 0 : MonitorAutoUnlock unlock(mMonitor);
1519 0 : driver->Start();
1520 : }
1521 0 : EnsureNextIterationLocked();
1522 : }
1523 :
1524 : NS_IMETHODIMP
1525 0 : MediaStreamGraphImpl::Notify(nsITimer* aTimer)
1526 : {
1527 0 : MonitorAutoLock lock(mMonitor);
1528 0 : NS_ASSERTION(!mForceShutdownTicket, "MediaStreamGraph took too long to shut down!");
1529 : // Sigh, graph took too long to shut down. Stop blocking system
1530 : // shutdown and hope all is well.
1531 0 : mForceShutdownTicket = nullptr;
1532 0 : return NS_OK;
1533 : }
1534 :
1535 :
1536 3 : /* static */ StaticRefPtr<nsIAsyncShutdownBlocker> gMediaStreamGraphShutdownBlocker;
1537 :
1538 : namespace {
1539 :
1540 0 : class MediaStreamGraphShutDownRunnable : public Runnable {
1541 : public:
1542 0 : explicit MediaStreamGraphShutDownRunnable(MediaStreamGraphImpl* aGraph)
1543 0 : : Runnable("MediaStreamGraphShutDownRunnable")
1544 0 : , mGraph(aGraph)
1545 0 : {}
1546 0 : NS_IMETHOD Run()
1547 : {
1548 0 : NS_ASSERTION(mGraph->mDetectedNotRunning,
1549 : "We should know the graph thread control loop isn't running!");
1550 :
1551 0 : LOG(LogLevel::Debug, ("Shutting down graph %p", mGraph.get()));
1552 :
1553 : // We've asserted the graph isn't running. Use mDriver instead of CurrentDriver
1554 : // to avoid thread-safety checks
1555 : #if 0 // AudioCallbackDrivers are released asynchronously anyways
1556 : // XXX a better test would be have setting mDetectedNotRunning make sure
1557 : // any current callback has finished and block future ones -- or just
1558 : // handle it all in Shutdown()!
1559 : if (mGraph->mDriver->AsAudioCallbackDriver()) {
1560 : MOZ_ASSERT(!mGraph->mDriver->AsAudioCallbackDriver()->InCallback());
1561 : }
1562 : #endif
1563 :
1564 0 : mGraph->mDriver->Shutdown(); // This will wait until it's shutdown since
1565 : // we'll start tearing down the graph after this
1566 :
1567 : // Safe to access these without the monitor since the graph isn't running.
1568 : // We may be one of several graphs. Drop ticket to eventually unblock shutdown.
1569 0 : if (mGraph->mShutdownTimer && !mGraph->mForceShutdownTicket) {
1570 0 : MOZ_ASSERT(false,
1571 : "AudioCallbackDriver took too long to shut down and we let shutdown"
1572 : " continue - freezing and leaking");
1573 :
1574 : // The timer fired, so we may be deeper in shutdown now. Block any further
1575 : // teardown and just leak, for safety.
1576 : return NS_OK;
1577 : }
1578 0 : mGraph->mForceShutdownTicket = nullptr;
1579 :
1580 : // We can't block past the final LIFECYCLE_WAITING_FOR_STREAM_DESTRUCTION
1581 : // stage, since completion of that stage requires all streams to be freed,
1582 : // which requires shutdown to proceed.
1583 :
1584 : // mGraph's thread is not running so it's OK to do whatever here
1585 0 : if (mGraph->IsEmpty()) {
1586 : // mGraph is no longer needed, so delete it.
1587 0 : mGraph->Destroy();
1588 : } else {
1589 : // The graph is not empty. We must be in a forced shutdown, or a
1590 : // non-realtime graph that has finished processing. Some later
1591 : // AppendMessage will detect that the manager has been emptied, and
1592 : // delete it.
1593 0 : NS_ASSERTION(mGraph->mForceShutDown || !mGraph->mRealtime,
1594 : "Not in forced shutdown?");
1595 0 : for (MediaStream* stream : mGraph->AllStreams()) {
1596 : // Clean up all MediaSegments since we cannot release Images too
1597 : // late during shutdown.
1598 0 : if (SourceMediaStream* source = stream->AsSourceStream()) {
1599 : // Finishing a SourceStream prevents new data from being appended.
1600 0 : source->Finish();
1601 : }
1602 0 : stream->GetStreamTracks().Clear();
1603 : }
1604 :
1605 0 : mGraph->mLifecycleState =
1606 : MediaStreamGraphImpl::LIFECYCLE_WAITING_FOR_STREAM_DESTRUCTION;
1607 : }
1608 0 : return NS_OK;
1609 : }
1610 : private:
1611 : RefPtr<MediaStreamGraphImpl> mGraph;
1612 : };
1613 :
1614 0 : class MediaStreamGraphStableStateRunnable : public Runnable {
1615 : public:
1616 0 : explicit MediaStreamGraphStableStateRunnable(MediaStreamGraphImpl* aGraph,
1617 : bool aSourceIsMSG)
1618 0 : : Runnable("MediaStreamGraphStableStateRunnable")
1619 : , mGraph(aGraph)
1620 0 : , mSourceIsMSG(aSourceIsMSG)
1621 : {
1622 0 : }
1623 0 : NS_IMETHOD Run() override
1624 : {
1625 0 : if (mGraph) {
1626 0 : mGraph->RunInStableState(mSourceIsMSG);
1627 : }
1628 0 : return NS_OK;
1629 : }
1630 : private:
1631 : RefPtr<MediaStreamGraphImpl> mGraph;
1632 : bool mSourceIsMSG;
1633 : };
1634 :
1635 : /*
1636 : * Control messages forwarded from main thread to graph manager thread
1637 : */
1638 0 : class CreateMessage : public ControlMessage {
1639 : public:
1640 0 : explicit CreateMessage(MediaStream* aStream) : ControlMessage(aStream) {}
1641 0 : void Run() override
1642 : {
1643 0 : mStream->GraphImpl()->AddStreamGraphThread(mStream);
1644 0 : }
1645 0 : void RunDuringShutdown() override
1646 : {
1647 : // Make sure to run this message during shutdown too, to make sure
1648 : // that we balance the number of streams registered with the graph
1649 : // as they're destroyed during shutdown.
1650 0 : Run();
1651 0 : }
1652 : };
1653 :
1654 : } // namespace
1655 :
1656 : void
1657 0 : MediaStreamGraphImpl::RunInStableState(bool aSourceIsMSG)
1658 : {
1659 0 : NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread");
1660 :
1661 0 : nsTArray<nsCOMPtr<nsIRunnable> > runnables;
1662 : // When we're doing a forced shutdown, pending control messages may be
1663 : // run on the main thread via RunDuringShutdown. Those messages must
1664 : // run without the graph monitor being held. So, we collect them here.
1665 0 : nsTArray<UniquePtr<ControlMessage>> controlMessagesToRunDuringShutdown;
1666 :
1667 : {
1668 0 : MonitorAutoLock lock(mMonitor);
1669 0 : if (aSourceIsMSG) {
1670 0 : MOZ_ASSERT(mPostedRunInStableStateEvent);
1671 0 : mPostedRunInStableStateEvent = false;
1672 : }
1673 :
1674 : // This should be kept in sync with the LifecycleState enum in
1675 : // MediaStreamGraphImpl.h
1676 : const char* LifecycleState_str[] = {
1677 : "LIFECYCLE_THREAD_NOT_STARTED",
1678 : "LIFECYCLE_RUNNING",
1679 : "LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP",
1680 : "LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN",
1681 : "LIFECYCLE_WAITING_FOR_STREAM_DESTRUCTION"
1682 0 : };
1683 :
1684 0 : if (mLifecycleState != LIFECYCLE_RUNNING) {
1685 0 : LOG(LogLevel::Debug,
1686 : ("Running %p in stable state. Current state: %s",
1687 : this,
1688 : LifecycleState_str[mLifecycleState]));
1689 : }
1690 :
1691 0 : runnables.SwapElements(mUpdateRunnables);
1692 0 : for (uint32_t i = 0; i < mStreamUpdates.Length(); ++i) {
1693 0 : StreamUpdate* update = &mStreamUpdates[i];
1694 0 : if (update->mStream) {
1695 0 : ApplyStreamUpdate(update);
1696 : }
1697 : }
1698 0 : mStreamUpdates.Clear();
1699 :
1700 0 : if (mCurrentTaskMessageQueue.IsEmpty()) {
1701 0 : if (mLifecycleState == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP && IsEmpty()) {
1702 : // Complete shutdown. First, ensure that this graph is no longer used.
1703 : // A new graph graph will be created if one is needed.
1704 : // Asynchronously clean up old graph. We don't want to do this
1705 : // synchronously because it spins the event loop waiting for threads
1706 : // to shut down, and we don't want to do that in a stable state handler.
1707 0 : mLifecycleState = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
1708 0 : LOG(LogLevel::Debug,
1709 : ("Sending MediaStreamGraphShutDownRunnable %p", this));
1710 0 : nsCOMPtr<nsIRunnable> event = new MediaStreamGraphShutDownRunnable(this );
1711 0 : mAbstractMainThread->Dispatch(event.forget());
1712 :
1713 0 : LOG(LogLevel::Debug, ("Disconnecting MediaStreamGraph %p", this));
1714 :
1715 : // Find the graph in the hash table and remove it.
1716 0 : for (auto iter = gGraphs.Iter(); !iter.Done(); iter.Next()) {
1717 0 : if (iter.UserData() == this) {
1718 0 : iter.Remove();
1719 0 : break;
1720 : }
1721 : }
1722 : }
1723 : } else {
1724 0 : if (mLifecycleState <= LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
1725 0 : MessageBlock* block = mBackMessageQueue.AppendElement();
1726 0 : block->mMessages.SwapElements(mCurrentTaskMessageQueue);
1727 0 : EnsureNextIterationLocked();
1728 : }
1729 :
1730 : // If the MediaStreamGraph has more messages going to it, try to revive
1731 : // it to process those messages. Don't do this if we're in a forced
1732 : // shutdown or it's a non-realtime graph that has already terminated
1733 : // processing.
1734 0 : if (mLifecycleState == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP &&
1735 0 : mRealtime && !mForceShutDown) {
1736 0 : mLifecycleState = LIFECYCLE_RUNNING;
1737 : // Revive the MediaStreamGraph since we have more messages going to it.
1738 : // Note that we need to put messages into its queue before reviving it,
1739 : // or it might exit immediately.
1740 : {
1741 0 : LOG(LogLevel::Debug,
1742 : ("Reviving a graph (%p) ! %s",
1743 : this,
1744 : CurrentDriver()->AsAudioCallbackDriver() ? "AudioDriver"
1745 : : "SystemDriver"));
1746 0 : RefPtr<GraphDriver> driver = CurrentDriver();
1747 0 : MonitorAutoUnlock unlock(mMonitor);
1748 0 : driver->Revive();
1749 : }
1750 : }
1751 : }
1752 :
1753 : // Don't start the thread for a non-realtime graph until it has been
1754 : // explicitly started by StartNonRealtimeProcessing.
1755 0 : if (mLifecycleState == LIFECYCLE_THREAD_NOT_STARTED &&
1756 0 : (mRealtime || mNonRealtimeProcessing)) {
1757 0 : mLifecycleState = LIFECYCLE_RUNNING;
1758 : // Start the thread now. We couldn't start it earlier because
1759 : // the graph might exit immediately on finding it has no streams. The
1760 : // first message for a new graph must create a stream.
1761 : {
1762 : // We should exit the monitor for now, because starting a stream might
1763 : // take locks, and we don't want to deadlock.
1764 0 : LOG(LogLevel::Debug,
1765 : ("Starting a graph (%p) ! %s",
1766 : this,
1767 : CurrentDriver()->AsAudioCallbackDriver() ? "AudioDriver"
1768 : : "SystemDriver"));
1769 0 : RefPtr<GraphDriver> driver = CurrentDriver();
1770 0 : MonitorAutoUnlock unlock(mMonitor);
1771 0 : driver->Start();
1772 : // It's not safe to Shutdown() a thread from StableState, and
1773 : // releasing this may shutdown a SystemClockDriver thread.
1774 : // Proxy the release to outside of StableState.
1775 : NS_ReleaseOnMainThread(
1776 0 : "MediaStreamGraphImpl::CurrentDriver", driver.forget(),
1777 0 : true); // always proxy
1778 : }
1779 : }
1780 :
1781 0 : if ((mForceShutDown || !mRealtime) &&
1782 0 : mLifecycleState == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
1783 : // Defer calls to RunDuringShutdown() to happen while mMonitor is not held.
1784 0 : for (uint32_t i = 0; i < mBackMessageQueue.Length(); ++i) {
1785 0 : MessageBlock& mb = mBackMessageQueue[i];
1786 0 : controlMessagesToRunDuringShutdown.AppendElements(Move(mb.mMessages));
1787 : }
1788 0 : mBackMessageQueue.Clear();
1789 0 : MOZ_ASSERT(mCurrentTaskMessageQueue.IsEmpty());
1790 : // Stop MediaStreamGraph threads. Do not clear gGraph since
1791 : // we have outstanding DOM objects that may need it.
1792 0 : mLifecycleState = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
1793 0 : nsCOMPtr<nsIRunnable> event = new MediaStreamGraphShutDownRunnable(this);
1794 0 : mAbstractMainThread->Dispatch(event.forget());
1795 : }
1796 :
1797 0 : mDetectedNotRunning = mLifecycleState > LIFECYCLE_RUNNING;
1798 : }
1799 :
1800 : // Make sure we get a new current time in the next event loop task
1801 0 : if (!aSourceIsMSG) {
1802 0 : MOZ_ASSERT(mPostedRunInStableState);
1803 0 : mPostedRunInStableState = false;
1804 : }
1805 :
1806 0 : for (uint32_t i = 0; i < controlMessagesToRunDuringShutdown.Length(); ++i) {
1807 0 : controlMessagesToRunDuringShutdown[i]->RunDuringShutdown();
1808 : }
1809 :
1810 : #ifdef DEBUG
1811 0 : mCanRunMessagesSynchronously = mDetectedNotRunning &&
1812 0 : mLifecycleState >= LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
1813 : #endif
1814 :
1815 0 : for (uint32_t i = 0; i < runnables.Length(); ++i) {
1816 0 : runnables[i]->Run();
1817 : }
1818 0 : }
1819 :
1820 :
1821 : void
1822 0 : MediaStreamGraphImpl::EnsureRunInStableState()
1823 : {
1824 0 : NS_ASSERTION(NS_IsMainThread(), "main thread only");
1825 :
1826 0 : if (mPostedRunInStableState)
1827 0 : return;
1828 0 : mPostedRunInStableState = true;
1829 0 : nsCOMPtr<nsIRunnable> event = new MediaStreamGraphStableStateRunnable(this, false);
1830 0 : nsContentUtils::RunInStableState(event.forget());
1831 : }
1832 :
1833 : void
1834 0 : MediaStreamGraphImpl::EnsureStableStateEventPosted()
1835 : {
1836 0 : mMonitor.AssertCurrentThreadOwns();
1837 :
1838 0 : if (mPostedRunInStableStateEvent)
1839 0 : return;
1840 0 : mPostedRunInStableStateEvent = true;
1841 0 : nsCOMPtr<nsIRunnable> event = new MediaStreamGraphStableStateRunnable(this, true);
1842 0 : mAbstractMainThread->Dispatch(event.forget());
1843 : }
1844 :
1845 : void
1846 0 : MediaStreamGraphImpl::AppendMessage(UniquePtr<ControlMessage> aMessage)
1847 : {
1848 0 : MOZ_ASSERT(NS_IsMainThread(), "main thread only");
1849 0 : MOZ_ASSERT(!aMessage->GetStream() ||
1850 : !aMessage->GetStream()->IsDestroyed(),
1851 : "Stream already destroyed");
1852 :
1853 0 : if (mDetectedNotRunning &&
1854 0 : mLifecycleState > LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
1855 : // The graph control loop is not running and main thread cleanup has
1856 : // happened. From now on we can't append messages to mCurrentTaskMessageQueue,
1857 : // because that will never be processed again, so just RunDuringShutdown
1858 : // this message.
1859 : // This should only happen during forced shutdown, or after a non-realtime
1860 : // graph has finished processing.
1861 : #ifdef DEBUG
1862 0 : MOZ_ASSERT(mCanRunMessagesSynchronously);
1863 0 : mCanRunMessagesSynchronously = false;
1864 : #endif
1865 0 : aMessage->RunDuringShutdown();
1866 : #ifdef DEBUG
1867 0 : mCanRunMessagesSynchronously = true;
1868 : #endif
1869 0 : if (IsEmpty() &&
1870 0 : mLifecycleState >= LIFECYCLE_WAITING_FOR_STREAM_DESTRUCTION) {
1871 :
1872 : // Find the graph in the hash table and remove it.
1873 0 : for (auto iter = gGraphs.Iter(); !iter.Done(); iter.Next()) {
1874 0 : if (iter.UserData() == this) {
1875 0 : iter.Remove();
1876 0 : break;
1877 : }
1878 : }
1879 :
1880 0 : Destroy();
1881 : }
1882 0 : return;
1883 : }
1884 :
1885 0 : mCurrentTaskMessageQueue.AppendElement(Move(aMessage));
1886 0 : EnsureRunInStableState();
1887 : }
1888 :
1889 : void
1890 0 : MediaStreamGraphImpl::Dispatch(already_AddRefed<nsIRunnable>&& aRunnable)
1891 : {
1892 0 : mAbstractMainThread->Dispatch(Move(aRunnable));
1893 0 : }
1894 :
1895 0 : MediaStream::MediaStream()
1896 : : mTracksStartTime(0)
1897 : , mStartBlocking(GRAPH_TIME_MAX)
1898 : , mSuspendedCount(0)
1899 : , mFinished(false)
1900 : , mNotifiedFinished(false)
1901 : , mNotifiedBlocked(false)
1902 : , mHasCurrentData(false)
1903 : , mNotifiedHasCurrentData(false)
1904 : , mMainThreadCurrentTime(0)
1905 : , mMainThreadFinished(false)
1906 : , mFinishedNotificationSent(false)
1907 : , mMainThreadDestroyed(false)
1908 : , mNrOfMainThreadUsers(0)
1909 : , mGraph(nullptr)
1910 0 : , mAudioChannelType(dom::AudioChannel::Normal)
1911 : {
1912 0 : MOZ_COUNT_CTOR(MediaStream);
1913 0 : }
1914 :
1915 0 : MediaStream::~MediaStream()
1916 : {
1917 0 : MOZ_COUNT_DTOR(MediaStream);
1918 0 : NS_ASSERTION(mMainThreadDestroyed, "Should have been destroyed already");
1919 0 : NS_ASSERTION(mMainThreadListeners.IsEmpty(),
1920 : "All main thread listeners should have been removed");
1921 0 : }
1922 :
1923 : size_t
1924 0 : MediaStream::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
1925 : {
1926 0 : size_t amount = 0;
1927 :
1928 : // Not owned:
1929 : // - mGraph - Not reported here
1930 : // - mConsumers - elements
1931 : // Future:
1932 : // - mVideoOutputs - elements
1933 : // - mLastPlayedVideoFrame
1934 : // - mListeners - elements
1935 : // - mAudioOutputStream - elements
1936 :
1937 0 : amount += mTracks.SizeOfExcludingThis(aMallocSizeOf);
1938 0 : amount += mAudioOutputs.ShallowSizeOfExcludingThis(aMallocSizeOf);
1939 0 : amount += mVideoOutputs.ShallowSizeOfExcludingThis(aMallocSizeOf);
1940 0 : amount += mListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
1941 0 : amount += mMainThreadListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
1942 0 : amount += mDisabledTracks.ShallowSizeOfExcludingThis(aMallocSizeOf);
1943 0 : amount += mConsumers.ShallowSizeOfExcludingThis(aMallocSizeOf);
1944 :
1945 0 : return amount;
1946 : }
1947 :
1948 : size_t
1949 0 : MediaStream::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const
1950 : {
1951 0 : return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
1952 : }
1953 :
1954 : void
1955 0 : MediaStream::IncrementSuspendCount()
1956 : {
1957 0 : ++mSuspendedCount;
1958 0 : if (mSuspendedCount == 1) {
1959 0 : for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
1960 0 : mConsumers[i]->Suspended();
1961 : }
1962 : }
1963 0 : }
1964 :
1965 : void
1966 0 : MediaStream::DecrementSuspendCount()
1967 : {
1968 0 : NS_ASSERTION(mSuspendedCount > 0, "Suspend count underrun");
1969 0 : --mSuspendedCount;
1970 :
1971 0 : if (mSuspendedCount == 0) {
1972 0 : for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
1973 0 : mConsumers[i]->Resumed();
1974 : }
1975 : }
1976 0 : }
1977 :
1978 : MediaStreamGraphImpl*
1979 0 : MediaStream::GraphImpl()
1980 : {
1981 0 : return mGraph;
1982 : }
1983 :
1984 : MediaStreamGraph*
1985 0 : MediaStream::Graph()
1986 : {
1987 0 : return mGraph;
1988 : }
1989 :
1990 : void
1991 0 : MediaStream::SetGraphImpl(MediaStreamGraphImpl* aGraph)
1992 : {
1993 0 : MOZ_ASSERT(!mGraph, "Should only be called once");
1994 0 : mGraph = aGraph;
1995 0 : mAudioChannelType = aGraph->AudioChannel();
1996 0 : mTracks.InitGraphRate(aGraph->GraphRate());
1997 0 : }
1998 :
1999 : void
2000 0 : MediaStream::SetGraphImpl(MediaStreamGraph* aGraph)
2001 : {
2002 0 : MediaStreamGraphImpl* graph = static_cast<MediaStreamGraphImpl*>(aGraph);
2003 0 : SetGraphImpl(graph);
2004 0 : }
2005 :
2006 : StreamTime
2007 0 : MediaStream::GraphTimeToStreamTime(GraphTime aTime)
2008 : {
2009 0 : NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
2010 : aTime <= mStartBlocking,
2011 : "Incorrectly ignoring blocking!");
2012 0 : return aTime - mTracksStartTime;
2013 : }
2014 :
2015 : GraphTime
2016 0 : MediaStream::StreamTimeToGraphTime(StreamTime aTime)
2017 : {
2018 0 : NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
2019 : aTime + mTracksStartTime <= mStartBlocking,
2020 : "Incorrectly ignoring blocking!");
2021 0 : return aTime + mTracksStartTime;
2022 : }
2023 :
2024 : StreamTime
2025 0 : MediaStream::GraphTimeToStreamTimeWithBlocking(GraphTime aTime)
2026 : {
2027 0 : return GraphImpl()->GraphTimeToStreamTimeWithBlocking(this, aTime);
2028 : }
2029 :
2030 : void
2031 0 : MediaStream::FinishOnGraphThread()
2032 : {
2033 0 : GraphImpl()->FinishStream(this);
2034 0 : }
2035 :
2036 : StreamTracks::Track*
2037 0 : MediaStream::FindTrack(TrackID aID)
2038 : {
2039 0 : return mTracks.FindTrack(aID);
2040 : }
2041 :
2042 : StreamTracks::Track*
2043 0 : MediaStream::EnsureTrack(TrackID aTrackId)
2044 : {
2045 0 : StreamTracks::Track* track = mTracks.FindTrack(aTrackId);
2046 0 : if (!track) {
2047 0 : nsAutoPtr<MediaSegment> segment(new AudioSegment());
2048 0 : for (uint32_t j = 0; j < mListeners.Length(); ++j) {
2049 0 : MediaStreamListener* l = mListeners[j];
2050 0 : l->NotifyQueuedTrackChanges(Graph(), aTrackId, 0,
2051 : TrackEventCommand::TRACK_EVENT_CREATED,
2052 0 : *segment);
2053 : // TODO If we ever need to ensure several tracks at once, we will have to
2054 : // change this.
2055 0 : l->NotifyFinishedTrackCreation(Graph());
2056 : }
2057 0 : track = &mTracks.AddTrack(aTrackId, 0, segment.forget());
2058 : }
2059 0 : return track;
2060 : }
2061 :
2062 : void
2063 0 : MediaStream::RemoveAllListenersImpl()
2064 : {
2065 0 : for (int32_t i = mListeners.Length() - 1; i >= 0; --i) {
2066 0 : RefPtr<MediaStreamListener> listener = mListeners[i].forget();
2067 0 : listener->NotifyEvent(GraphImpl(), MediaStreamGraphEvent::EVENT_REMOVED);
2068 : }
2069 0 : mListeners.Clear();
2070 0 : }
2071 :
2072 : void
2073 0 : MediaStream::DestroyImpl()
2074 : {
2075 0 : for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
2076 0 : mConsumers[i]->Disconnect();
2077 : }
2078 0 : mGraph = nullptr;
2079 0 : }
2080 :
2081 : void
2082 0 : MediaStream::Destroy()
2083 : {
2084 0 : NS_ASSERTION(mNrOfMainThreadUsers == 0,
2085 : "Do not mix Destroy() and RegisterUser()/UnregisterUser()");
2086 : // Keep this stream alive until we leave this method
2087 0 : RefPtr<MediaStream> kungFuDeathGrip = this;
2088 :
2089 0 : class Message : public ControlMessage {
2090 : public:
2091 0 : explicit Message(MediaStream* aStream) : ControlMessage(aStream) {}
2092 0 : void Run() override
2093 : {
2094 0 : mStream->RemoveAllListenersImpl();
2095 0 : auto graph = mStream->GraphImpl();
2096 0 : mStream->DestroyImpl();
2097 0 : graph->RemoveStreamGraphThread(mStream);
2098 0 : }
2099 0 : void RunDuringShutdown() override
2100 0 : { Run(); }
2101 : };
2102 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this));
2103 : // Message::RunDuringShutdown may have removed this stream from the graph,
2104 : // but our kungFuDeathGrip above will have kept this stream alive if
2105 : // necessary.
2106 0 : mMainThreadDestroyed = true;
2107 0 : }
2108 :
2109 : void
2110 0 : MediaStream::RegisterUser()
2111 : {
2112 0 : MOZ_ASSERT(NS_IsMainThread());
2113 0 : ++mNrOfMainThreadUsers;
2114 0 : }
2115 :
2116 : void
2117 0 : MediaStream::UnregisterUser()
2118 : {
2119 0 : MOZ_ASSERT(NS_IsMainThread());
2120 :
2121 0 : --mNrOfMainThreadUsers;
2122 0 : NS_ASSERTION(mNrOfMainThreadUsers >= 0, "Double-removal of main thread user");
2123 0 : NS_ASSERTION(!IsDestroyed(), "Do not mix Destroy() and RegisterUser()/UnregisterUser()");
2124 0 : if (mNrOfMainThreadUsers == 0) {
2125 0 : Destroy();
2126 : }
2127 0 : }
2128 :
2129 : void
2130 0 : MediaStream::AddAudioOutput(void* aKey)
2131 : {
2132 0 : class Message : public ControlMessage {
2133 : public:
2134 0 : Message(MediaStream* aStream, void* aKey) : ControlMessage(aStream), mKey(aKey) {}
2135 0 : void Run() override
2136 : {
2137 0 : mStream->AddAudioOutputImpl(mKey);
2138 0 : }
2139 : void* mKey;
2140 : };
2141 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aKey));
2142 0 : }
2143 :
2144 : void
2145 0 : MediaStream::SetAudioOutputVolumeImpl(void* aKey, float aVolume)
2146 : {
2147 0 : for (uint32_t i = 0; i < mAudioOutputs.Length(); ++i) {
2148 0 : if (mAudioOutputs[i].mKey == aKey) {
2149 0 : mAudioOutputs[i].mVolume = aVolume;
2150 0 : return;
2151 : }
2152 : }
2153 0 : NS_ERROR("Audio output key not found");
2154 : }
2155 :
2156 : void
2157 0 : MediaStream::SetAudioOutputVolume(void* aKey, float aVolume)
2158 : {
2159 0 : class Message : public ControlMessage {
2160 : public:
2161 0 : Message(MediaStream* aStream, void* aKey, float aVolume) :
2162 0 : ControlMessage(aStream), mKey(aKey), mVolume(aVolume) {}
2163 0 : void Run() override
2164 : {
2165 0 : mStream->SetAudioOutputVolumeImpl(mKey, mVolume);
2166 0 : }
2167 : void* mKey;
2168 : float mVolume;
2169 : };
2170 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aKey, aVolume));
2171 0 : }
2172 :
2173 : void
2174 0 : MediaStream::AddAudioOutputImpl(void* aKey)
2175 : {
2176 0 : LOG(LogLevel::Info,
2177 : ("MediaStream %p Adding AudioOutput for key %p", this, aKey));
2178 0 : mAudioOutputs.AppendElement(AudioOutput(aKey));
2179 0 : }
2180 :
2181 : void
2182 0 : MediaStream::RemoveAudioOutputImpl(void* aKey)
2183 : {
2184 0 : LOG(LogLevel::Info,
2185 : ("MediaStream %p Removing AudioOutput for key %p", this, aKey));
2186 0 : for (uint32_t i = 0; i < mAudioOutputs.Length(); ++i) {
2187 0 : if (mAudioOutputs[i].mKey == aKey) {
2188 0 : mAudioOutputs.RemoveElementAt(i);
2189 0 : return;
2190 : }
2191 : }
2192 0 : NS_ERROR("Audio output key not found");
2193 : }
2194 :
2195 : void
2196 0 : MediaStream::RemoveAudioOutput(void* aKey)
2197 : {
2198 0 : class Message : public ControlMessage {
2199 : public:
2200 0 : Message(MediaStream* aStream, void* aKey) :
2201 0 : ControlMessage(aStream), mKey(aKey) {}
2202 0 : void Run() override
2203 : {
2204 0 : mStream->RemoveAudioOutputImpl(mKey);
2205 0 : }
2206 : void* mKey;
2207 : };
2208 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aKey));
2209 0 : }
2210 :
2211 : void
2212 0 : MediaStream::AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink,
2213 : TrackID aID)
2214 : {
2215 0 : RefPtr<MediaStreamVideoSink> sink = aSink;
2216 0 : LOG(LogLevel::Info,
2217 : ("MediaStream %p Adding MediaStreamVideoSink %p as output",
2218 : this,
2219 : sink.get()));
2220 0 : MOZ_ASSERT(aID != TRACK_NONE);
2221 0 : for (auto entry : mVideoOutputs) {
2222 0 : if (entry.mListener == sink &&
2223 0 : (entry.mTrackID == TRACK_ANY || entry.mTrackID == aID)) {
2224 0 : return;
2225 : }
2226 : }
2227 0 : TrackBound<MediaStreamVideoSink>* l = mVideoOutputs.AppendElement();
2228 0 : l->mListener = sink;
2229 0 : l->mTrackID = aID;
2230 :
2231 0 : AddDirectTrackListenerImpl(sink.forget(), aID);
2232 : }
2233 :
2234 : void
2235 0 : MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink,
2236 : TrackID aID)
2237 : {
2238 0 : LOG(
2239 : LogLevel::Info,
2240 : ("MediaStream %p Removing MediaStreamVideoSink %p as output", this, aSink));
2241 0 : MOZ_ASSERT(aID != TRACK_NONE);
2242 :
2243 : // Ensure that any frames currently queued for playback by the compositor
2244 : // are removed.
2245 0 : aSink->ClearFrames();
2246 0 : for (size_t i = 0; i < mVideoOutputs.Length(); ++i) {
2247 0 : if (mVideoOutputs[i].mListener == aSink &&
2248 0 : (mVideoOutputs[i].mTrackID == TRACK_ANY ||
2249 0 : mVideoOutputs[i].mTrackID == aID)) {
2250 0 : mVideoOutputs.RemoveElementAt(i);
2251 : }
2252 : }
2253 :
2254 0 : RemoveDirectTrackListenerImpl(aSink, aID);
2255 0 : }
2256 :
2257 : void
2258 0 : MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
2259 : {
2260 0 : class Message : public ControlMessage {
2261 : public:
2262 0 : Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) :
2263 0 : ControlMessage(aStream), mSink(aSink), mID(aID) {}
2264 0 : void Run() override
2265 : {
2266 0 : mStream->AddVideoOutputImpl(mSink.forget(), mID);
2267 0 : }
2268 : RefPtr<MediaStreamVideoSink> mSink;
2269 : TrackID mID;
2270 : };
2271 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink, aID));
2272 0 : }
2273 :
2274 : void
2275 0 : MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
2276 : {
2277 0 : class Message : public ControlMessage {
2278 : public:
2279 0 : Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) :
2280 0 : ControlMessage(aStream), mSink(aSink), mID(aID) {}
2281 0 : void Run() override
2282 : {
2283 0 : mStream->RemoveVideoOutputImpl(mSink, mID);
2284 0 : }
2285 : RefPtr<MediaStreamVideoSink> mSink;
2286 : TrackID mID;
2287 : };
2288 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink, aID));
2289 0 : }
2290 :
2291 : void
2292 0 : MediaStream::Suspend()
2293 : {
2294 0 : class Message : public ControlMessage {
2295 : public:
2296 0 : explicit Message(MediaStream* aStream) :
2297 0 : ControlMessage(aStream) {}
2298 0 : void Run() override
2299 : {
2300 0 : mStream->GraphImpl()->IncrementSuspendCount(mStream);
2301 0 : }
2302 : };
2303 :
2304 : // This can happen if this method has been called asynchronously, and the
2305 : // stream has been destroyed since then.
2306 0 : if (mMainThreadDestroyed) {
2307 0 : return;
2308 : }
2309 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this));
2310 : }
2311 :
2312 : void
2313 0 : MediaStream::Resume()
2314 : {
2315 0 : class Message : public ControlMessage {
2316 : public:
2317 0 : explicit Message(MediaStream* aStream) :
2318 0 : ControlMessage(aStream) {}
2319 0 : void Run() override
2320 : {
2321 0 : mStream->GraphImpl()->DecrementSuspendCount(mStream);
2322 0 : }
2323 : };
2324 :
2325 : // This can happen if this method has been called asynchronously, and the
2326 : // stream has been destroyed since then.
2327 0 : if (mMainThreadDestroyed) {
2328 0 : return;
2329 : }
2330 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this));
2331 : }
2332 :
2333 : void
2334 0 : MediaStream::AddListenerImpl(already_AddRefed<MediaStreamListener> aListener)
2335 : {
2336 0 : MediaStreamListener* listener = *mListeners.AppendElement() = aListener;
2337 0 : listener->NotifyBlockingChanged(GraphImpl(),
2338 0 : mNotifiedBlocked ? MediaStreamListener::BLOCKED : MediaStreamListener::UNBLOCKED);
2339 :
2340 0 : for (StreamTracks::TrackIter it(mTracks); !it.IsEnded(); it.Next()) {
2341 0 : MediaStream* inputStream = nullptr;
2342 0 : TrackID inputTrackID = TRACK_INVALID;
2343 0 : if (ProcessedMediaStream* ps = AsProcessedStream()) {
2344 : // The only ProcessedMediaStream where we should have listeners is
2345 : // TrackUnionStream - it's what's used as owned stream in DOMMediaStream,
2346 : // the only main-thread exposed stream type.
2347 : // TrackUnionStream guarantees that each of its tracks has an input track.
2348 : // Other types do not implement GetInputStreamFor() and will return null.
2349 0 : inputStream = ps->GetInputStreamFor(it->GetID());
2350 0 : MOZ_ASSERT(inputStream);
2351 0 : inputTrackID = ps->GetInputTrackIDFor(it->GetID());
2352 0 : MOZ_ASSERT(IsTrackIDExplicit(inputTrackID));
2353 : }
2354 :
2355 0 : uint32_t flags = TrackEventCommand::TRACK_EVENT_CREATED;
2356 0 : if (it->IsEnded()) {
2357 0 : flags |= TrackEventCommand::TRACK_EVENT_ENDED;
2358 : }
2359 0 : nsAutoPtr<MediaSegment> segment(it->GetSegment()->CreateEmptyClone());
2360 0 : listener->NotifyQueuedTrackChanges(Graph(), it->GetID(), it->GetEnd(),
2361 0 : static_cast<TrackEventCommand>(flags), *segment,
2362 0 : inputStream, inputTrackID);
2363 : }
2364 0 : if (mNotifiedFinished) {
2365 0 : listener->NotifyEvent(GraphImpl(), MediaStreamGraphEvent::EVENT_FINISHED);
2366 : }
2367 0 : if (mNotifiedHasCurrentData) {
2368 0 : listener->NotifyHasCurrentData(GraphImpl());
2369 : }
2370 0 : }
2371 :
2372 : void
2373 0 : MediaStream::AddListener(MediaStreamListener* aListener)
2374 : {
2375 0 : class Message : public ControlMessage {
2376 : public:
2377 0 : Message(MediaStream* aStream, MediaStreamListener* aListener) :
2378 0 : ControlMessage(aStream), mListener(aListener) {}
2379 0 : void Run() override
2380 : {
2381 0 : mStream->AddListenerImpl(mListener.forget());
2382 0 : }
2383 : RefPtr<MediaStreamListener> mListener;
2384 : };
2385 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener));
2386 0 : }
2387 :
2388 : void
2389 0 : MediaStream::RemoveListenerImpl(MediaStreamListener* aListener)
2390 : {
2391 : // wouldn't need this if we could do it in the opposite order
2392 0 : RefPtr<MediaStreamListener> listener(aListener);
2393 0 : mListeners.RemoveElement(aListener);
2394 0 : listener->NotifyEvent(GraphImpl(), MediaStreamGraphEvent::EVENT_REMOVED);
2395 0 : }
2396 :
2397 : void
2398 0 : MediaStream::RemoveListener(MediaStreamListener* aListener)
2399 : {
2400 0 : class Message : public ControlMessage {
2401 : public:
2402 0 : Message(MediaStream* aStream, MediaStreamListener* aListener) :
2403 0 : ControlMessage(aStream), mListener(aListener) {}
2404 0 : void Run() override
2405 : {
2406 0 : mStream->RemoveListenerImpl(mListener);
2407 0 : }
2408 : RefPtr<MediaStreamListener> mListener;
2409 : };
2410 : // If the stream is destroyed the Listeners have or will be
2411 : // removed.
2412 0 : if (!IsDestroyed()) {
2413 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener));
2414 : }
2415 0 : }
2416 :
2417 : void
2418 0 : MediaStream::AddTrackListenerImpl(already_AddRefed<MediaStreamTrackListener> aListener,
2419 : TrackID aTrackID)
2420 : {
2421 0 : TrackBound<MediaStreamTrackListener>* l = mTrackListeners.AppendElement();
2422 0 : l->mListener = aListener;
2423 0 : l->mTrackID = aTrackID;
2424 :
2425 0 : StreamTracks::Track* track = FindTrack(aTrackID);
2426 0 : if (!track) {
2427 0 : return;
2428 : }
2429 : PrincipalHandle lastPrincipalHandle =
2430 0 : track->GetSegment()->GetLastPrincipalHandle();
2431 0 : l->mListener->NotifyPrincipalHandleChanged(Graph(), lastPrincipalHandle);
2432 : }
2433 :
2434 : void
2435 0 : MediaStream::AddTrackListener(MediaStreamTrackListener* aListener,
2436 : TrackID aTrackID)
2437 : {
2438 0 : class Message : public ControlMessage {
2439 : public:
2440 0 : Message(MediaStream* aStream, MediaStreamTrackListener* aListener,
2441 0 : TrackID aTrackID) :
2442 0 : ControlMessage(aStream), mListener(aListener), mTrackID(aTrackID) {}
2443 0 : virtual void Run()
2444 : {
2445 0 : mStream->AddTrackListenerImpl(mListener.forget(), mTrackID);
2446 0 : }
2447 : RefPtr<MediaStreamTrackListener> mListener;
2448 : TrackID mTrackID;
2449 : };
2450 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener, aTrackID));
2451 0 : }
2452 :
2453 : void
2454 0 : MediaStream::RemoveTrackListenerImpl(MediaStreamTrackListener* aListener,
2455 : TrackID aTrackID)
2456 : {
2457 0 : for (size_t i = 0; i < mTrackListeners.Length(); ++i) {
2458 0 : if (mTrackListeners[i].mListener == aListener &&
2459 0 : mTrackListeners[i].mTrackID == aTrackID) {
2460 0 : mTrackListeners[i].mListener->NotifyRemoved();
2461 0 : mTrackListeners.RemoveElementAt(i);
2462 0 : return;
2463 : }
2464 : }
2465 : }
2466 :
2467 : void
2468 0 : MediaStream::RemoveTrackListener(MediaStreamTrackListener* aListener,
2469 : TrackID aTrackID)
2470 : {
2471 0 : class Message : public ControlMessage {
2472 : public:
2473 0 : Message(MediaStream* aStream, MediaStreamTrackListener* aListener,
2474 0 : TrackID aTrackID) :
2475 0 : ControlMessage(aStream), mListener(aListener), mTrackID(aTrackID) {}
2476 0 : virtual void Run()
2477 : {
2478 0 : mStream->RemoveTrackListenerImpl(mListener, mTrackID);
2479 0 : }
2480 : RefPtr<MediaStreamTrackListener> mListener;
2481 : TrackID mTrackID;
2482 : };
2483 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener, aTrackID));
2484 0 : }
2485 :
2486 : void
2487 0 : MediaStream::AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
2488 : TrackID aTrackID)
2489 : {
2490 : // Base implementation, for streams that don't support direct track listeners.
2491 0 : RefPtr<DirectMediaStreamTrackListener> listener = aListener;
2492 0 : listener->NotifyDirectListenerInstalled(
2493 0 : DirectMediaStreamTrackListener::InstallationResult::STREAM_NOT_SUPPORTED);
2494 0 : }
2495 :
2496 : void
2497 0 : MediaStream::AddDirectTrackListener(DirectMediaStreamTrackListener* aListener,
2498 : TrackID aTrackID)
2499 : {
2500 0 : class Message : public ControlMessage {
2501 : public:
2502 0 : Message(MediaStream* aStream, DirectMediaStreamTrackListener* aListener,
2503 0 : TrackID aTrackID) :
2504 0 : ControlMessage(aStream), mListener(aListener), mTrackID(aTrackID) {}
2505 0 : virtual void Run()
2506 : {
2507 0 : mStream->AddDirectTrackListenerImpl(mListener.forget(), mTrackID);
2508 0 : }
2509 : RefPtr<DirectMediaStreamTrackListener> mListener;
2510 : TrackID mTrackID;
2511 : };
2512 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener, aTrackID));
2513 0 : }
2514 :
2515 : void
2516 0 : MediaStream::RemoveDirectTrackListenerImpl(DirectMediaStreamTrackListener* aListener,
2517 : TrackID aTrackID)
2518 : {
2519 : // Base implementation, the listener was never added so nothing to do.
2520 0 : RefPtr<DirectMediaStreamTrackListener> listener = aListener;
2521 0 : }
2522 :
2523 : void
2524 0 : MediaStream::RemoveDirectTrackListener(DirectMediaStreamTrackListener* aListener,
2525 : TrackID aTrackID)
2526 : {
2527 0 : class Message : public ControlMessage {
2528 : public:
2529 0 : Message(MediaStream* aStream, DirectMediaStreamTrackListener* aListener,
2530 0 : TrackID aTrackID) :
2531 0 : ControlMessage(aStream), mListener(aListener), mTrackID(aTrackID) {}
2532 0 : virtual void Run()
2533 : {
2534 0 : mStream->RemoveDirectTrackListenerImpl(mListener, mTrackID);
2535 0 : }
2536 : RefPtr<DirectMediaStreamTrackListener> mListener;
2537 : TrackID mTrackID;
2538 : };
2539 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener, aTrackID));
2540 0 : }
2541 :
2542 : void
2543 0 : MediaStream::RunAfterPendingUpdates(already_AddRefed<nsIRunnable> aRunnable)
2544 : {
2545 0 : MOZ_ASSERT(NS_IsMainThread());
2546 0 : MediaStreamGraphImpl* graph = GraphImpl();
2547 0 : nsCOMPtr<nsIRunnable> runnable(aRunnable);
2548 :
2549 : // Special case when a non-realtime graph has not started, to ensure the
2550 : // runnable will run in finite time.
2551 0 : if (!(graph->mRealtime || graph->mNonRealtimeProcessing)) {
2552 0 : runnable->Run();
2553 0 : return;
2554 : }
2555 :
2556 0 : class Message : public ControlMessage {
2557 : public:
2558 0 : Message(MediaStream* aStream, already_AddRefed<nsIRunnable> aRunnable)
2559 0 : : ControlMessage(aStream)
2560 0 : , mRunnable(aRunnable)
2561 0 : {}
2562 0 : void Run() override
2563 : {
2564 0 : mStream->Graph()->DispatchToMainThreadAfterStreamStateUpdate(
2565 0 : mRunnable.forget());
2566 0 : }
2567 0 : void RunDuringShutdown() override
2568 : {
2569 : // Don't run mRunnable now as it may call AppendMessage() which would
2570 : // assume that there are no remaining controlMessagesToRunDuringShutdown.
2571 0 : MOZ_ASSERT(NS_IsMainThread());
2572 0 : mStream->GraphImpl()->Dispatch(mRunnable.forget());
2573 0 : }
2574 : private:
2575 : nsCOMPtr<nsIRunnable> mRunnable;
2576 : };
2577 :
2578 0 : graph->AppendMessage(MakeUnique<Message>(this, runnable.forget()));
2579 : }
2580 :
2581 : void
2582 0 : MediaStream::SetTrackEnabledImpl(TrackID aTrackID, DisabledTrackMode aMode)
2583 : {
2584 0 : if (aMode == DisabledTrackMode::ENABLED) {
2585 0 : for (int32_t i = mDisabledTracks.Length() - 1; i >= 0; --i) {
2586 0 : if (aTrackID == mDisabledTracks[i].mTrackID) {
2587 0 : mDisabledTracks.RemoveElementAt(i);
2588 0 : return;
2589 : }
2590 : }
2591 : } else {
2592 0 : for (const DisabledTrack& t : mDisabledTracks) {
2593 0 : if (aTrackID == t.mTrackID) {
2594 0 : NS_ERROR("Changing disabled track mode for a track is not allowed");
2595 0 : return;
2596 : }
2597 : }
2598 0 : mDisabledTracks.AppendElement(Move(DisabledTrack(aTrackID, aMode)));
2599 : }
2600 : }
2601 :
2602 : DisabledTrackMode
2603 0 : MediaStream::GetDisabledTrackMode(TrackID aTrackID)
2604 : {
2605 0 : for (const DisabledTrack& t : mDisabledTracks) {
2606 0 : if (t.mTrackID == aTrackID) {
2607 0 : return t.mMode;
2608 : }
2609 : }
2610 0 : return DisabledTrackMode::ENABLED;
2611 : }
2612 :
2613 : void
2614 0 : MediaStream::SetTrackEnabled(TrackID aTrackID, DisabledTrackMode aMode)
2615 : {
2616 0 : class Message : public ControlMessage {
2617 : public:
2618 0 : Message(MediaStream* aStream, TrackID aTrackID, DisabledTrackMode aMode) :
2619 : ControlMessage(aStream),
2620 : mTrackID(aTrackID),
2621 0 : mMode(aMode) {}
2622 0 : void Run() override
2623 : {
2624 0 : mStream->SetTrackEnabledImpl(mTrackID, mMode);
2625 0 : }
2626 : TrackID mTrackID;
2627 : DisabledTrackMode mMode;
2628 : };
2629 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aTrackID, aMode));
2630 0 : }
2631 :
2632 : void
2633 0 : MediaStream::ApplyTrackDisabling(TrackID aTrackID, MediaSegment* aSegment, MediaSegment* aRawSegment)
2634 : {
2635 0 : DisabledTrackMode mode = GetDisabledTrackMode(aTrackID);
2636 0 : if (mode == DisabledTrackMode::ENABLED) {
2637 0 : return;
2638 : }
2639 0 : if (mode == DisabledTrackMode::SILENCE_BLACK) {
2640 0 : aSegment->ReplaceWithDisabled();
2641 0 : if (aRawSegment) {
2642 0 : aRawSegment->ReplaceWithDisabled();
2643 : }
2644 0 : } else if (mode == DisabledTrackMode::SILENCE_FREEZE) {
2645 0 : aSegment->ReplaceWithNull();
2646 0 : if (aRawSegment) {
2647 0 : aRawSegment->ReplaceWithNull();
2648 : }
2649 : } else {
2650 0 : MOZ_CRASH("Unsupported mode");
2651 : }
2652 : }
2653 :
2654 : void
2655 0 : MediaStream::AddMainThreadListener(MainThreadMediaStreamListener* aListener)
2656 : {
2657 0 : MOZ_ASSERT(NS_IsMainThread());
2658 0 : MOZ_ASSERT(aListener);
2659 0 : MOZ_ASSERT(!mMainThreadListeners.Contains(aListener));
2660 :
2661 0 : mMainThreadListeners.AppendElement(aListener);
2662 :
2663 : // If it is not yet time to send the notification, then finish here.
2664 0 : if (!mFinishedNotificationSent) {
2665 0 : return;
2666 : }
2667 :
2668 : class NotifyRunnable final : public Runnable
2669 : {
2670 : public:
2671 0 : explicit NotifyRunnable(MediaStream* aStream)
2672 0 : : Runnable("MediaStream::NotifyRunnable")
2673 0 : , mStream(aStream)
2674 0 : {}
2675 :
2676 0 : NS_IMETHOD Run() override
2677 : {
2678 0 : MOZ_ASSERT(NS_IsMainThread());
2679 0 : mStream->NotifyMainThreadListeners();
2680 0 : return NS_OK;
2681 : }
2682 :
2683 : private:
2684 0 : ~NotifyRunnable() {}
2685 :
2686 : RefPtr<MediaStream> mStream;
2687 : };
2688 :
2689 0 : nsCOMPtr<nsIRunnable> runnable = new NotifyRunnable(this);
2690 0 : GraphImpl()->Dispatch(runnable.forget());
2691 : }
2692 :
2693 0 : SourceMediaStream::SourceMediaStream()
2694 : : MediaStream()
2695 : , mMutex("mozilla::media::SourceMediaStream")
2696 : , mUpdateKnownTracksTime(0)
2697 : , mPullEnabled(false)
2698 : , mUpdateFinished(false)
2699 0 : , mNeedsMixing(false)
2700 : {
2701 0 : }
2702 :
2703 : nsresult
2704 0 : SourceMediaStream::OpenAudioInput(int aID,
2705 : AudioDataListener *aListener)
2706 : {
2707 0 : if (GraphImpl()) {
2708 0 : mInputListener = aListener;
2709 0 : return GraphImpl()->OpenAudioInput(aID, aListener);
2710 : }
2711 0 : return NS_ERROR_FAILURE;
2712 : }
2713 :
2714 : void
2715 0 : SourceMediaStream::CloseAudioInput()
2716 : {
2717 : // Destroy() may have run already and cleared this
2718 0 : if (GraphImpl() && mInputListener) {
2719 0 : GraphImpl()->CloseAudioInput(mInputListener);
2720 : }
2721 0 : mInputListener = nullptr;
2722 0 : }
2723 :
2724 : void
2725 0 : SourceMediaStream::DestroyImpl()
2726 : {
2727 0 : CloseAudioInput();
2728 :
2729 0 : GraphImpl()->AssertOnGraphThreadOrNotRunning();
2730 0 : for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
2731 : // Disconnect before we come under mMutex's lock since it can call back
2732 : // through RemoveDirectTrackListenerImpl() and deadlock.
2733 0 : mConsumers[i]->Disconnect();
2734 : }
2735 :
2736 : // Hold mMutex while mGraph is reset so that other threads holding mMutex
2737 : // can null-check know that the graph will not destroyed.
2738 0 : MutexAutoLock lock(mMutex);
2739 0 : MediaStream::DestroyImpl();
2740 0 : }
2741 :
2742 : void
2743 0 : SourceMediaStream::SetPullEnabled(bool aEnabled)
2744 : {
2745 0 : MutexAutoLock lock(mMutex);
2746 0 : mPullEnabled = aEnabled;
2747 0 : if (mPullEnabled && GraphImpl()) {
2748 0 : GraphImpl()->EnsureNextIteration();
2749 : }
2750 0 : }
2751 :
2752 : void
2753 0 : SourceMediaStream::AddTrackInternal(TrackID aID, TrackRate aRate, StreamTime aStart,
2754 : MediaSegment* aSegment, uint32_t aFlags)
2755 : {
2756 0 : MutexAutoLock lock(mMutex);
2757 0 : nsTArray<TrackData> *track_data = (aFlags & ADDTRACK_QUEUED) ?
2758 0 : &mPendingTracks : &mUpdateTracks;
2759 0 : TrackData* data = track_data->AppendElement();
2760 0 : LOG(LogLevel::Debug,
2761 : ("AddTrackInternal: %lu/%lu",
2762 : (long)mPendingTracks.Length(),
2763 : (long)mUpdateTracks.Length()));
2764 0 : data->mID = aID;
2765 0 : data->mInputRate = aRate;
2766 0 : data->mResamplerChannelCount = 0;
2767 0 : data->mStart = aStart;
2768 0 : data->mEndOfFlushedData = aStart;
2769 0 : data->mCommands = TRACK_CREATE;
2770 0 : data->mData = aSegment;
2771 0 : ResampleAudioToGraphSampleRate(data, aSegment);
2772 0 : if (!(aFlags & ADDTRACK_QUEUED) && GraphImpl()) {
2773 0 : GraphImpl()->EnsureNextIteration();
2774 : }
2775 0 : }
2776 :
2777 : void
2778 0 : SourceMediaStream::AddAudioTrack(TrackID aID, TrackRate aRate, StreamTime aStart,
2779 : AudioSegment* aSegment, uint32_t aFlags)
2780 : {
2781 0 : AddTrackInternal(aID, aRate, aStart, aSegment, aFlags);
2782 0 : }
2783 :
2784 : void
2785 0 : SourceMediaStream::FinishAddTracks()
2786 : {
2787 0 : MutexAutoLock lock(mMutex);
2788 0 : mUpdateTracks.AppendElements(Move(mPendingTracks));
2789 0 : LOG(LogLevel::Debug,
2790 : ("FinishAddTracks: %lu/%lu",
2791 : (long)mPendingTracks.Length(),
2792 : (long)mUpdateTracks.Length()));
2793 0 : if (GraphImpl()) {
2794 0 : GraphImpl()->EnsureNextIteration();
2795 : }
2796 0 : }
2797 :
2798 : void
2799 0 : SourceMediaStream::ResampleAudioToGraphSampleRate(TrackData* aTrackData, MediaSegment* aSegment)
2800 : {
2801 0 : if (aSegment->GetType() != MediaSegment::AUDIO ||
2802 0 : aTrackData->mInputRate == GraphImpl()->GraphRate()) {
2803 0 : return;
2804 : }
2805 0 : AudioSegment* segment = static_cast<AudioSegment*>(aSegment);
2806 0 : int channels = segment->ChannelCount();
2807 :
2808 : // If this segment is just silence, we delay instanciating the resampler. We
2809 : // also need to recreate the resampler if the channel count changes.
2810 0 : if (channels && aTrackData->mResamplerChannelCount != channels) {
2811 0 : SpeexResamplerState* state = speex_resampler_init(channels,
2812 0 : aTrackData->mInputRate,
2813 0 : GraphImpl()->GraphRate(),
2814 : SPEEX_RESAMPLER_QUALITY_MIN,
2815 0 : nullptr);
2816 0 : if (!state) {
2817 0 : return;
2818 : }
2819 0 : aTrackData->mResampler.own(state);
2820 0 : aTrackData->mResamplerChannelCount = channels;
2821 : }
2822 0 : segment->ResampleChunks(aTrackData->mResampler, aTrackData->mInputRate, GraphImpl()->GraphRate());
2823 : }
2824 :
2825 : void
2826 0 : SourceMediaStream::AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
2827 : GraphTime aBlockedTime)
2828 : {
2829 0 : MutexAutoLock lock(mMutex);
2830 0 : mTracksStartTime += aBlockedTime;
2831 0 : mStreamTracksStartTimeStamp += TimeDuration::FromSeconds(GraphImpl()->MediaTimeToSeconds(aBlockedTime));
2832 0 : mTracks.ForgetUpTo(aCurrentTime - mTracksStartTime);
2833 0 : }
2834 :
2835 : bool
2836 0 : SourceMediaStream::AppendToTrack(TrackID aID, MediaSegment* aSegment, MediaSegment *aRawSegment)
2837 : {
2838 0 : MutexAutoLock lock(mMutex);
2839 : // ::EndAllTrackAndFinished() can end these before the sources notice
2840 0 : bool appended = false;
2841 0 : auto graph = GraphImpl();
2842 0 : if (!mFinished && graph) {
2843 0 : TrackData *track = FindDataForTrack(aID);
2844 0 : if (track) {
2845 : // Data goes into mData, and on the next iteration of the MSG moves
2846 : // into the track's segment after NotifyQueuedTrackChanges(). This adds
2847 : // 0-10ms of delay before data gets to direct listeners.
2848 : // Indirect listeners (via subsequent TrackUnion nodes) are synced to
2849 : // playout time, and so can be delayed by buffering.
2850 :
2851 : // Apply track disabling before notifying any consumers directly
2852 : // or inserting into the graph
2853 0 : ApplyTrackDisabling(aID, aSegment, aRawSegment);
2854 :
2855 0 : ResampleAudioToGraphSampleRate(track, aSegment);
2856 :
2857 : // Must notify first, since AppendFrom() will empty out aSegment
2858 0 : NotifyDirectConsumers(track, aRawSegment ? aRawSegment : aSegment);
2859 0 : track->mData->AppendFrom(aSegment); // note: aSegment is now dead
2860 0 : appended = true;
2861 0 : GraphImpl()->EnsureNextIteration();
2862 : } else {
2863 0 : aSegment->Clear();
2864 : }
2865 : }
2866 0 : return appended;
2867 : }
2868 :
2869 : void
2870 0 : SourceMediaStream::NotifyDirectConsumers(TrackData *aTrack,
2871 : MediaSegment *aSegment)
2872 : {
2873 0 : mMutex.AssertCurrentThreadOwns();
2874 0 : MOZ_ASSERT(aTrack);
2875 :
2876 0 : for (uint32_t j = 0; j < mDirectListeners.Length(); ++j) {
2877 0 : DirectMediaStreamListener* l = mDirectListeners[j];
2878 0 : StreamTime offset = 0; // FIX! need a separate StreamTime.... or the end of the internal buffer
2879 0 : l->NotifyRealtimeData(static_cast<MediaStreamGraph*>(GraphImpl()), aTrack->mID,
2880 0 : offset, aTrack->mCommands, *aSegment);
2881 : }
2882 :
2883 0 : for (const TrackBound<DirectMediaStreamTrackListener>& source
2884 0 : : mDirectTrackListeners) {
2885 0 : if (aTrack->mID != source.mTrackID) {
2886 0 : continue;
2887 : }
2888 0 : StreamTime offset = 0; // FIX! need a separate StreamTime.... or the end of the internal buffer
2889 0 : source.mListener->NotifyRealtimeTrackDataAndApplyTrackDisabling(Graph(), offset, *aSegment);
2890 : }
2891 0 : }
2892 :
2893 : // These handle notifying all the listeners of an event
2894 : void
2895 0 : SourceMediaStream::NotifyListenersEventImpl(MediaStreamGraphEvent aEvent)
2896 : {
2897 0 : for (uint32_t j = 0; j < mListeners.Length(); ++j) {
2898 0 : MediaStreamListener* l = mListeners[j];
2899 0 : l->NotifyEvent(GraphImpl(), aEvent);
2900 : }
2901 0 : }
2902 :
2903 : void
2904 0 : SourceMediaStream::NotifyListenersEvent(MediaStreamGraphEvent aNewEvent)
2905 : {
2906 0 : class Message : public ControlMessage {
2907 : public:
2908 0 : Message(SourceMediaStream* aStream, MediaStreamGraphEvent aEvent) :
2909 0 : ControlMessage(aStream), mEvent(aEvent) {}
2910 0 : void Run() override
2911 : {
2912 0 : mStream->AsSourceStream()->NotifyListenersEventImpl(mEvent);
2913 0 : }
2914 : MediaStreamGraphEvent mEvent;
2915 : };
2916 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aNewEvent));
2917 0 : }
2918 :
2919 : void
2920 0 : SourceMediaStream::AddDirectListener(DirectMediaStreamListener* aListener)
2921 : {
2922 : bool wasEmpty;
2923 : {
2924 0 : MutexAutoLock lock(mMutex);
2925 0 : wasEmpty = mDirectListeners.IsEmpty();
2926 0 : mDirectListeners.AppendElement(aListener);
2927 : }
2928 :
2929 0 : if (wasEmpty) {
2930 : // Async
2931 0 : NotifyListenersEvent(MediaStreamGraphEvent::EVENT_HAS_DIRECT_LISTENERS);
2932 : }
2933 0 : }
2934 :
2935 : void
2936 0 : SourceMediaStream::RemoveDirectListener(DirectMediaStreamListener* aListener)
2937 : {
2938 : bool isEmpty;
2939 : {
2940 0 : MutexAutoLock lock(mMutex);
2941 0 : mDirectListeners.RemoveElement(aListener);
2942 0 : isEmpty = mDirectListeners.IsEmpty();
2943 : }
2944 :
2945 0 : if (isEmpty) {
2946 : // Async
2947 0 : NotifyListenersEvent(MediaStreamGraphEvent::EVENT_HAS_NO_DIRECT_LISTENERS);
2948 : }
2949 0 : }
2950 :
2951 : void
2952 0 : SourceMediaStream::AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
2953 : TrackID aTrackID)
2954 : {
2955 0 : MOZ_ASSERT(IsTrackIDExplicit(aTrackID));
2956 0 : TrackData* updateData = nullptr;
2957 0 : StreamTracks::Track* track = nullptr;
2958 0 : bool isAudio = false;
2959 0 : bool isVideo = false;
2960 0 : RefPtr<DirectMediaStreamTrackListener> listener = aListener;
2961 0 : LOG(LogLevel::Debug,
2962 : ("Adding direct track listener %p bound to track %d to source stream %p",
2963 : listener.get(),
2964 : aTrackID,
2965 : this));
2966 :
2967 : {
2968 0 : MutexAutoLock lock(mMutex);
2969 0 : updateData = FindDataForTrack(aTrackID);
2970 0 : track = FindTrack(aTrackID);
2971 0 : if (track) {
2972 0 : isAudio = track->GetType() == MediaSegment::AUDIO;
2973 0 : isVideo = track->GetType() == MediaSegment::VIDEO;
2974 : }
2975 :
2976 0 : if (track && isVideo && listener->AsMediaStreamVideoSink()) {
2977 : // Re-send missed VideoSegment to new added MediaStreamVideoSink.
2978 0 : VideoSegment* trackSegment = static_cast<VideoSegment*>(track->GetSegment());
2979 0 : VideoSegment videoSegment;
2980 0 : if (mTracks.GetForgottenDuration() < trackSegment->GetDuration()) {
2981 0 : videoSegment.AppendSlice(*trackSegment,
2982 : mTracks.GetForgottenDuration(),
2983 0 : trackSegment->GetDuration());
2984 : }
2985 0 : if (updateData) {
2986 0 : videoSegment.AppendSlice(*updateData->mData, 0, updateData->mData->GetDuration());
2987 : }
2988 0 : listener->NotifyRealtimeTrackData(Graph(), 0, videoSegment);
2989 : }
2990 :
2991 0 : if (track && (isAudio || isVideo)) {
2992 0 : for (auto entry : mDirectTrackListeners) {
2993 0 : if (entry.mListener == listener &&
2994 0 : (entry.mTrackID == TRACK_ANY || entry.mTrackID == aTrackID)) {
2995 0 : listener->NotifyDirectListenerInstalled(
2996 0 : DirectMediaStreamTrackListener::InstallationResult::ALREADY_EXISTS);
2997 0 : return;
2998 : }
2999 : }
3000 :
3001 : TrackBound<DirectMediaStreamTrackListener>* sourceListener =
3002 0 : mDirectTrackListeners.AppendElement();
3003 0 : sourceListener->mListener = listener;
3004 0 : sourceListener->mTrackID = aTrackID;
3005 : }
3006 : }
3007 0 : if (!track) {
3008 0 : LOG(LogLevel::Warning,
3009 : ("Couldn't find source track for direct track listener %p",
3010 : listener.get()));
3011 0 : listener->NotifyDirectListenerInstalled(
3012 0 : DirectMediaStreamTrackListener::InstallationResult::TRACK_NOT_FOUND_AT_SOURCE);
3013 0 : return;
3014 : }
3015 0 : if (!isAudio && !isVideo) {
3016 0 : LOG(
3017 : LogLevel::Warning,
3018 : ("Source track for direct track listener %p is unknown", listener.get()));
3019 : // It is not a video or audio track.
3020 0 : MOZ_ASSERT(false);
3021 : return;
3022 : }
3023 0 : LOG(
3024 : LogLevel::Debug,
3025 : ("Added direct track listener %p. ended=%d", listener.get(), !updateData));
3026 0 : listener->NotifyDirectListenerInstalled(
3027 0 : DirectMediaStreamTrackListener::InstallationResult::SUCCESS);
3028 0 : if (!updateData) {
3029 : // The track exists but the mUpdateTracks entry was removed.
3030 : // This means that the track has ended.
3031 0 : listener->NotifyEnded();
3032 : }
3033 : }
3034 :
3035 : void
3036 0 : SourceMediaStream::RemoveDirectTrackListenerImpl(DirectMediaStreamTrackListener* aListener,
3037 : TrackID aTrackID)
3038 : {
3039 0 : MutexAutoLock lock(mMutex);
3040 0 : for (int32_t i = mDirectTrackListeners.Length() - 1; i >= 0; --i) {
3041 : const TrackBound<DirectMediaStreamTrackListener>& source =
3042 0 : mDirectTrackListeners[i];
3043 0 : if (source.mListener == aListener && source.mTrackID == aTrackID) {
3044 0 : aListener->NotifyDirectListenerUninstalled();
3045 0 : mDirectTrackListeners.RemoveElementAt(i);
3046 : }
3047 : }
3048 0 : }
3049 :
3050 : StreamTime
3051 0 : SourceMediaStream::GetEndOfAppendedData(TrackID aID)
3052 : {
3053 0 : MutexAutoLock lock(mMutex);
3054 0 : TrackData *track = FindDataForTrack(aID);
3055 0 : if (track) {
3056 0 : return track->mEndOfFlushedData + track->mData->GetDuration();
3057 : }
3058 0 : NS_ERROR("Track not found");
3059 0 : return 0;
3060 : }
3061 :
3062 : void
3063 0 : SourceMediaStream::EndTrack(TrackID aID)
3064 : {
3065 0 : MutexAutoLock lock(mMutex);
3066 0 : TrackData *track = FindDataForTrack(aID);
3067 0 : if (track) {
3068 0 : track->mCommands |= TrackEventCommand::TRACK_EVENT_ENDED;
3069 : }
3070 0 : if (auto graph = GraphImpl()) {
3071 0 : graph->EnsureNextIteration();
3072 : }
3073 0 : }
3074 :
3075 : void
3076 0 : SourceMediaStream::AdvanceKnownTracksTime(StreamTime aKnownTime)
3077 : {
3078 0 : MutexAutoLock lock(mMutex);
3079 0 : MOZ_ASSERT(aKnownTime >= mUpdateKnownTracksTime);
3080 0 : mUpdateKnownTracksTime = aKnownTime;
3081 0 : if (auto graph = GraphImpl()) {
3082 0 : graph->EnsureNextIteration();
3083 : }
3084 0 : }
3085 :
3086 : void
3087 0 : SourceMediaStream::FinishWithLockHeld()
3088 : {
3089 0 : mMutex.AssertCurrentThreadOwns();
3090 0 : mUpdateFinished = true;
3091 0 : if (auto graph = GraphImpl()) {
3092 0 : graph->EnsureNextIteration();
3093 : }
3094 0 : }
3095 :
3096 : void
3097 0 : SourceMediaStream::SetTrackEnabledImpl(TrackID aTrackID, DisabledTrackMode aMode)
3098 : {
3099 : {
3100 0 : MutexAutoLock lock(mMutex);
3101 0 : for (TrackBound<DirectMediaStreamTrackListener>& l: mDirectTrackListeners) {
3102 0 : if (l.mTrackID != aTrackID) {
3103 0 : continue;
3104 : }
3105 0 : DisabledTrackMode oldMode = GetDisabledTrackMode(aTrackID);
3106 0 : bool oldEnabled = oldMode == DisabledTrackMode::ENABLED;
3107 0 : if (!oldEnabled && aMode == DisabledTrackMode::ENABLED) {
3108 0 : LOG(LogLevel::Debug,
3109 : ("SourceMediaStream %p track %d setting "
3110 : "direct listener enabled",
3111 : this,
3112 : aTrackID));
3113 0 : l.mListener->DecreaseDisabled(oldMode);
3114 0 : } else if (oldEnabled && aMode != DisabledTrackMode::ENABLED) {
3115 0 : LOG(LogLevel::Debug,
3116 : ("SourceMediaStream %p track %d setting "
3117 : "direct listener disabled",
3118 : this,
3119 : aTrackID));
3120 0 : l.mListener->IncreaseDisabled(aMode);
3121 : }
3122 : }
3123 : }
3124 0 : MediaStream::SetTrackEnabledImpl(aTrackID, aMode);
3125 0 : }
3126 :
3127 : void
3128 0 : SourceMediaStream::EndAllTrackAndFinish()
3129 : {
3130 0 : MutexAutoLock lock(mMutex);
3131 0 : for (uint32_t i = 0; i < mUpdateTracks.Length(); ++i) {
3132 0 : SourceMediaStream::TrackData* data = &mUpdateTracks[i];
3133 0 : data->mCommands |= TrackEventCommand::TRACK_EVENT_ENDED;
3134 : }
3135 0 : mPendingTracks.Clear();
3136 0 : FinishWithLockHeld();
3137 : // we will call NotifyEvent() to let GetUserMedia know
3138 0 : }
3139 :
3140 0 : SourceMediaStream::~SourceMediaStream()
3141 : {
3142 0 : }
3143 :
3144 : void
3145 0 : SourceMediaStream::RegisterForAudioMixing()
3146 : {
3147 0 : MutexAutoLock lock(mMutex);
3148 0 : mNeedsMixing = true;
3149 0 : }
3150 :
3151 : bool
3152 0 : SourceMediaStream::NeedsMixing()
3153 : {
3154 0 : MutexAutoLock lock(mMutex);
3155 0 : return mNeedsMixing;
3156 : }
3157 :
3158 : bool
3159 0 : SourceMediaStream::HasPendingAudioTrack()
3160 : {
3161 0 : MutexAutoLock lock(mMutex);
3162 0 : bool audioTrackPresent = false;
3163 :
3164 0 : for (auto& data : mPendingTracks) {
3165 0 : if (data.mData->GetType() == MediaSegment::AUDIO) {
3166 0 : audioTrackPresent = true;
3167 0 : break;
3168 : }
3169 : }
3170 :
3171 0 : return audioTrackPresent;
3172 : }
3173 :
3174 : bool
3175 0 : SourceMediaStream::OpenNewAudioCallbackDriver(AudioDataListener * aListener)
3176 : {
3177 0 : MOZ_ASSERT(GraphImpl()->mLifecycleState ==
3178 : MediaStreamGraphImpl::LifecycleState::LIFECYCLE_RUNNING);
3179 0 : AudioCallbackDriver* nextDriver = new AudioCallbackDriver(GraphImpl());
3180 0 : nextDriver->SetInputListener(aListener);
3181 : {
3182 0 : MonitorAutoLock lock(GraphImpl()->GetMonitor());
3183 0 : GraphImpl()->CurrentDriver()->SwitchAtNextIteration(nextDriver);
3184 : }
3185 :
3186 0 : return true;
3187 : }
3188 :
3189 :
3190 : void
3191 0 : MediaInputPort::Init()
3192 : {
3193 0 : LOG(LogLevel::Debug,
3194 : ("Adding MediaInputPort %p (from %p to %p) to the graph",
3195 : this,
3196 : mSource,
3197 : mDest));
3198 0 : mSource->AddConsumer(this);
3199 0 : mDest->AddInput(this);
3200 : // mPortCount decremented via MediaInputPort::Destroy's message
3201 0 : ++mDest->GraphImpl()->mPortCount;
3202 0 : }
3203 :
3204 : void
3205 0 : MediaInputPort::Disconnect()
3206 : {
3207 0 : GraphImpl()->AssertOnGraphThreadOrNotRunning();
3208 0 : NS_ASSERTION(!mSource == !mDest,
3209 : "mSource must either both be null or both non-null");
3210 0 : if (!mSource)
3211 0 : return;
3212 :
3213 0 : mSource->RemoveConsumer(this);
3214 0 : mDest->RemoveInput(this);
3215 0 : mSource = nullptr;
3216 0 : mDest = nullptr;
3217 :
3218 0 : GraphImpl()->SetStreamOrderDirty();
3219 : }
3220 :
3221 : MediaInputPort::InputInterval
3222 0 : MediaInputPort::GetNextInputInterval(GraphTime aTime)
3223 : {
3224 0 : InputInterval result = { GRAPH_TIME_MAX, GRAPH_TIME_MAX, false };
3225 0 : if (aTime >= mDest->mStartBlocking) {
3226 0 : return result;
3227 : }
3228 0 : result.mStart = aTime;
3229 0 : result.mEnd = mDest->mStartBlocking;
3230 0 : result.mInputIsBlocked = aTime >= mSource->mStartBlocking;
3231 0 : if (!result.mInputIsBlocked) {
3232 0 : result.mEnd = std::min(result.mEnd, mSource->mStartBlocking);
3233 : }
3234 0 : return result;
3235 : }
3236 :
3237 : void
3238 0 : MediaInputPort::Suspended()
3239 : {
3240 0 : mDest->InputSuspended(this);
3241 0 : }
3242 :
3243 : void
3244 0 : MediaInputPort::Resumed()
3245 : {
3246 0 : mDest->InputResumed(this);
3247 0 : }
3248 :
3249 : void
3250 0 : MediaInputPort::Destroy()
3251 : {
3252 0 : class Message : public ControlMessage {
3253 : public:
3254 0 : explicit Message(MediaInputPort* aPort)
3255 0 : : ControlMessage(nullptr), mPort(aPort) {}
3256 0 : void Run() override
3257 : {
3258 0 : mPort->Disconnect();
3259 0 : --mPort->GraphImpl()->mPortCount;
3260 0 : mPort->SetGraphImpl(nullptr);
3261 0 : NS_RELEASE(mPort);
3262 0 : }
3263 0 : void RunDuringShutdown() override
3264 : {
3265 0 : Run();
3266 0 : }
3267 : MediaInputPort* mPort;
3268 : };
3269 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this));
3270 0 : }
3271 :
3272 : MediaStreamGraphImpl*
3273 0 : MediaInputPort::GraphImpl()
3274 : {
3275 0 : return mGraph;
3276 : }
3277 :
3278 : MediaStreamGraph*
3279 0 : MediaInputPort::Graph()
3280 : {
3281 0 : return mGraph;
3282 : }
3283 :
3284 : void
3285 0 : MediaInputPort::SetGraphImpl(MediaStreamGraphImpl* aGraph)
3286 : {
3287 0 : MOZ_ASSERT(!mGraph || !aGraph, "Should only be set once");
3288 0 : mGraph = aGraph;
3289 0 : }
3290 :
3291 : void
3292 0 : MediaInputPort::BlockSourceTrackIdImpl(TrackID aTrackId, BlockingMode aBlockingMode)
3293 : {
3294 0 : mBlockedTracks.AppendElement(Pair<TrackID, BlockingMode>(aTrackId, aBlockingMode));
3295 0 : }
3296 :
3297 : already_AddRefed<Pledge<bool>>
3298 0 : MediaInputPort::BlockSourceTrackId(TrackID aTrackId, BlockingMode aBlockingMode)
3299 : {
3300 0 : class Message : public ControlMessage {
3301 : public:
3302 0 : Message(MediaInputPort* aPort,
3303 : TrackID aTrackId,
3304 : BlockingMode aBlockingMode,
3305 : already_AddRefed<nsIRunnable> aRunnable)
3306 0 : : ControlMessage(aPort->GetDestination())
3307 : , mPort(aPort)
3308 : , mTrackId(aTrackId)
3309 : , mBlockingMode(aBlockingMode)
3310 0 : , mRunnable(aRunnable)
3311 : {
3312 0 : }
3313 0 : void Run() override
3314 : {
3315 0 : mPort->BlockSourceTrackIdImpl(mTrackId, mBlockingMode);
3316 0 : if (mRunnable) {
3317 0 : mStream->Graph()->DispatchToMainThreadAfterStreamStateUpdate(
3318 0 : mRunnable.forget());
3319 : }
3320 0 : }
3321 0 : void RunDuringShutdown() override
3322 : {
3323 0 : Run();
3324 0 : }
3325 : RefPtr<MediaInputPort> mPort;
3326 : TrackID mTrackId;
3327 : BlockingMode mBlockingMode;
3328 : nsCOMPtr<nsIRunnable> mRunnable;
3329 : };
3330 :
3331 0 : MOZ_ASSERT(IsTrackIDExplicit(aTrackId),
3332 : "Only explicit TrackID is allowed");
3333 :
3334 0 : auto pledge = MakeRefPtr<Pledge<bool>>();
3335 0 : nsCOMPtr<nsIRunnable> runnable = NewRunnableFrom([pledge]() {
3336 0 : MOZ_ASSERT(NS_IsMainThread());
3337 0 : pledge->Resolve(true);
3338 0 : return NS_OK;
3339 0 : });
3340 0 : GraphImpl()->AppendMessage(
3341 0 : MakeUnique<Message>(this, aTrackId, aBlockingMode, runnable.forget()));
3342 0 : return pledge.forget();
3343 : }
3344 :
3345 : already_AddRefed<MediaInputPort>
3346 0 : ProcessedMediaStream::AllocateInputPort(MediaStream* aStream, TrackID aTrackID,
3347 : TrackID aDestTrackID,
3348 : uint16_t aInputNumber, uint16_t aOutputNumber,
3349 : nsTArray<TrackID>* aBlockedTracks)
3350 : {
3351 : // This method creates two references to the MediaInputPort: one for
3352 : // the main thread, and one for the MediaStreamGraph.
3353 0 : class Message : public ControlMessage {
3354 : public:
3355 0 : explicit Message(MediaInputPort* aPort)
3356 0 : : ControlMessage(aPort->GetDestination()),
3357 0 : mPort(aPort) {}
3358 0 : void Run() override
3359 : {
3360 0 : mPort->Init();
3361 : // The graph holds its reference implicitly
3362 0 : mPort->GraphImpl()->SetStreamOrderDirty();
3363 0 : Unused << mPort.forget();
3364 0 : }
3365 0 : void RunDuringShutdown() override
3366 : {
3367 0 : Run();
3368 0 : }
3369 : RefPtr<MediaInputPort> mPort;
3370 : };
3371 :
3372 0 : MOZ_ASSERT(aStream->GraphImpl() == GraphImpl());
3373 0 : MOZ_ASSERT(aTrackID == TRACK_ANY || IsTrackIDExplicit(aTrackID),
3374 : "Only TRACK_ANY and explicit ID are allowed for source track");
3375 0 : MOZ_ASSERT(aDestTrackID == TRACK_ANY || IsTrackIDExplicit(aDestTrackID),
3376 : "Only TRACK_ANY and explicit ID are allowed for destination track");
3377 0 : MOZ_ASSERT(aTrackID != TRACK_ANY || aDestTrackID == TRACK_ANY,
3378 : "Generic MediaInputPort cannot produce a single destination track");
3379 : RefPtr<MediaInputPort> port = new MediaInputPort(
3380 0 : aStream, aTrackID, this, aDestTrackID, aInputNumber, aOutputNumber);
3381 0 : if (aBlockedTracks) {
3382 0 : for (TrackID trackID : *aBlockedTracks) {
3383 0 : port->BlockSourceTrackIdImpl(trackID, BlockingMode::CREATION);
3384 : }
3385 : }
3386 0 : port->SetGraphImpl(GraphImpl());
3387 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(port));
3388 0 : return port.forget();
3389 : }
3390 :
3391 : void
3392 0 : ProcessedMediaStream::Finish()
3393 : {
3394 0 : class Message : public ControlMessage {
3395 : public:
3396 0 : explicit Message(ProcessedMediaStream* aStream)
3397 0 : : ControlMessage(aStream) {}
3398 0 : void Run() override
3399 : {
3400 0 : mStream->GraphImpl()->FinishStream(mStream);
3401 0 : }
3402 : };
3403 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this));
3404 0 : }
3405 :
3406 : void
3407 0 : ProcessedMediaStream::SetAutofinish(bool aAutofinish)
3408 : {
3409 0 : class Message : public ControlMessage {
3410 : public:
3411 0 : Message(ProcessedMediaStream* aStream, bool aAutofinish)
3412 0 : : ControlMessage(aStream), mAutofinish(aAutofinish) {}
3413 0 : void Run() override
3414 : {
3415 0 : static_cast<ProcessedMediaStream*>(mStream)->SetAutofinishImpl(mAutofinish);
3416 0 : }
3417 : bool mAutofinish;
3418 : };
3419 0 : GraphImpl()->AppendMessage(MakeUnique<Message>(this, aAutofinish));
3420 0 : }
3421 :
3422 : void
3423 0 : ProcessedMediaStream::DestroyImpl()
3424 : {
3425 0 : for (int32_t i = mInputs.Length() - 1; i >= 0; --i) {
3426 0 : mInputs[i]->Disconnect();
3427 : }
3428 :
3429 0 : for (int32_t i = mSuspendedInputs.Length() - 1; i >= 0; --i) {
3430 0 : mSuspendedInputs[i]->Disconnect();
3431 : }
3432 :
3433 0 : MediaStream::DestroyImpl();
3434 : // The stream order is only important if there are connections, in which
3435 : // case MediaInputPort::Disconnect() called SetStreamOrderDirty().
3436 : // MediaStreamGraphImpl::RemoveStreamGraphThread() will also call
3437 : // SetStreamOrderDirty(), for other reasons.
3438 0 : }
3439 :
3440 0 : MediaStreamGraphImpl::MediaStreamGraphImpl(GraphDriverType aDriverRequested,
3441 : TrackRate aSampleRate,
3442 : dom::AudioChannel aChannel,
3443 0 : AbstractThread* aMainThread)
3444 : : MediaStreamGraph(aSampleRate)
3445 : , mPortCount(0)
3446 : , mInputWanted(false)
3447 : , mInputDeviceID(-1)
3448 : , mOutputWanted(true)
3449 : , mOutputDeviceID(-1)
3450 : , mNeedAnotherIteration(false)
3451 : , mGraphDriverAsleep(false)
3452 : , mMonitor("MediaStreamGraphImpl")
3453 : , mLifecycleState(LIFECYCLE_THREAD_NOT_STARTED)
3454 : , mEndTime(GRAPH_TIME_MAX)
3455 : , mForceShutDown(false)
3456 : , mPostedRunInStableStateEvent(false)
3457 : , mDetectedNotRunning(false)
3458 : , mPostedRunInStableState(false)
3459 0 : , mRealtime(aDriverRequested != OFFLINE_THREAD_DRIVER)
3460 : , mNonRealtimeProcessing(false)
3461 : , mStreamOrderDirty(false)
3462 : , mLatencyLog(AsyncLatencyLogger::Get())
3463 : , mAbstractMainThread(aMainThread)
3464 : #ifdef MOZ_WEBRTC
3465 : , mFarendObserverRef(nullptr)
3466 : #endif
3467 : , mSelfRef(this)
3468 : #ifdef DEBUG
3469 : , mCanRunMessagesSynchronously(false)
3470 : #endif
3471 0 : , mAudioChannel(aChannel)
3472 : {
3473 0 : if (mRealtime) {
3474 0 : if (aDriverRequested == AUDIO_THREAD_DRIVER) {
3475 0 : AudioCallbackDriver* driver = new AudioCallbackDriver(this);
3476 0 : mDriver = driver;
3477 : } else {
3478 0 : mDriver = new SystemClockDriver(this);
3479 : }
3480 : } else {
3481 0 : mDriver = new OfflineClockDriver(this, MEDIA_GRAPH_TARGET_PERIOD_MS);
3482 : }
3483 :
3484 0 : mLastMainThreadUpdate = TimeStamp::Now();
3485 :
3486 0 : RegisterWeakAsyncMemoryReporter(this);
3487 0 : }
3488 :
3489 : AbstractThread*
3490 0 : MediaStreamGraph::AbstractMainThread()
3491 : {
3492 0 : MOZ_ASSERT(static_cast<MediaStreamGraphImpl*>(this)->mAbstractMainThread);
3493 0 : return static_cast<MediaStreamGraphImpl*>(this)->mAbstractMainThread;
3494 : }
3495 :
3496 : void
3497 0 : MediaStreamGraphImpl::Destroy()
3498 : {
3499 : // First unregister from memory reporting.
3500 0 : UnregisterWeakMemoryReporter(this);
3501 :
3502 : // Clear the self reference which will destroy this instance.
3503 0 : mSelfRef = nullptr;
3504 0 : }
3505 :
3506 : static
3507 0 : uint32_t ChannelAndWindowToHash(dom::AudioChannel aChannel,
3508 : nsPIDOMWindowInner* aWindow)
3509 : {
3510 0 : uint32_t hashkey = 0;
3511 :
3512 0 : hashkey = AddToHash(hashkey, static_cast<uint32_t>(aChannel));
3513 0 : hashkey = AddToHash(hashkey, aWindow);
3514 :
3515 0 : return hashkey;
3516 : }
3517 :
3518 : MediaStreamGraph*
3519 0 : MediaStreamGraph::GetInstance(MediaStreamGraph::GraphDriverType aGraphDriverRequested,
3520 : dom::AudioChannel aChannel,
3521 : nsPIDOMWindowInner* aWindow)
3522 : {
3523 0 : NS_ASSERTION(NS_IsMainThread(), "Main thread only");
3524 :
3525 0 : uint32_t channel = static_cast<uint32_t>(aChannel);
3526 0 : MediaStreamGraphImpl* graph = nullptr;
3527 :
3528 : // We hash the AudioChannel and the nsPIDOMWindowInner together to form a key
3529 : // to the gloabl MediaStreamGraph hashtable. Effectively, this means there is
3530 : // a graph per document and AudioChannel.
3531 :
3532 :
3533 0 : uint32_t hashkey = ChannelAndWindowToHash(aChannel, aWindow);
3534 :
3535 0 : if (!gGraphs.Get(hashkey, &graph)) {
3536 0 : if (!gMediaStreamGraphShutdownBlocker) {
3537 :
3538 0 : class Blocker : public media::ShutdownBlocker
3539 : {
3540 : public:
3541 0 : Blocker()
3542 0 : : media::ShutdownBlocker(NS_LITERAL_STRING(
3543 0 : "MediaStreamGraph shutdown: blocking on msg thread"))
3544 0 : {}
3545 :
3546 : NS_IMETHOD
3547 0 : BlockShutdown(nsIAsyncShutdownClient* aProfileBeforeChange) override
3548 : {
3549 : // Distribute the global async shutdown blocker in a ticket. If there
3550 : // are zero graphs then shutdown is unblocked when we go out of scope.
3551 : RefPtr<MediaStreamGraphImpl::ShutdownTicket> ticket =
3552 0 : new MediaStreamGraphImpl::ShutdownTicket(gMediaStreamGraphShutdownBlocker.get());
3553 0 : gMediaStreamGraphShutdownBlocker = nullptr;
3554 :
3555 0 : for (auto iter = gGraphs.Iter(); !iter.Done(); iter.Next()) {
3556 0 : iter.UserData()->ForceShutDown(ticket);
3557 : }
3558 0 : return NS_OK;
3559 : }
3560 : };
3561 :
3562 0 : gMediaStreamGraphShutdownBlocker = new Blocker();
3563 0 : nsCOMPtr<nsIAsyncShutdownClient> barrier = MediaStreamGraphImpl::GetShutdownBarrier();
3564 0 : nsresult rv = barrier->
3565 0 : AddBlocker(gMediaStreamGraphShutdownBlocker,
3566 0 : NS_LITERAL_STRING(__FILE__), __LINE__,
3567 0 : NS_LITERAL_STRING("MediaStreamGraph shutdown"));
3568 0 : MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
3569 : }
3570 :
3571 : AbstractThread* mainThread;
3572 0 : if (aWindow) {
3573 0 : nsCOMPtr<nsIGlobalObject> parentObject = do_QueryInterface(aWindow);
3574 0 : mainThread = parentObject->AbstractMainThreadFor(TaskCategory::Other);
3575 : } else {
3576 : // Uncommon case, only for some old configuration of webspeech.
3577 0 : mainThread = AbstractThread::MainThread();
3578 : }
3579 0 : graph = new MediaStreamGraphImpl(aGraphDriverRequested,
3580 0 : CubebUtils::PreferredSampleRate(),
3581 : aChannel,
3582 0 : mainThread);
3583 :
3584 0 : gGraphs.Put(hashkey, graph);
3585 :
3586 0 : LOG(LogLevel::Debug,
3587 : ("Starting up MediaStreamGraph %p for channel %s and window %p",
3588 : graph, AudioChannelValues::strings[channel].value, aWindow));
3589 : }
3590 :
3591 0 : return graph;
3592 : }
3593 :
3594 : MediaStreamGraph*
3595 0 : MediaStreamGraph::CreateNonRealtimeInstance(TrackRate aSampleRate,
3596 : nsPIDOMWindowInner* aWindow)
3597 : {
3598 0 : NS_ASSERTION(NS_IsMainThread(), "Main thread only");
3599 :
3600 0 : nsCOMPtr<nsIGlobalObject> parentObject = do_QueryInterface(aWindow);
3601 : MediaStreamGraphImpl* graph = new MediaStreamGraphImpl(
3602 : OFFLINE_THREAD_DRIVER,
3603 : aSampleRate,
3604 : AudioChannel::Normal,
3605 0 : parentObject->AbstractMainThreadFor(TaskCategory::Other));
3606 :
3607 0 : LOG(LogLevel::Debug, ("Starting up Offline MediaStreamGraph %p", graph));
3608 :
3609 0 : return graph;
3610 : }
3611 :
3612 : void
3613 0 : MediaStreamGraph::DestroyNonRealtimeInstance(MediaStreamGraph* aGraph)
3614 : {
3615 0 : NS_ASSERTION(NS_IsMainThread(), "Main thread only");
3616 0 : MOZ_ASSERT(aGraph->IsNonRealtime(), "Should not destroy the global graph here");
3617 :
3618 0 : MediaStreamGraphImpl* graph = static_cast<MediaStreamGraphImpl*>(aGraph);
3619 :
3620 0 : if (!graph->mNonRealtimeProcessing) {
3621 : // Start the graph, but don't produce anything
3622 0 : graph->StartNonRealtimeProcessing(0);
3623 : }
3624 0 : graph->ForceShutDown(nullptr);
3625 0 : }
3626 :
3627 0 : NS_IMPL_ISUPPORTS(MediaStreamGraphImpl, nsIMemoryReporter, nsITimerCallback)
3628 :
3629 : NS_IMETHODIMP
3630 0 : MediaStreamGraphImpl::CollectReports(nsIHandleReportCallback* aHandleReport,
3631 : nsISupports* aData, bool aAnonymize)
3632 : {
3633 0 : if (mLifecycleState >= LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN) {
3634 : // Shutting down, nothing to report.
3635 0 : FinishCollectReports(aHandleReport, aData, nsTArray<AudioNodeSizes>());
3636 0 : return NS_OK;
3637 : }
3638 :
3639 0 : class Message final : public ControlMessage {
3640 : public:
3641 0 : Message(MediaStreamGraphImpl *aGraph,
3642 : nsIHandleReportCallback* aHandleReport,
3643 : nsISupports *aHandlerData)
3644 0 : : ControlMessage(nullptr)
3645 : , mGraph(aGraph)
3646 : , mHandleReport(aHandleReport)
3647 0 : , mHandlerData(aHandlerData) {}
3648 0 : void Run() override
3649 : {
3650 0 : mGraph->CollectSizesForMemoryReport(mHandleReport.forget(),
3651 0 : mHandlerData.forget());
3652 0 : }
3653 0 : void RunDuringShutdown() override
3654 : {
3655 : // Run this message during shutdown too, so that endReports is called.
3656 0 : Run();
3657 0 : }
3658 : MediaStreamGraphImpl *mGraph;
3659 : // nsMemoryReporterManager keeps the callback and data alive only if it
3660 : // does not time out.
3661 : nsCOMPtr<nsIHandleReportCallback> mHandleReport;
3662 : nsCOMPtr<nsISupports> mHandlerData;
3663 : };
3664 :
3665 : // When a non-realtime graph has not started, there is no thread yet, so
3666 : // collect sizes on this thread.
3667 0 : if (!(mRealtime || mNonRealtimeProcessing)) {
3668 0 : CollectSizesForMemoryReport(do_AddRef(aHandleReport), do_AddRef(aData));
3669 0 : return NS_OK;
3670 : }
3671 :
3672 0 : AppendMessage(MakeUnique<Message>(this, aHandleReport, aData));
3673 :
3674 0 : return NS_OK;
3675 : }
3676 :
3677 : void
3678 0 : MediaStreamGraphImpl::CollectSizesForMemoryReport(
3679 : already_AddRefed<nsIHandleReportCallback> aHandleReport,
3680 : already_AddRefed<nsISupports> aHandlerData)
3681 : {
3682 : class FinishCollectRunnable final : public Runnable
3683 : {
3684 : public:
3685 0 : explicit FinishCollectRunnable(
3686 : already_AddRefed<nsIHandleReportCallback> aHandleReport,
3687 : already_AddRefed<nsISupports> aHandlerData)
3688 0 : : mozilla::Runnable("FinishCollectRunnable")
3689 : , mHandleReport(aHandleReport)
3690 0 : , mHandlerData(aHandlerData)
3691 0 : {}
3692 :
3693 0 : NS_IMETHOD Run() override
3694 : {
3695 0 : MediaStreamGraphImpl::FinishCollectReports(mHandleReport, mHandlerData,
3696 0 : Move(mAudioStreamSizes));
3697 0 : return NS_OK;
3698 : }
3699 :
3700 : nsTArray<AudioNodeSizes> mAudioStreamSizes;
3701 :
3702 : private:
3703 0 : ~FinishCollectRunnable() {}
3704 :
3705 : // Avoiding nsCOMPtr because NSCAP_ASSERT_NO_QUERY_NEEDED in its
3706 : // constructor modifies the ref-count, which cannot be done off main
3707 : // thread.
3708 : RefPtr<nsIHandleReportCallback> mHandleReport;
3709 : RefPtr<nsISupports> mHandlerData;
3710 : };
3711 :
3712 : RefPtr<FinishCollectRunnable> runnable =
3713 0 : new FinishCollectRunnable(Move(aHandleReport), Move(aHandlerData));
3714 :
3715 0 : auto audioStreamSizes = &runnable->mAudioStreamSizes;
3716 :
3717 0 : for (MediaStream* s : AllStreams()) {
3718 0 : AudioNodeStream* stream = s->AsAudioNodeStream();
3719 0 : if (stream) {
3720 0 : AudioNodeSizes* usage = audioStreamSizes->AppendElement();
3721 0 : stream->SizeOfAudioNodesIncludingThis(MallocSizeOf, *usage);
3722 : }
3723 : }
3724 :
3725 0 : mAbstractMainThread->Dispatch(runnable.forget());
3726 0 : }
3727 :
3728 : void
3729 0 : MediaStreamGraphImpl::
3730 : FinishCollectReports(nsIHandleReportCallback* aHandleReport, nsISupports* aData,
3731 : const nsTArray<AudioNodeSizes>& aAudioStreamSizes)
3732 : {
3733 0 : MOZ_ASSERT(NS_IsMainThread());
3734 :
3735 : nsCOMPtr<nsIMemoryReporterManager> manager =
3736 0 : do_GetService("@mozilla.org/memory-reporter-manager;1");
3737 :
3738 0 : if (!manager)
3739 0 : return;
3740 :
3741 : #define REPORT(_path, _amount, _desc) \
3742 : aHandleReport->Callback(EmptyCString(), _path, KIND_HEAP, UNITS_BYTES, \
3743 : _amount, NS_LITERAL_CSTRING(_desc), aData);
3744 :
3745 0 : for (size_t i = 0; i < aAudioStreamSizes.Length(); i++) {
3746 0 : const AudioNodeSizes& usage = aAudioStreamSizes[i];
3747 : const char* const nodeType =
3748 0 : usage.mNodeType ? usage.mNodeType : "<unknown>";
3749 :
3750 : nsPrintfCString enginePath("explicit/webaudio/audio-node/%s/engine-objects",
3751 0 : nodeType);
3752 0 : REPORT(enginePath, usage.mEngine,
3753 : "Memory used by AudioNode engine objects (Web Audio).");
3754 :
3755 : nsPrintfCString streamPath("explicit/webaudio/audio-node/%s/stream-objects",
3756 0 : nodeType);
3757 0 : REPORT(streamPath, usage.mStream,
3758 : "Memory used by AudioNode stream objects (Web Audio).");
3759 :
3760 : }
3761 :
3762 0 : size_t hrtfLoaders = WebCore::HRTFDatabaseLoader::sizeOfLoaders(MallocSizeOf);
3763 0 : if (hrtfLoaders) {
3764 0 : REPORT(NS_LITERAL_CSTRING(
3765 : "explicit/webaudio/audio-node/PannerNode/hrtf-databases"),
3766 : hrtfLoaders,
3767 : "Memory used by PannerNode databases (Web Audio).");
3768 : }
3769 :
3770 : #undef REPORT
3771 :
3772 0 : manager->EndReport();
3773 : }
3774 :
3775 : SourceMediaStream*
3776 0 : MediaStreamGraph::CreateSourceStream()
3777 : {
3778 0 : SourceMediaStream* stream = new SourceMediaStream();
3779 0 : AddStream(stream);
3780 0 : return stream;
3781 : }
3782 :
3783 : ProcessedMediaStream*
3784 0 : MediaStreamGraph::CreateTrackUnionStream()
3785 : {
3786 0 : TrackUnionStream* stream = new TrackUnionStream();
3787 0 : AddStream(stream);
3788 0 : return stream;
3789 : }
3790 :
3791 : ProcessedMediaStream*
3792 0 : MediaStreamGraph::CreateAudioCaptureStream(TrackID aTrackId)
3793 : {
3794 0 : AudioCaptureStream* stream = new AudioCaptureStream(aTrackId);
3795 0 : AddStream(stream);
3796 0 : return stream;
3797 : }
3798 :
3799 : void
3800 0 : MediaStreamGraph::AddStream(MediaStream* aStream)
3801 : {
3802 0 : NS_ADDREF(aStream);
3803 0 : MediaStreamGraphImpl* graph = static_cast<MediaStreamGraphImpl*>(this);
3804 0 : aStream->SetGraphImpl(graph);
3805 0 : graph->AppendMessage(MakeUnique<CreateMessage>(aStream));
3806 0 : }
3807 :
3808 0 : class GraphStartedRunnable final : public Runnable
3809 : {
3810 : public:
3811 0 : GraphStartedRunnable(AudioNodeStream* aStream, MediaStreamGraph* aGraph)
3812 0 : : Runnable("GraphStartedRunnable")
3813 : , mStream(aStream)
3814 0 : , mGraph(aGraph)
3815 0 : { }
3816 :
3817 0 : NS_IMETHOD Run() override {
3818 0 : mGraph->NotifyWhenGraphStarted(mStream);
3819 0 : return NS_OK;
3820 : }
3821 :
3822 : private:
3823 : RefPtr<AudioNodeStream> mStream;
3824 : MediaStreamGraph* mGraph;
3825 : };
3826 :
3827 : void
3828 0 : MediaStreamGraph::NotifyWhenGraphStarted(AudioNodeStream* aStream)
3829 : {
3830 0 : MOZ_ASSERT(NS_IsMainThread());
3831 :
3832 0 : class GraphStartedNotificationControlMessage : public ControlMessage
3833 : {
3834 : public:
3835 0 : explicit GraphStartedNotificationControlMessage(AudioNodeStream* aStream)
3836 0 : : ControlMessage(aStream)
3837 : {
3838 0 : }
3839 0 : void Run() override
3840 : {
3841 : // This runs on the graph thread, so when this runs, and the current
3842 : // driver is an AudioCallbackDriver, we know the audio hardware is
3843 : // started. If not, we are going to switch soon, keep reposting this
3844 : // ControlMessage.
3845 0 : MediaStreamGraphImpl* graphImpl = mStream->GraphImpl();
3846 0 : if (graphImpl->CurrentDriver()->AsAudioCallbackDriver()) {
3847 : nsCOMPtr<nsIRunnable> event = new dom::StateChangeTask(
3848 0 : mStream->AsAudioNodeStream(), nullptr, AudioContextState::Running);
3849 0 : graphImpl->Dispatch(event.forget());
3850 : } else {
3851 : nsCOMPtr<nsIRunnable> event = new GraphStartedRunnable(
3852 0 : mStream->AsAudioNodeStream(), mStream->Graph());
3853 0 : graphImpl->Dispatch(event.forget());
3854 : }
3855 0 : }
3856 0 : void RunDuringShutdown() override
3857 : {
3858 0 : }
3859 : };
3860 :
3861 0 : if (!aStream->IsDestroyed()) {
3862 0 : MediaStreamGraphImpl* graphImpl = static_cast<MediaStreamGraphImpl*>(this);
3863 0 : graphImpl->AppendMessage(MakeUnique<GraphStartedNotificationControlMessage>(aStream));
3864 : }
3865 0 : }
3866 :
3867 : void
3868 0 : MediaStreamGraphImpl::IncrementSuspendCount(MediaStream* aStream)
3869 : {
3870 0 : if (!aStream->IsSuspended()) {
3871 0 : MOZ_ASSERT(mStreams.Contains(aStream));
3872 0 : mStreams.RemoveElement(aStream);
3873 0 : mSuspendedStreams.AppendElement(aStream);
3874 0 : SetStreamOrderDirty();
3875 : }
3876 0 : aStream->IncrementSuspendCount();
3877 0 : }
3878 :
3879 : void
3880 0 : MediaStreamGraphImpl::DecrementSuspendCount(MediaStream* aStream)
3881 : {
3882 0 : bool wasSuspended = aStream->IsSuspended();
3883 0 : aStream->DecrementSuspendCount();
3884 0 : if (wasSuspended && !aStream->IsSuspended()) {
3885 0 : MOZ_ASSERT(mSuspendedStreams.Contains(aStream));
3886 0 : mSuspendedStreams.RemoveElement(aStream);
3887 0 : mStreams.AppendElement(aStream);
3888 0 : ProcessedMediaStream* ps = aStream->AsProcessedStream();
3889 0 : if (ps) {
3890 0 : ps->mCycleMarker = NOT_VISITED;
3891 : }
3892 0 : SetStreamOrderDirty();
3893 : }
3894 0 : }
3895 :
3896 : void
3897 0 : MediaStreamGraphImpl::SuspendOrResumeStreams(AudioContextOperation aAudioContextOperation,
3898 : const nsTArray<MediaStream*>& aStreamSet)
3899 : {
3900 : // For our purpose, Suspend and Close are equivalent: we want to remove the
3901 : // streams from the set of streams that are going to be processed.
3902 0 : for (MediaStream* stream : aStreamSet) {
3903 0 : if (aAudioContextOperation == AudioContextOperation::Resume) {
3904 0 : DecrementSuspendCount(stream);
3905 : } else {
3906 0 : IncrementSuspendCount(stream);
3907 : }
3908 : }
3909 0 : LOG(LogLevel::Debug,
3910 : ("Moving streams between suspended and running"
3911 : "state: mStreams: %" PRIuSIZE ", mSuspendedStreams: %" PRIuSIZE,
3912 : mStreams.Length(),
3913 : mSuspendedStreams.Length()));
3914 : #ifdef DEBUG
3915 : // The intersection of the two arrays should be null.
3916 0 : for (uint32_t i = 0; i < mStreams.Length(); i++) {
3917 0 : for (uint32_t j = 0; j < mSuspendedStreams.Length(); j++) {
3918 0 : MOZ_ASSERT(
3919 : mStreams[i] != mSuspendedStreams[j],
3920 : "The suspended stream set and running stream set are not disjoint.");
3921 : }
3922 : }
3923 : #endif
3924 0 : }
3925 :
3926 : void
3927 0 : MediaStreamGraphImpl::AudioContextOperationCompleted(MediaStream* aStream,
3928 : void* aPromise,
3929 : AudioContextOperation aOperation)
3930 : {
3931 : // This can be called from the thread created to do cubeb operation, or the
3932 : // MSG thread. The pointers passed back here are refcounted, so are still
3933 : // alive.
3934 0 : MonitorAutoLock lock(mMonitor);
3935 :
3936 : AudioContextState state;
3937 0 : switch (aOperation) {
3938 : case AudioContextOperation::Suspend:
3939 0 : state = AudioContextState::Suspended;
3940 0 : break;
3941 : case AudioContextOperation::Resume:
3942 0 : state = AudioContextState::Running;
3943 0 : break;
3944 : case AudioContextOperation::Close:
3945 0 : state = AudioContextState::Closed;
3946 0 : break;
3947 0 : default: MOZ_CRASH("Not handled.");
3948 : }
3949 :
3950 : nsCOMPtr<nsIRunnable> event = new dom::StateChangeTask(
3951 0 : aStream->AsAudioNodeStream(), aPromise, state);
3952 0 : mAbstractMainThread->Dispatch(event.forget());
3953 0 : }
3954 :
3955 : void
3956 0 : MediaStreamGraphImpl::ApplyAudioContextOperationImpl(
3957 : MediaStream* aDestinationStream, const nsTArray<MediaStream*>& aStreams,
3958 : AudioContextOperation aOperation, void* aPromise)
3959 : {
3960 0 : MOZ_ASSERT(CurrentDriver()->OnThread());
3961 :
3962 0 : SuspendOrResumeStreams(aOperation, aStreams);
3963 :
3964 0 : bool switching = false;
3965 0 : GraphDriver* nextDriver = nullptr;
3966 : {
3967 0 : MonitorAutoLock lock(mMonitor);
3968 0 : switching = CurrentDriver()->Switching();
3969 0 : if (switching) {
3970 0 : nextDriver = CurrentDriver()->NextDriver();
3971 : }
3972 : }
3973 :
3974 : // If we have suspended the last AudioContext, and we don't have other
3975 : // streams that have audio, this graph will automatically switch to a
3976 : // SystemCallbackDriver, because it can't find a MediaStream that has an audio
3977 : // track. When resuming, force switching to an AudioCallbackDriver (if we're
3978 : // not already switching). It would have happened at the next iteration
3979 : // anyways, but doing this now save some time.
3980 0 : if (aOperation == AudioContextOperation::Resume) {
3981 0 : if (!CurrentDriver()->AsAudioCallbackDriver()) {
3982 : AudioCallbackDriver* driver;
3983 0 : if (switching) {
3984 0 : MOZ_ASSERT(nextDriver->AsAudioCallbackDriver());
3985 0 : driver = nextDriver->AsAudioCallbackDriver();
3986 : } else {
3987 0 : driver = new AudioCallbackDriver(this);
3988 0 : MonitorAutoLock lock(mMonitor);
3989 0 : CurrentDriver()->SwitchAtNextIteration(driver);
3990 : }
3991 : driver->EnqueueStreamAndPromiseForOperation(aDestinationStream,
3992 0 : aPromise, aOperation);
3993 : } else {
3994 : // We are resuming a context, but we are already using an
3995 : // AudioCallbackDriver, we can resolve the promise now.
3996 0 : AudioContextOperationCompleted(aDestinationStream, aPromise, aOperation);
3997 : }
3998 : }
3999 : // Close, suspend: check if we are going to switch to a
4000 : // SystemAudioCallbackDriver, and pass the promise to the AudioCallbackDriver
4001 : // if that's the case, so it can notify the content.
4002 : // This is the same logic as in UpdateStreamOrder, but it's simpler to have it
4003 : // here as well so we don't have to store the Promise(s) on the Graph.
4004 0 : if (aOperation != AudioContextOperation::Resume) {
4005 0 : bool shouldAEC = false;
4006 0 : bool audioTrackPresent = AudioTrackPresent(shouldAEC);
4007 :
4008 0 : if (!audioTrackPresent && CurrentDriver()->AsAudioCallbackDriver()) {
4009 0 : CurrentDriver()->AsAudioCallbackDriver()->
4010 0 : EnqueueStreamAndPromiseForOperation(aDestinationStream, aPromise,
4011 0 : aOperation);
4012 :
4013 : SystemClockDriver* driver;
4014 0 : if (nextDriver) {
4015 0 : MOZ_ASSERT(!nextDriver->AsAudioCallbackDriver());
4016 : } else {
4017 0 : driver = new SystemClockDriver(this);
4018 0 : MonitorAutoLock lock(mMonitor);
4019 0 : CurrentDriver()->SwitchAtNextIteration(driver);
4020 : }
4021 : // We are closing or suspending an AudioContext, but we just got resumed.
4022 : // Queue the operation on the next driver so that the ordering is
4023 : // preserved.
4024 0 : } else if (!audioTrackPresent && switching) {
4025 0 : MOZ_ASSERT(nextDriver->AsAudioCallbackDriver());
4026 0 : nextDriver->AsAudioCallbackDriver()->
4027 0 : EnqueueStreamAndPromiseForOperation(aDestinationStream, aPromise,
4028 0 : aOperation);
4029 : } else {
4030 : // We are closing or suspending an AudioContext, but something else is
4031 : // using the audio stream, we can resolve the promise now.
4032 0 : AudioContextOperationCompleted(aDestinationStream, aPromise, aOperation);
4033 : }
4034 : }
4035 0 : }
4036 :
4037 : void
4038 0 : MediaStreamGraph::ApplyAudioContextOperation(MediaStream* aDestinationStream,
4039 : const nsTArray<MediaStream*>& aStreams,
4040 : AudioContextOperation aOperation,
4041 : void* aPromise)
4042 : {
4043 0 : class AudioContextOperationControlMessage : public ControlMessage
4044 : {
4045 : public:
4046 0 : AudioContextOperationControlMessage(MediaStream* aDestinationStream,
4047 : const nsTArray<MediaStream*>& aStreams,
4048 : AudioContextOperation aOperation,
4049 : void* aPromise)
4050 0 : : ControlMessage(aDestinationStream)
4051 : , mStreams(aStreams)
4052 : , mAudioContextOperation(aOperation)
4053 0 : , mPromise(aPromise)
4054 : {
4055 0 : }
4056 0 : void Run() override
4057 : {
4058 0 : mStream->GraphImpl()->ApplyAudioContextOperationImpl(mStream,
4059 0 : mStreams, mAudioContextOperation, mPromise);
4060 0 : }
4061 0 : void RunDuringShutdown() override
4062 : {
4063 0 : MOZ_ASSERT(mAudioContextOperation == AudioContextOperation::Close,
4064 : "We should be reviving the graph?");
4065 0 : }
4066 :
4067 : private:
4068 : // We don't need strong references here for the same reason ControlMessage
4069 : // doesn't.
4070 : nsTArray<MediaStream*> mStreams;
4071 : AudioContextOperation mAudioContextOperation;
4072 : void* mPromise;
4073 : };
4074 :
4075 0 : MediaStreamGraphImpl* graphImpl = static_cast<MediaStreamGraphImpl*>(this);
4076 0 : graphImpl->AppendMessage(
4077 0 : MakeUnique<AudioContextOperationControlMessage>(aDestinationStream, aStreams,
4078 0 : aOperation, aPromise));
4079 0 : }
4080 :
4081 : bool
4082 0 : MediaStreamGraph::IsNonRealtime() const
4083 : {
4084 0 : return !static_cast<const MediaStreamGraphImpl*>(this)->mRealtime;
4085 : }
4086 :
4087 : void
4088 0 : MediaStreamGraph::StartNonRealtimeProcessing(uint32_t aTicksToProcess)
4089 : {
4090 0 : NS_ASSERTION(NS_IsMainThread(), "main thread only");
4091 :
4092 0 : MediaStreamGraphImpl* graph = static_cast<MediaStreamGraphImpl*>(this);
4093 0 : NS_ASSERTION(!graph->mRealtime, "non-realtime only");
4094 :
4095 0 : if (graph->mNonRealtimeProcessing)
4096 0 : return;
4097 :
4098 0 : graph->mEndTime =
4099 0 : graph->RoundUpToNextAudioBlock(graph->mStateComputedTime +
4100 : aTicksToProcess - 1);
4101 0 : graph->mNonRealtimeProcessing = true;
4102 0 : graph->EnsureRunInStableState();
4103 : }
4104 :
4105 : void
4106 0 : ProcessedMediaStream::AddInput(MediaInputPort* aPort)
4107 : {
4108 0 : MediaStream* s = aPort->GetSource();
4109 0 : if (!s->IsSuspended()) {
4110 0 : mInputs.AppendElement(aPort);
4111 : } else {
4112 0 : mSuspendedInputs.AppendElement(aPort);
4113 : }
4114 0 : GraphImpl()->SetStreamOrderDirty();
4115 0 : }
4116 :
4117 : void
4118 0 : ProcessedMediaStream::InputSuspended(MediaInputPort* aPort)
4119 : {
4120 0 : GraphImpl()->AssertOnGraphThreadOrNotRunning();
4121 0 : mInputs.RemoveElement(aPort);
4122 0 : mSuspendedInputs.AppendElement(aPort);
4123 0 : GraphImpl()->SetStreamOrderDirty();
4124 0 : }
4125 :
4126 : void
4127 0 : ProcessedMediaStream::InputResumed(MediaInputPort* aPort)
4128 : {
4129 0 : GraphImpl()->AssertOnGraphThreadOrNotRunning();
4130 0 : mSuspendedInputs.RemoveElement(aPort);
4131 0 : mInputs.AppendElement(aPort);
4132 0 : GraphImpl()->SetStreamOrderDirty();
4133 0 : }
4134 :
4135 : void
4136 0 : MediaStreamGraph::RegisterCaptureStreamForWindow(
4137 : uint64_t aWindowId, ProcessedMediaStream* aCaptureStream)
4138 : {
4139 0 : MOZ_ASSERT(NS_IsMainThread());
4140 0 : MediaStreamGraphImpl* graphImpl = static_cast<MediaStreamGraphImpl*>(this);
4141 0 : graphImpl->RegisterCaptureStreamForWindow(aWindowId, aCaptureStream);
4142 0 : }
4143 :
4144 : void
4145 0 : MediaStreamGraphImpl::RegisterCaptureStreamForWindow(
4146 : uint64_t aWindowId, ProcessedMediaStream* aCaptureStream)
4147 : {
4148 0 : MOZ_ASSERT(NS_IsMainThread());
4149 0 : WindowAndStream winAndStream;
4150 0 : winAndStream.mWindowId = aWindowId;
4151 0 : winAndStream.mCaptureStreamSink = aCaptureStream;
4152 0 : mWindowCaptureStreams.AppendElement(winAndStream);
4153 0 : }
4154 :
4155 : void
4156 0 : MediaStreamGraph::UnregisterCaptureStreamForWindow(uint64_t aWindowId)
4157 : {
4158 0 : MOZ_ASSERT(NS_IsMainThread());
4159 0 : MediaStreamGraphImpl* graphImpl = static_cast<MediaStreamGraphImpl*>(this);
4160 0 : graphImpl->UnregisterCaptureStreamForWindow(aWindowId);
4161 0 : }
4162 :
4163 : void
4164 0 : MediaStreamGraphImpl::UnregisterCaptureStreamForWindow(uint64_t aWindowId)
4165 : {
4166 0 : MOZ_ASSERT(NS_IsMainThread());
4167 0 : for (int32_t i = mWindowCaptureStreams.Length() - 1; i >= 0; i--) {
4168 0 : if (mWindowCaptureStreams[i].mWindowId == aWindowId) {
4169 0 : mWindowCaptureStreams.RemoveElementAt(i);
4170 : }
4171 : }
4172 0 : }
4173 :
4174 : already_AddRefed<MediaInputPort>
4175 0 : MediaStreamGraph::ConnectToCaptureStream(uint64_t aWindowId,
4176 : MediaStream* aMediaStream)
4177 : {
4178 : return aMediaStream->GraphImpl()->ConnectToCaptureStream(aWindowId,
4179 0 : aMediaStream);
4180 : }
4181 :
4182 : already_AddRefed<MediaInputPort>
4183 0 : MediaStreamGraphImpl::ConnectToCaptureStream(uint64_t aWindowId,
4184 : MediaStream* aMediaStream)
4185 : {
4186 0 : MOZ_ASSERT(NS_IsMainThread());
4187 0 : for (uint32_t i = 0; i < mWindowCaptureStreams.Length(); i++) {
4188 0 : if (mWindowCaptureStreams[i].mWindowId == aWindowId) {
4189 0 : ProcessedMediaStream* sink = mWindowCaptureStreams[i].mCaptureStreamSink;
4190 0 : return sink->AllocateInputPort(aMediaStream);
4191 : }
4192 : }
4193 0 : return nullptr;
4194 : }
4195 :
4196 : void
4197 0 : MediaStreamGraph::DispatchToMainThreadAfterStreamStateUpdate(
4198 : already_AddRefed<nsIRunnable> aRunnable)
4199 : {
4200 0 : AssertOnGraphThreadOrNotRunning();
4201 : *mPendingUpdateRunnables.AppendElement() =
4202 0 : AbstractMainThread()->CreateDirectTaskDrainer(Move(aRunnable));
4203 0 : }
4204 :
4205 : } // namespace mozilla
|
