Line data Source code
1 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 : /* This Source Code Form is subject to the terms of the Mozilla Public
3 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
5 :
6 : #include "MediaStreamGraphImpl.h"
7 : #include "MediaStreamListener.h"
8 : #include "mozilla/MathAlgorithms.h"
9 : #include "mozilla/Unused.h"
10 :
11 : #include "AudioSegment.h"
12 : #include "VideoSegment.h"
13 : #include "nsContentUtils.h"
14 : #include "nsIAppShell.h"
15 : #include "nsIObserver.h"
16 : #include "nsPrintfCString.h"
17 : #include "nsServiceManagerUtils.h"
18 : #include "nsWidgetsCID.h"
19 : #include "prerror.h"
20 : #include "mozilla/Logging.h"
21 : #include "mozilla/Attributes.h"
22 : #include "TrackUnionStream.h"
23 : #include "ImageContainer.h"
24 : #include "AudioChannelService.h"
25 : #include "AudioNodeEngine.h"
26 : #include "AudioNodeStream.h"
27 : #include "AudioNodeExternalInputStream.h"
28 : #include "webaudio/MediaStreamAudioDestinationNode.h"
29 : #include <algorithm>
30 : #include "DOMMediaStream.h"
31 : #include "GeckoProfiler.h"
32 : #ifdef MOZ_WEBRTC
33 : #include "AudioOutputObserver.h"
34 : #endif
35 :
36 : using namespace mozilla::layers;
37 : using namespace mozilla::dom;
38 : using namespace mozilla::gfx;
39 :
40 : namespace mozilla {
41 :
42 : #ifdef STREAM_LOG
43 : #undef STREAM_LOG
44 : #endif
45 :
46 : LazyLogModule gTrackUnionStreamLog("TrackUnionStream");
47 : #define STREAM_LOG(type, msg) MOZ_LOG(gTrackUnionStreamLog, type, msg)
48 :
49 0 : TrackUnionStream::TrackUnionStream()
50 : : ProcessedMediaStream()
51 0 : , mNextAvailableTrackID(1)
52 : {
53 0 : }
54 :
55 0 : void TrackUnionStream::RemoveInput(MediaInputPort* aPort)
56 : {
57 0 : STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p removing input %p", this, aPort));
58 0 : for (int32_t i = mTrackMap.Length() - 1; i >= 0; --i) {
59 0 : if (mTrackMap[i].mInputPort == aPort) {
60 0 : STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p removing trackmap entry %d", this, i));
61 : nsTArray<RefPtr<DirectMediaStreamTrackListener>> listeners(
62 0 : mTrackMap[i].mOwnedDirectListeners);
63 0 : for (auto listener : listeners) {
64 : // Remove listeners while the entry still exists.
65 0 : RemoveDirectTrackListenerImpl(listener, mTrackMap[i].mOutputTrackID);
66 : }
67 0 : EndTrack(i);
68 0 : mTrackMap.RemoveElementAt(i);
69 : }
70 : }
71 0 : ProcessedMediaStream::RemoveInput(aPort);
72 0 : }
73 0 : void TrackUnionStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags)
74 : {
75 0 : if (IsFinishedOnGraphThread()) {
76 0 : return;
77 : }
78 0 : AutoTArray<bool,8> mappedTracksFinished;
79 0 : AutoTArray<bool,8> mappedTracksWithMatchingInputTracks;
80 0 : for (uint32_t i = 0; i < mTrackMap.Length(); ++i) {
81 0 : mappedTracksFinished.AppendElement(true);
82 0 : mappedTracksWithMatchingInputTracks.AppendElement(false);
83 : }
84 :
85 0 : AutoTArray<MediaInputPort*, 32> inputs(mInputs);
86 0 : inputs.AppendElements(mSuspendedInputs);
87 :
88 0 : bool allFinished = !inputs.IsEmpty();
89 0 : bool allHaveCurrentData = !inputs.IsEmpty();
90 0 : for (uint32_t i = 0; i < inputs.Length(); ++i) {
91 0 : MediaStream* stream = inputs[i]->GetSource();
92 0 : if (!stream->IsFinishedOnGraphThread()) {
93 : // XXX we really should check whether 'stream' has finished within time aTo,
94 : // not just that it's finishing when all its queued data eventually runs
95 : // out.
96 0 : allFinished = false;
97 : }
98 0 : if (!stream->HasCurrentData()) {
99 0 : allHaveCurrentData = false;
100 : }
101 0 : bool trackAdded = false;
102 0 : for (StreamTracks::TrackIter tracks(stream->GetStreamTracks());
103 0 : !tracks.IsEnded(); tracks.Next()) {
104 0 : bool found = false;
105 0 : for (uint32_t j = 0; j < mTrackMap.Length(); ++j) {
106 0 : TrackMapEntry* map = &mTrackMap[j];
107 0 : if (map->mInputPort == inputs[i] && map->mInputTrackID == tracks->GetID()) {
108 0 : bool trackFinished = false;
109 0 : StreamTracks::Track* outputTrack = mTracks.FindTrack(map->mOutputTrackID);
110 0 : found = true;
111 0 : if (!outputTrack || outputTrack->IsEnded() ||
112 0 : !inputs[i]->PassTrackThrough(tracks->GetID())) {
113 0 : trackFinished = true;
114 : } else {
115 0 : CopyTrackData(tracks.get(), j, aFrom, aTo, &trackFinished);
116 : }
117 0 : mappedTracksFinished[j] = trackFinished;
118 0 : mappedTracksWithMatchingInputTracks[j] = true;
119 0 : break;
120 : }
121 : }
122 0 : if (!found && inputs[i]->AllowCreationOf(tracks->GetID())) {
123 0 : bool trackFinished = false;
124 0 : trackAdded = true;
125 0 : uint32_t mapIndex = AddTrack(inputs[i], tracks.get(), aFrom);
126 0 : CopyTrackData(tracks.get(), mapIndex, aFrom, aTo, &trackFinished);
127 0 : mappedTracksFinished.AppendElement(trackFinished);
128 0 : mappedTracksWithMatchingInputTracks.AppendElement(true);
129 : }
130 : }
131 0 : if (trackAdded) {
132 0 : for (MediaStreamListener* l : mListeners) {
133 0 : l->NotifyFinishedTrackCreation(Graph());
134 : }
135 : }
136 : }
137 0 : for (int32_t i = mTrackMap.Length() - 1; i >= 0; --i) {
138 0 : if (mappedTracksFinished[i]) {
139 0 : EndTrack(i);
140 : } else {
141 0 : allFinished = false;
142 : }
143 0 : if (!mappedTracksWithMatchingInputTracks[i]) {
144 0 : for (auto listener : mTrackMap[i].mOwnedDirectListeners) {
145 : // Remove listeners while the entry still exists.
146 0 : RemoveDirectTrackListenerImpl(listener, mTrackMap[i].mOutputTrackID);
147 : }
148 0 : mTrackMap.RemoveElementAt(i);
149 : }
150 : }
151 0 : if (allFinished && mAutofinish && (aFlags & ALLOW_FINISH)) {
152 : // All streams have finished and won't add any more tracks, and
153 : // all our tracks have actually finished and been removed from our map,
154 : // so we're finished now.
155 0 : FinishOnGraphThread();
156 : } else {
157 0 : mTracks.AdvanceKnownTracksTime(GraphTimeToStreamTimeWithBlocking(aTo));
158 : }
159 0 : if (allHaveCurrentData) {
160 : // We can make progress if we're not blocked
161 0 : mHasCurrentData = true;
162 : }
163 : }
164 :
165 0 : uint32_t TrackUnionStream::AddTrack(MediaInputPort* aPort, StreamTracks::Track* aTrack,
166 : GraphTime aFrom)
167 : {
168 0 : STREAM_LOG(LogLevel::Verbose, ("TrackUnionStream %p adding track %d for "
169 : "input stream %p track %d, desired id %d",
170 : this, aTrack->GetID(), aPort->GetSource(),
171 : aTrack->GetID(),
172 : aPort->GetDestinationTrackId()));
173 :
174 : TrackID id;
175 0 : if (IsTrackIDExplicit(id = aPort->GetDestinationTrackId())) {
176 0 : MOZ_ASSERT(id >= mNextAvailableTrackID &&
177 : mUsedTracks.BinaryIndexOf(id) == mUsedTracks.NoIndex,
178 : "Desired destination id taken. Only provide a destination ID "
179 : "if you can assure its availability, or we may not be able "
180 : "to bind to the correct DOM-side track.");
181 : #ifdef DEBUG
182 0 : AutoTArray<MediaInputPort*, 32> inputs(mInputs);
183 0 : inputs.AppendElements(mSuspendedInputs);
184 0 : for (size_t i = 0; inputs[i] != aPort; ++i) {
185 0 : MOZ_ASSERT(inputs[i]->GetSourceTrackId() != TRACK_ANY,
186 : "You are adding a MediaInputPort with a track mapping "
187 : "while there already exist generic MediaInputPorts for this "
188 : "destination stream. This can lead to TrackID collisions!");
189 : }
190 : #endif
191 0 : mUsedTracks.InsertElementSorted(id);
192 0 : } else if ((id = aTrack->GetID()) &&
193 0 : id > mNextAvailableTrackID &&
194 0 : mUsedTracks.BinaryIndexOf(id) == mUsedTracks.NoIndex) {
195 : // Input id available. Mark it used in mUsedTracks.
196 0 : mUsedTracks.InsertElementSorted(id);
197 : } else {
198 : // No desired destination id and Input id taken, allocate a new one.
199 0 : id = mNextAvailableTrackID;
200 :
201 : // Update mNextAvailableTrackID and prune any mUsedTracks members it now
202 : // covers.
203 : while (1) {
204 0 : if (!mUsedTracks.RemoveElementSorted(++mNextAvailableTrackID)) {
205 : // Not in use. We're done.
206 0 : break;
207 : }
208 : }
209 : }
210 :
211 : // Round up the track start time so the track, if anything, starts a
212 : // little later than the true time. This means we'll have enough
213 : // samples in our input stream to go just beyond the destination time.
214 0 : StreamTime outputStart = GraphTimeToStreamTimeWithBlocking(aFrom);
215 :
216 0 : nsAutoPtr<MediaSegment> segment;
217 0 : segment = aTrack->GetSegment()->CreateEmptyClone();
218 0 : for (uint32_t j = 0; j < mListeners.Length(); ++j) {
219 0 : MediaStreamListener* l = mListeners[j];
220 0 : l->NotifyQueuedTrackChanges(Graph(), id, outputStart,
221 : TrackEventCommand::TRACK_EVENT_CREATED,
222 0 : *segment,
223 0 : aPort->GetSource(), aTrack->GetID());
224 : }
225 0 : segment->AppendNullData(outputStart);
226 : StreamTracks::Track* track =
227 0 : &mTracks.AddTrack(id, outputStart, segment.forget());
228 0 : STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p added track %d for input stream %p track %d, start ticks %lld",
229 : this, track->GetID(), aPort->GetSource(), aTrack->GetID(),
230 : (long long)outputStart));
231 :
232 0 : TrackMapEntry* map = mTrackMap.AppendElement();
233 0 : map->mEndOfConsumedInputTicks = 0;
234 0 : map->mEndOfLastInputIntervalInInputStream = -1;
235 0 : map->mEndOfLastInputIntervalInOutputStream = -1;
236 0 : map->mInputPort = aPort;
237 0 : map->mInputTrackID = aTrack->GetID();
238 0 : map->mOutputTrackID = track->GetID();
239 0 : map->mSegment = aTrack->GetSegment()->CreateEmptyClone();
240 :
241 0 : for (int32_t i = mPendingDirectTrackListeners.Length() - 1; i >= 0; --i) {
242 : TrackBound<DirectMediaStreamTrackListener>& bound =
243 0 : mPendingDirectTrackListeners[i];
244 0 : if (bound.mTrackID != map->mOutputTrackID) {
245 0 : continue;
246 : }
247 0 : MediaStream* source = map->mInputPort->GetSource();
248 0 : map->mOwnedDirectListeners.AppendElement(bound.mListener);
249 0 : DisabledTrackMode currentMode = GetDisabledTrackMode(bound.mTrackID);
250 0 : if (currentMode != DisabledTrackMode::ENABLED) {
251 0 : bound.mListener->IncreaseDisabled(currentMode);
252 : }
253 0 : STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p adding direct listener "
254 : "%p for track %d. Forwarding to input "
255 : "stream %p track %d.",
256 : this, bound.mListener.get(), bound.mTrackID,
257 : source, map->mInputTrackID));
258 0 : source->AddDirectTrackListenerImpl(bound.mListener.forget(),
259 0 : map->mInputTrackID);
260 0 : mPendingDirectTrackListeners.RemoveElementAt(i);
261 : }
262 :
263 0 : return mTrackMap.Length() - 1;
264 : }
265 :
266 0 : void TrackUnionStream::EndTrack(uint32_t aIndex)
267 : {
268 0 : StreamTracks::Track* outputTrack = mTracks.FindTrack(mTrackMap[aIndex].mOutputTrackID);
269 0 : if (!outputTrack || outputTrack->IsEnded())
270 0 : return;
271 0 : STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p ending track %d", this, outputTrack->GetID()));
272 0 : for (uint32_t j = 0; j < mListeners.Length(); ++j) {
273 0 : MediaStreamListener* l = mListeners[j];
274 0 : StreamTime offset = outputTrack->GetSegment()->GetDuration();
275 0 : nsAutoPtr<MediaSegment> segment;
276 0 : segment = outputTrack->GetSegment()->CreateEmptyClone();
277 0 : l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(), offset,
278 : TrackEventCommand::TRACK_EVENT_ENDED,
279 0 : *segment,
280 0 : mTrackMap[aIndex].mInputPort->GetSource(),
281 0 : mTrackMap[aIndex].mInputTrackID);
282 : }
283 0 : for (TrackBound<MediaStreamTrackListener>& b : mTrackListeners) {
284 0 : if (b.mTrackID == outputTrack->GetID()) {
285 0 : b.mListener->NotifyEnded();
286 : }
287 : }
288 0 : outputTrack->SetEnded();
289 : }
290 :
291 0 : void TrackUnionStream::CopyTrackData(StreamTracks::Track* aInputTrack,
292 : uint32_t aMapIndex, GraphTime aFrom, GraphTime aTo,
293 : bool* aOutputTrackFinished)
294 : {
295 0 : TrackMapEntry* map = &mTrackMap[aMapIndex];
296 0 : StreamTracks::Track* outputTrack = mTracks.FindTrack(map->mOutputTrackID);
297 0 : MOZ_ASSERT(outputTrack && !outputTrack->IsEnded(), "Can't copy to ended track");
298 :
299 0 : MediaSegment* segment = map->mSegment;
300 0 : MediaStream* source = map->mInputPort->GetSource();
301 :
302 : GraphTime next;
303 0 : *aOutputTrackFinished = false;
304 0 : for (GraphTime t = aFrom; t < aTo; t = next) {
305 0 : MediaInputPort::InputInterval interval = map->mInputPort->GetNextInputInterval(t);
306 0 : interval.mEnd = std::min(interval.mEnd, aTo);
307 0 : StreamTime inputEnd = source->GraphTimeToStreamTimeWithBlocking(interval.mEnd);
308 0 : StreamTime inputTrackEndPoint = STREAM_TIME_MAX;
309 :
310 0 : if (aInputTrack->IsEnded() &&
311 0 : aInputTrack->GetEnd() <= inputEnd) {
312 0 : inputTrackEndPoint = aInputTrack->GetEnd();
313 0 : *aOutputTrackFinished = true;
314 : }
315 :
316 0 : if (interval.mStart >= interval.mEnd) {
317 0 : break;
318 : }
319 0 : StreamTime ticks = interval.mEnd - interval.mStart;
320 0 : next = interval.mEnd;
321 :
322 0 : StreamTime outputStart = outputTrack->GetEnd();
323 :
324 0 : if (interval.mInputIsBlocked) {
325 : // Maybe the input track ended?
326 0 : segment->AppendNullData(ticks);
327 0 : STREAM_LOG(LogLevel::Verbose, ("TrackUnionStream %p appending %lld ticks of null data to track %d",
328 : this, (long long)ticks, outputTrack->GetID()));
329 0 : } else if (InMutedCycle()) {
330 0 : segment->AppendNullData(ticks);
331 : } else {
332 0 : if (source->IsSuspended()) {
333 0 : segment->AppendNullData(aTo - aFrom);
334 : } else {
335 0 : MOZ_ASSERT(outputTrack->GetEnd() == GraphTimeToStreamTimeWithBlocking(interval.mStart),
336 : "Samples missing");
337 0 : StreamTime inputStart = source->GraphTimeToStreamTimeWithBlocking(interval.mStart);
338 0 : segment->AppendSlice(*aInputTrack->GetSegment(),
339 0 : std::min(inputTrackEndPoint, inputStart),
340 0 : std::min(inputTrackEndPoint, inputEnd));
341 : }
342 : }
343 0 : ApplyTrackDisabling(outputTrack->GetID(), segment);
344 0 : for (uint32_t j = 0; j < mListeners.Length(); ++j) {
345 0 : MediaStreamListener* l = mListeners[j];
346 : // Separate Audio and Video.
347 0 : if (segment->GetType() == MediaSegment::AUDIO) {
348 0 : l->NotifyQueuedAudioData(Graph(), outputTrack->GetID(),
349 : outputStart,
350 : *static_cast<AudioSegment*>(segment),
351 0 : map->mInputPort->GetSource(),
352 0 : map->mInputTrackID);
353 : }
354 : }
355 0 : for (TrackBound<MediaStreamTrackListener>& b : mTrackListeners) {
356 0 : if (b.mTrackID != outputTrack->GetID()) {
357 0 : continue;
358 : }
359 0 : b.mListener->NotifyQueuedChanges(Graph(), outputStart, *segment);
360 : }
361 0 : outputTrack->GetSegment()->AppendFrom(segment);
362 : }
363 0 : }
364 :
365 : void
366 0 : TrackUnionStream::SetTrackEnabledImpl(TrackID aTrackID, DisabledTrackMode aMode) {
367 0 : bool enabled = aMode == DisabledTrackMode::ENABLED;
368 0 : for (TrackMapEntry& entry : mTrackMap) {
369 0 : if (entry.mOutputTrackID == aTrackID) {
370 0 : STREAM_LOG(LogLevel::Info, ("TrackUnionStream %p track %d was explicitly %s",
371 : this, aTrackID, enabled ? "enabled" : "disabled"));
372 0 : for (DirectMediaStreamTrackListener* listener : entry.mOwnedDirectListeners) {
373 0 : DisabledTrackMode oldMode = GetDisabledTrackMode(aTrackID);
374 0 : bool oldEnabled = oldMode == DisabledTrackMode::ENABLED;
375 0 : if (!oldEnabled && enabled) {
376 0 : STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p track %d setting "
377 : "direct listener enabled",
378 : this, aTrackID));
379 0 : listener->DecreaseDisabled(oldMode);
380 0 : } else if (oldEnabled && !enabled) {
381 0 : STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p track %d setting "
382 : "direct listener disabled",
383 : this, aTrackID));
384 0 : listener->IncreaseDisabled(aMode);
385 : }
386 : }
387 : }
388 : }
389 0 : MediaStream::SetTrackEnabledImpl(aTrackID, aMode);
390 0 : }
391 :
392 : MediaStream*
393 0 : TrackUnionStream::GetInputStreamFor(TrackID aTrackID)
394 : {
395 0 : for (TrackMapEntry& entry : mTrackMap) {
396 0 : if (entry.mOutputTrackID == aTrackID && entry.mInputPort) {
397 0 : return entry.mInputPort->GetSource();
398 : }
399 : }
400 :
401 0 : return nullptr;
402 : }
403 :
404 : TrackID
405 0 : TrackUnionStream::GetInputTrackIDFor(TrackID aTrackID)
406 : {
407 0 : for (TrackMapEntry& entry : mTrackMap) {
408 0 : if (entry.mOutputTrackID == aTrackID) {
409 0 : return entry.mInputTrackID;
410 : }
411 : }
412 :
413 0 : return TRACK_NONE;
414 : }
415 :
416 : void
417 0 : TrackUnionStream::AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
418 : TrackID aTrackID)
419 : {
420 0 : RefPtr<DirectMediaStreamTrackListener> listener = aListener;
421 :
422 0 : for (TrackMapEntry& entry : mTrackMap) {
423 0 : if (entry.mOutputTrackID == aTrackID) {
424 0 : MediaStream* source = entry.mInputPort->GetSource();
425 0 : STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p adding direct listener "
426 : "%p for track %d. Forwarding to input "
427 : "stream %p track %d.",
428 : this, listener.get(), aTrackID, source,
429 : entry.mInputTrackID));
430 0 : entry.mOwnedDirectListeners.AppendElement(listener);
431 0 : DisabledTrackMode currentMode = GetDisabledTrackMode(aTrackID);
432 0 : if (currentMode != DisabledTrackMode::ENABLED) {
433 0 : listener->IncreaseDisabled(currentMode);
434 : }
435 0 : source->AddDirectTrackListenerImpl(listener.forget(),
436 0 : entry.mInputTrackID);
437 0 : return;
438 : }
439 : }
440 :
441 : TrackBound<DirectMediaStreamTrackListener>* bound =
442 0 : mPendingDirectTrackListeners.AppendElement();
443 0 : bound->mListener = listener.forget();
444 0 : bound->mTrackID = aTrackID;
445 : }
446 :
447 : void
448 0 : TrackUnionStream::RemoveDirectTrackListenerImpl(DirectMediaStreamTrackListener* aListener,
449 : TrackID aTrackID)
450 : {
451 0 : for (TrackMapEntry& entry : mTrackMap) {
452 : // OutputTrackID is unique to this stream so we only need to do this once.
453 0 : if (entry.mOutputTrackID != aTrackID) {
454 0 : continue;
455 : }
456 0 : for (size_t i = 0; i < entry.mOwnedDirectListeners.Length(); ++i) {
457 0 : if (entry.mOwnedDirectListeners[i] == aListener) {
458 0 : STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p removing direct "
459 : "listener %p for track %d, forwarding "
460 : "to input stream %p track %d",
461 : this, aListener, aTrackID,
462 : entry.mInputPort->GetSource(),
463 : entry.mInputTrackID));
464 0 : DisabledTrackMode currentMode = GetDisabledTrackMode(aTrackID);
465 0 : if (currentMode != DisabledTrackMode::ENABLED) {
466 : // Reset the listener's state.
467 0 : aListener->DecreaseDisabled(currentMode);
468 : }
469 0 : entry.mOwnedDirectListeners.RemoveElementAt(i);
470 0 : break;
471 : }
472 : }
473 : // Forward to the input
474 0 : MediaStream* source = entry.mInputPort->GetSource();
475 0 : source->RemoveDirectTrackListenerImpl(aListener, entry.mInputTrackID);
476 0 : return;
477 : }
478 :
479 0 : for (size_t i = 0; i < mPendingDirectTrackListeners.Length(); ++i) {
480 : TrackBound<DirectMediaStreamTrackListener>& bound =
481 0 : mPendingDirectTrackListeners[i];
482 0 : if (bound.mListener == aListener && bound.mTrackID == aTrackID) {
483 0 : mPendingDirectTrackListeners.RemoveElementAt(i);
484 0 : return;
485 : }
486 : }
487 : }
488 : } // namespace mozilla
|
