Line data Source code
1 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 : /* This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this
5 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #include "MediaSourceDemuxer.h"
8 :
9 : #include "MediaSourceUtils.h"
10 : #include "OpusDecoder.h"
11 : #include "SourceBufferList.h"
12 : #include "nsPrintfCString.h"
13 :
14 : #include <algorithm>
15 : #include <limits>
16 : #include <stdint.h>
17 :
18 : namespace mozilla {
19 :
20 : typedef TrackInfo::TrackType TrackType;
21 : using media::TimeUnit;
22 : using media::TimeIntervals;
23 :
24 0 : MediaSourceDemuxer::MediaSourceDemuxer(AbstractThread* aAbstractMainThread)
25 0 : : mTaskQueue(new AutoTaskQueue(GetMediaThreadPool(MediaThreadType::PLAYBACK),
26 0 : "MediaSourceDemuxer::mTaskQueue"))
27 0 : , mMonitor("MediaSourceDemuxer")
28 : {
29 0 : MOZ_ASSERT(NS_IsMainThread());
30 0 : }
31 :
32 : constexpr TimeUnit MediaSourceDemuxer::EOS_FUZZ;
33 :
34 : RefPtr<MediaSourceDemuxer::InitPromise>
35 0 : MediaSourceDemuxer::Init()
36 : {
37 0 : RefPtr<MediaSourceDemuxer> self = this;
38 0 : return InvokeAsync(GetTaskQueue(), __func__,
39 0 : [self](){
40 0 : if (self->ScanSourceBuffersForContent()) {
41 0 : return InitPromise::CreateAndResolve(NS_OK, __func__);
42 : }
43 :
44 0 : RefPtr<InitPromise> p = self->mInitPromise.Ensure(__func__);
45 :
46 0 : return p;
47 0 : });
48 : }
49 :
50 : void
51 0 : MediaSourceDemuxer::AddSizeOfResources(
52 : MediaSourceDecoder::ResourceSizes* aSizes)
53 : {
54 0 : MOZ_ASSERT(NS_IsMainThread());
55 :
56 : // NB: The track buffers must only be accessed on the TaskQueue.
57 0 : RefPtr<MediaSourceDemuxer> self = this;
58 0 : RefPtr<MediaSourceDecoder::ResourceSizes> sizes = aSizes;
59 0 : nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
60 0 : "MediaSourceDemuxer::AddSizeOfResources", [self, sizes]() {
61 0 : for (TrackBuffersManager* manager : self->mSourceBuffers) {
62 0 : manager->AddSizeOfResources(sizes);
63 : }
64 0 : });
65 :
66 0 : GetTaskQueue()->Dispatch(task.forget());
67 0 : }
68 :
69 0 : void MediaSourceDemuxer::NotifyInitDataArrived()
70 : {
71 0 : RefPtr<MediaSourceDemuxer> self = this;
72 0 : nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
73 0 : "MediaSourceDemuxer::NotifyInitDataArrived", [self]() {
74 0 : if (self->mInitPromise.IsEmpty()) {
75 0 : return;
76 : }
77 0 : if (self->ScanSourceBuffersForContent()) {
78 0 : self->mInitPromise.ResolveIfExists(NS_OK, __func__);
79 : }
80 0 : });
81 0 : GetTaskQueue()->Dispatch(task.forget());
82 0 : }
83 :
84 : bool
85 0 : MediaSourceDemuxer::ScanSourceBuffersForContent()
86 : {
87 0 : MOZ_ASSERT(OnTaskQueue());
88 :
89 0 : if (mSourceBuffers.IsEmpty()) {
90 0 : return false;
91 : }
92 :
93 0 : MonitorAutoLock mon(mMonitor);
94 :
95 0 : bool haveEmptySourceBuffer = false;
96 0 : for (const auto& sourceBuffer : mSourceBuffers) {
97 0 : MediaInfo info = sourceBuffer->GetMetadata();
98 0 : if (!info.HasAudio() && !info.HasVideo()) {
99 0 : haveEmptySourceBuffer = true;
100 : }
101 0 : if (info.HasAudio() && !mAudioTrack) {
102 0 : mInfo.mAudio = info.mAudio;
103 0 : mAudioTrack = sourceBuffer;
104 : }
105 0 : if (info.HasVideo() && !mVideoTrack) {
106 0 : mInfo.mVideo = info.mVideo;
107 0 : mVideoTrack = sourceBuffer;
108 : }
109 0 : if (info.IsEncrypted() && !mInfo.IsEncrypted()) {
110 0 : mInfo.mCrypto = info.mCrypto;
111 : }
112 : }
113 0 : if (mInfo.HasAudio() && mInfo.HasVideo()) {
114 : // We have both audio and video. We can ignore non-ready source buffer.
115 0 : return true;
116 : }
117 0 : return !haveEmptySourceBuffer;
118 : }
119 :
120 : bool
121 0 : MediaSourceDemuxer::HasTrackType(TrackType aType) const
122 : {
123 0 : MonitorAutoLock mon(mMonitor);
124 :
125 0 : switch (aType) {
126 : case TrackType::kAudioTrack:
127 0 : return mInfo.HasAudio();
128 : case TrackType::kVideoTrack:
129 0 : return mInfo.HasVideo();
130 : default:
131 0 : return false;
132 : }
133 : }
134 :
135 : uint32_t
136 0 : MediaSourceDemuxer::GetNumberTracks(TrackType aType) const
137 : {
138 0 : return HasTrackType(aType) ? 1u : 0;
139 : }
140 :
141 : already_AddRefed<MediaTrackDemuxer>
142 0 : MediaSourceDemuxer::GetTrackDemuxer(TrackType aType, uint32_t aTrackNumber)
143 : {
144 0 : RefPtr<TrackBuffersManager> manager = GetManager(aType);
145 0 : if (!manager) {
146 0 : return nullptr;
147 : }
148 : RefPtr<MediaSourceTrackDemuxer> e =
149 0 : new MediaSourceTrackDemuxer(this, aType, manager);
150 0 : mDemuxers.AppendElement(e);
151 0 : return e.forget();
152 : }
153 :
154 : bool
155 0 : MediaSourceDemuxer::IsSeekable() const
156 : {
157 0 : return true;
158 : }
159 :
160 : UniquePtr<EncryptionInfo>
161 0 : MediaSourceDemuxer::GetCrypto()
162 : {
163 0 : MonitorAutoLock mon(mMonitor);
164 0 : auto crypto = MakeUnique<EncryptionInfo>();
165 0 : *crypto = mInfo.mCrypto;
166 0 : return crypto;
167 : }
168 :
169 : void
170 0 : MediaSourceDemuxer::AttachSourceBuffer(TrackBuffersManager* aSourceBuffer)
171 : {
172 0 : nsCOMPtr<nsIRunnable> task = NewRunnableMethod<TrackBuffersManager*>(
173 : "MediaSourceDemuxer::DoAttachSourceBuffer",
174 : this,
175 : &MediaSourceDemuxer::DoAttachSourceBuffer,
176 0 : aSourceBuffer);
177 0 : GetTaskQueue()->Dispatch(task.forget());
178 0 : }
179 :
180 : void
181 0 : MediaSourceDemuxer::DoAttachSourceBuffer(mozilla::TrackBuffersManager* aSourceBuffer)
182 : {
183 0 : MOZ_ASSERT(OnTaskQueue());
184 0 : mSourceBuffers.AppendElement(aSourceBuffer);
185 0 : ScanSourceBuffersForContent();
186 0 : }
187 :
188 : void
189 0 : MediaSourceDemuxer::DetachSourceBuffer(TrackBuffersManager* aSourceBuffer)
190 : {
191 0 : nsCOMPtr<nsIRunnable> task = NewRunnableMethod<TrackBuffersManager*>(
192 : "MediaSourceDemuxer::DoDetachSourceBuffer",
193 : this,
194 : &MediaSourceDemuxer::DoDetachSourceBuffer,
195 0 : aSourceBuffer);
196 0 : GetTaskQueue()->Dispatch(task.forget());
197 0 : }
198 :
199 : void
200 0 : MediaSourceDemuxer::DoDetachSourceBuffer(TrackBuffersManager* aSourceBuffer)
201 : {
202 0 : MOZ_ASSERT(OnTaskQueue());
203 0 : for (uint32_t i = 0; i < mSourceBuffers.Length(); i++) {
204 0 : if (mSourceBuffers[i].get() == aSourceBuffer) {
205 0 : mSourceBuffers.RemoveElementAt(i);
206 : }
207 : }
208 : {
209 0 : MonitorAutoLock mon(mMonitor);
210 0 : if (aSourceBuffer == mAudioTrack) {
211 0 : mAudioTrack = nullptr;
212 : }
213 0 : if (aSourceBuffer == mVideoTrack) {
214 0 : mVideoTrack = nullptr;
215 : }
216 : }
217 0 : ScanSourceBuffersForContent();
218 0 : }
219 :
220 : TrackInfo*
221 0 : MediaSourceDemuxer::GetTrackInfo(TrackType aTrack)
222 : {
223 0 : MonitorAutoLock mon(mMonitor);
224 0 : switch (aTrack) {
225 : case TrackType::kAudioTrack:
226 0 : return &mInfo.mAudio;
227 : case TrackType::kVideoTrack:
228 0 : return &mInfo.mVideo;
229 : default:
230 0 : return nullptr;
231 : }
232 : }
233 :
234 : TrackBuffersManager*
235 0 : MediaSourceDemuxer::GetManager(TrackType aTrack)
236 : {
237 0 : MonitorAutoLock mon(mMonitor);
238 0 : switch (aTrack) {
239 : case TrackType::kAudioTrack:
240 0 : return mAudioTrack;
241 : case TrackType::kVideoTrack:
242 0 : return mVideoTrack;
243 : default:
244 0 : return nullptr;
245 : }
246 : }
247 :
248 0 : MediaSourceDemuxer::~MediaSourceDemuxer()
249 : {
250 0 : mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
251 0 : }
252 :
253 : void
254 0 : MediaSourceDemuxer::GetMozDebugReaderData(nsACString& aString)
255 : {
256 0 : MonitorAutoLock mon(mMonitor);
257 0 : nsAutoCString result;
258 0 : result += nsPrintfCString("Dumping Data for Demuxer: %p\n", this);
259 0 : if (mAudioTrack) {
260 0 : result += nsPrintfCString(
261 : "\tDumping Audio Track Buffer(%s): mLastAudioTime=%f\n"
262 : "\t\tAudio Track Buffer Details: NumSamples=%" PRIuSIZE
263 : " Size=%u Evictable=%u "
264 : "NextGetSampleIndex=%u NextInsertionIndex=%d\n",
265 0 : mAudioTrack->mAudioTracks.mInfo->mMimeType.get(),
266 0 : mAudioTrack->mAudioTracks.mNextSampleTime.ToSeconds(),
267 0 : mAudioTrack->mAudioTracks.mBuffers[0].Length(),
268 0 : mAudioTrack->mAudioTracks.mSizeBuffer,
269 : mAudioTrack->Evictable(TrackInfo::kAudioTrack),
270 0 : mAudioTrack->mAudioTracks.mNextGetSampleIndex.valueOr(-1),
271 0 : mAudioTrack->mAudioTracks.mNextInsertionIndex.valueOr(-1));
272 :
273 0 : result += nsPrintfCString(
274 : "\t\tAudio Track Buffered: ranges=%s\n",
275 0 : DumpTimeRanges(mAudioTrack->SafeBuffered(TrackInfo::kAudioTrack)).get());
276 : }
277 0 : if (mVideoTrack) {
278 0 : result += nsPrintfCString(
279 : "\tDumping Video Track Buffer(%s): mLastVideoTime=%f\n"
280 : "\t\tVideo Track Buffer Details: NumSamples=%" PRIuSIZE
281 : " Size=%u Evictable=%u "
282 : "NextGetSampleIndex=%u NextInsertionIndex=%d\n",
283 0 : mVideoTrack->mVideoTracks.mInfo->mMimeType.get(),
284 0 : mVideoTrack->mVideoTracks.mNextSampleTime.ToSeconds(),
285 0 : mVideoTrack->mVideoTracks.mBuffers[0].Length(),
286 0 : mVideoTrack->mVideoTracks.mSizeBuffer,
287 : mVideoTrack->Evictable(TrackInfo::kVideoTrack),
288 0 : mVideoTrack->mVideoTracks.mNextGetSampleIndex.valueOr(-1),
289 0 : mVideoTrack->mVideoTracks.mNextInsertionIndex.valueOr(-1));
290 :
291 0 : result += nsPrintfCString(
292 : "\t\tVideo Track Buffered: ranges=%s\n",
293 0 : DumpTimeRanges(mVideoTrack->SafeBuffered(TrackInfo::kVideoTrack)).get());
294 : }
295 0 : aString += result;
296 0 : }
297 :
298 0 : MediaSourceTrackDemuxer::MediaSourceTrackDemuxer(MediaSourceDemuxer* aParent,
299 : TrackInfo::TrackType aType,
300 0 : TrackBuffersManager* aManager)
301 : : mParent(aParent)
302 : , mManager(aManager)
303 : , mType(aType)
304 : , mMonitor("MediaSourceTrackDemuxer")
305 : , mReset(true)
306 : , mPreRoll(TimeUnit::FromMicroseconds(
307 0 : OpusDataDecoder::IsOpus(mParent->GetTrackInfo(mType)->mMimeType)
308 0 : ? 80000
309 0 : : mParent->GetTrackInfo(mType)->mMimeType.EqualsLiteral("audio/mp4a-latm")
310 : // AAC encoder delay is by default 2112 audio frames.
311 : // See https://developer.apple.com/library/content/documentation/QuickTime/QTFF/QTFFAppenG/QTFFAppenG.html
312 : // So we always seek 2112 frames
313 : ? (2112 * 1000000ULL
314 0 : / mParent->GetTrackInfo(mType)->GetAsAudioInfo()->mRate)
315 0 : : 0))
316 : {
317 0 : }
318 :
319 : UniquePtr<TrackInfo>
320 0 : MediaSourceTrackDemuxer::GetInfo() const
321 : {
322 0 : return mParent->GetTrackInfo(mType)->Clone();
323 : }
324 :
325 : RefPtr<MediaSourceTrackDemuxer::SeekPromise>
326 0 : MediaSourceTrackDemuxer::Seek(const TimeUnit& aTime)
327 : {
328 0 : MOZ_ASSERT(mParent, "Called after BreackCycle()");
329 : return InvokeAsync(
330 0 : mParent->GetTaskQueue(), this, __func__,
331 0 : &MediaSourceTrackDemuxer::DoSeek, aTime);
332 : }
333 :
334 : RefPtr<MediaSourceTrackDemuxer::SamplesPromise>
335 0 : MediaSourceTrackDemuxer::GetSamples(int32_t aNumSamples)
336 : {
337 0 : MOZ_ASSERT(mParent, "Called after BreackCycle()");
338 0 : return InvokeAsync(mParent->GetTaskQueue(), this, __func__,
339 0 : &MediaSourceTrackDemuxer::DoGetSamples, aNumSamples);
340 : }
341 :
342 : void
343 0 : MediaSourceTrackDemuxer::Reset()
344 : {
345 0 : MOZ_ASSERT(mParent, "Called after BreackCycle()");
346 0 : RefPtr<MediaSourceTrackDemuxer> self = this;
347 : nsCOMPtr<nsIRunnable> task =
348 0 : NS_NewRunnableFunction("MediaSourceTrackDemuxer::Reset", [self]() {
349 0 : self->mNextSample.reset();
350 0 : self->mReset = true;
351 0 : self->mManager->Seek(self->mType, TimeUnit::Zero(), TimeUnit::Zero());
352 : {
353 0 : MonitorAutoLock mon(self->mMonitor);
354 0 : self->mNextRandomAccessPoint = self->mManager->GetNextRandomAccessPoint(
355 0 : self->mType, MediaSourceDemuxer::EOS_FUZZ);
356 : }
357 0 : });
358 0 : mParent->GetTaskQueue()->Dispatch(task.forget());
359 0 : }
360 :
361 : nsresult
362 0 : MediaSourceTrackDemuxer::GetNextRandomAccessPoint(TimeUnit* aTime)
363 : {
364 0 : MonitorAutoLock mon(mMonitor);
365 0 : *aTime = mNextRandomAccessPoint;
366 0 : return NS_OK;
367 : }
368 :
369 : RefPtr<MediaSourceTrackDemuxer::SkipAccessPointPromise>
370 0 : MediaSourceTrackDemuxer::SkipToNextRandomAccessPoint(
371 : const TimeUnit& aTimeThreshold)
372 : {
373 : return InvokeAsync(
374 0 : mParent->GetTaskQueue(), this, __func__,
375 : &MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint,
376 0 : aTimeThreshold);
377 : }
378 :
379 : media::TimeIntervals
380 0 : MediaSourceTrackDemuxer::GetBuffered()
381 : {
382 0 : return mManager->Buffered();
383 : }
384 :
385 : void
386 0 : MediaSourceTrackDemuxer::BreakCycles()
387 : {
388 0 : RefPtr<MediaSourceTrackDemuxer> self = this;
389 : nsCOMPtr<nsIRunnable> task =
390 0 : NS_NewRunnableFunction("MediaSourceTrackDemuxer::BreakCycles", [self]() {
391 0 : self->mParent = nullptr;
392 0 : self->mManager = nullptr;
393 0 : });
394 0 : mParent->GetTaskQueue()->Dispatch(task.forget());
395 0 : }
396 :
397 : RefPtr<MediaSourceTrackDemuxer::SeekPromise>
398 0 : MediaSourceTrackDemuxer::DoSeek(const TimeUnit& aTime)
399 : {
400 0 : TimeIntervals buffered = mManager->Buffered(mType);
401 : // Fuzz factor represents a +/- threshold. So when seeking it allows the gap
402 : // to be twice as big as the fuzz value. We only want to allow EOS_FUZZ gap.
403 0 : buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
404 0 : TimeUnit seekTime = std::max(aTime - mPreRoll, TimeUnit::Zero());
405 :
406 0 : if (mManager->IsEnded() && seekTime >= buffered.GetEnd()) {
407 : // We're attempting to seek past the end time. Cap seekTime so that we seek
408 : // to the last sample instead.
409 0 : seekTime =
410 0 : std::max(mManager->HighestStartTime(mType) - mPreRoll, TimeUnit::Zero());
411 : }
412 0 : if (!buffered.ContainsWithStrictEnd(seekTime)) {
413 0 : if (!buffered.ContainsWithStrictEnd(aTime)) {
414 : // We don't have the data to seek to.
415 : return SeekPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA,
416 0 : __func__);
417 : }
418 : // Theoretically we should reject the promise with WAITING_FOR_DATA,
419 : // however, to avoid unwanted regressions we assume that if at this time
420 : // we don't have the wanted data it won't come later.
421 : // Instead of using the pre-rolled time, use the earliest time available in
422 : // the interval.
423 0 : TimeIntervals::IndexType index = buffered.Find(aTime);
424 0 : MOZ_ASSERT(index != TimeIntervals::NoIndex);
425 0 : seekTime = buffered[index].mStart;
426 : }
427 0 : seekTime = mManager->Seek(mType, seekTime, MediaSourceDemuxer::EOS_FUZZ);
428 0 : MediaResult result = NS_OK;
429 : RefPtr<MediaRawData> sample =
430 0 : mManager->GetSample(mType,
431 0 : TimeUnit::Zero(),
432 0 : result);
433 0 : MOZ_ASSERT(NS_SUCCEEDED(result) && sample);
434 0 : mNextSample = Some(sample);
435 0 : mReset = false;
436 : {
437 0 : MonitorAutoLock mon(mMonitor);
438 : mNextRandomAccessPoint =
439 0 : mManager->GetNextRandomAccessPoint(mType, MediaSourceDemuxer::EOS_FUZZ);
440 : }
441 0 : return SeekPromise::CreateAndResolve(seekTime, __func__);
442 : }
443 :
444 : RefPtr<MediaSourceTrackDemuxer::SamplesPromise>
445 0 : MediaSourceTrackDemuxer::DoGetSamples(int32_t aNumSamples)
446 : {
447 0 : if (mReset) {
448 : // If a seek (or reset) was recently performed, we ensure that the data
449 : // we are about to retrieve is still available.
450 0 : TimeIntervals buffered = mManager->Buffered(mType);
451 0 : buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
452 :
453 0 : if (!buffered.Length() && mManager->IsEnded()) {
454 : return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_END_OF_STREAM,
455 0 : __func__);
456 : }
457 0 : if (!buffered.ContainsWithStrictEnd(TimeUnit::Zero())) {
458 : return SamplesPromise::CreateAndReject(
459 0 : NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
460 : }
461 0 : mReset = false;
462 : }
463 0 : RefPtr<MediaRawData> sample;
464 0 : if (mNextSample) {
465 0 : sample = mNextSample.ref();
466 0 : mNextSample.reset();
467 : } else {
468 0 : MediaResult result = NS_OK;
469 0 : sample = mManager->GetSample(mType, MediaSourceDemuxer::EOS_FUZZ, result);
470 0 : if (!sample) {
471 0 : if (result == NS_ERROR_DOM_MEDIA_END_OF_STREAM ||
472 0 : result == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
473 : return SamplesPromise::CreateAndReject(
474 0 : (result == NS_ERROR_DOM_MEDIA_END_OF_STREAM && mManager->IsEnded())
475 : ? NS_ERROR_DOM_MEDIA_END_OF_STREAM
476 0 : : NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
477 : }
478 0 : return SamplesPromise::CreateAndReject(result, __func__);
479 : }
480 : }
481 0 : RefPtr<SamplesHolder> samples = new SamplesHolder;
482 0 : samples->mSamples.AppendElement(sample);
483 0 : if (mNextRandomAccessPoint <= sample->mTime) {
484 0 : MonitorAutoLock mon(mMonitor);
485 : mNextRandomAccessPoint =
486 0 : mManager->GetNextRandomAccessPoint(mType, MediaSourceDemuxer::EOS_FUZZ);
487 : }
488 0 : return SamplesPromise::CreateAndResolve(samples, __func__);
489 : }
490 :
491 : RefPtr<MediaSourceTrackDemuxer::SkipAccessPointPromise>
492 0 : MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint(
493 : const TimeUnit& aTimeThreadshold)
494 : {
495 0 : uint32_t parsed = 0;
496 : // Ensure that the data we are about to skip to is still available.
497 0 : TimeIntervals buffered = mManager->Buffered(mType);
498 0 : buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
499 0 : if (buffered.ContainsWithStrictEnd(aTimeThreadshold)) {
500 : bool found;
501 0 : parsed = mManager->SkipToNextRandomAccessPoint(mType,
502 : aTimeThreadshold,
503 : MediaSourceDemuxer::EOS_FUZZ,
504 : found);
505 0 : if (found) {
506 0 : return SkipAccessPointPromise::CreateAndResolve(parsed, __func__);
507 : }
508 : }
509 : SkipFailureHolder holder(
510 0 : mManager->IsEnded() ? NS_ERROR_DOM_MEDIA_END_OF_STREAM :
511 0 : NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, parsed);
512 0 : return SkipAccessPointPromise::CreateAndReject(holder, __func__);
513 : }
514 :
515 : } // namespace mozilla
|
