Line data Source code
1 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 : /* This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this
5 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #if !defined(MediaFormatReader_h_)
8 : #define MediaFormatReader_h_
9 :
10 : #include "mozilla/Atomics.h"
11 : #include "mozilla/Maybe.h"
12 : #include "mozilla/TaskQueue.h"
13 : #include "mozilla/Mutex.h"
14 :
15 : #include "MediaEventSource.h"
16 : #include "MediaDataDemuxer.h"
17 : #include "MediaDecoderReader.h"
18 : #include "MediaPrefs.h"
19 : #include "nsAutoPtr.h"
20 : #include "PDMFactory.h"
21 :
22 : namespace mozilla {
23 :
24 : class CDMProxy;
25 :
26 : class MediaFormatReader final : public MediaDecoderReader
27 : {
28 : typedef TrackInfo::TrackType TrackType;
29 : typedef MozPromise<bool, MediaResult, /* IsExclusive = */ true> NotifyDataArrivedPromise;
30 :
31 : public:
32 : MediaFormatReader(const MediaDecoderReaderInit& aInit, MediaDataDemuxer* aDemuxer);
33 :
34 : virtual ~MediaFormatReader();
35 :
36 : size_t SizeOfVideoQueueInFrames() override;
37 : size_t SizeOfAudioQueueInFrames() override;
38 :
39 : RefPtr<VideoDataPromise>
40 : RequestVideoData(const media::TimeUnit& aTimeThreshold) override;
41 :
42 : RefPtr<AudioDataPromise> RequestAudioData() override;
43 :
44 : RefPtr<MetadataPromise> AsyncReadMetadata() override;
45 :
46 : void ReadUpdatedMetadata(MediaInfo* aInfo) override;
47 :
48 : RefPtr<SeekPromise> Seek(const SeekTarget& aTarget) override;
49 :
50 : protected:
51 : void NotifyDataArrived() override;
52 : void UpdateBuffered() override;
53 :
54 : public:
55 : // For Media Resource Management
56 : void ReleaseResources() override;
57 :
58 : nsresult ResetDecode(TrackSet aTracks) override;
59 :
60 : RefPtr<ShutdownPromise> Shutdown() override;
61 :
62 0 : bool IsAsync() const override { return true; }
63 :
64 : bool VideoIsHardwareAccelerated() const override;
65 :
66 0 : bool IsWaitForDataSupported() const override { return true; }
67 : RefPtr<WaitForDataPromise> WaitForData(MediaData::Type aType) override;
68 :
69 0 : bool UseBufferingHeuristics() const override
70 : {
71 0 : return mTrackDemuxersMayBlock;
72 : }
73 :
74 : void SetCDMProxy(CDMProxy* aProxy) override;
75 :
76 : // Returns a string describing the state of the decoder data.
77 : // Used for debugging purposes.
78 : void GetMozDebugReaderData(nsACString& aString);
79 :
80 : void SetVideoNullDecode(bool aIsNullDecode) override;
81 :
82 : private:
83 : nsresult InitInternal() override;
84 :
85 0 : bool HasVideo() const { return mVideo.mTrackDemuxer; }
86 0 : bool HasAudio() const { return mAudio.mTrackDemuxer; }
87 :
88 : bool IsWaitingOnCDMResource();
89 :
90 : bool InitDemuxer();
91 : // Notify the track demuxers that new data has been received.
92 : void NotifyTrackDemuxers();
93 : void ReturnOutput(MediaData* aData, TrackType aTrack);
94 :
95 : // Enqueues a task to call Update(aTrack) on the decoder task queue.
96 : // Lock for corresponding track must be held.
97 : void ScheduleUpdate(TrackType aTrack);
98 : void Update(TrackType aTrack);
99 : // Handle actions should more data be received.
100 : // Returns true if no more action is required.
101 : bool UpdateReceivedNewData(TrackType aTrack);
102 : // Called when new samples need to be demuxed.
103 : void RequestDemuxSamples(TrackType aTrack);
104 : // Handle demuxed samples by the input behavior.
105 : void HandleDemuxedSamples(TrackType aTrack,
106 : AbstractMediaDecoder::AutoNotifyDecoded& aA);
107 : // Decode any pending already demuxed samples.
108 : void DecodeDemuxedSamples(TrackType aTrack,
109 : MediaRawData* aSample);
110 :
111 0 : struct InternalSeekTarget
112 : {
113 0 : InternalSeekTarget(const media::TimeInterval& aTime, bool aDropTarget)
114 0 : : mTime(aTime)
115 : , mDropTarget(aDropTarget)
116 : , mWaiting(false)
117 0 : , mHasSeeked(false)
118 : {
119 0 : }
120 :
121 0 : media::TimeUnit Time() const { return mTime.mStart; }
122 0 : media::TimeUnit EndTime() const { return mTime.mEnd; }
123 0 : bool Contains(const media::TimeUnit& aTime) const
124 : {
125 0 : return mTime.Contains(aTime);
126 : }
127 :
128 : media::TimeInterval mTime;
129 : bool mDropTarget;
130 : bool mWaiting;
131 : bool mHasSeeked;
132 : };
133 :
134 : // Perform an internal seek to aTime. If aDropTarget is true then
135 : // the first sample past the target will be dropped.
136 : void InternalSeek(TrackType aTrack, const InternalSeekTarget& aTarget);
137 :
138 : // Drain the current decoder.
139 : void DrainDecoder(TrackType aTrack);
140 : void NotifyNewOutput(TrackType aTrack,
141 : const MediaDataDecoder::DecodedData& aResults);
142 : void NotifyError(TrackType aTrack, const MediaResult& aError);
143 : void NotifyWaitingForData(TrackType aTrack);
144 : void NotifyWaitingForKey(TrackType aTrack);
145 : void NotifyEndOfStream(TrackType aTrack);
146 :
147 : void ExtractCryptoInitData(nsTArray<uint8_t>& aInitData);
148 :
149 : // Initializes mLayersBackendType if possible.
150 : void InitLayersBackendType();
151 :
152 : void Reset(TrackType aTrack);
153 : void DropDecodedSamples(TrackType aTrack);
154 :
155 : bool ShouldSkip(media::TimeUnit aTimeThreshold);
156 :
157 : void SetVideoDecodeThreshold();
158 :
159 : size_t SizeOfQueue(TrackType aTrack);
160 :
161 : RefPtr<PDMFactory> mPlatform;
162 :
163 : enum class DrainState
164 : {
165 : None,
166 : DrainRequested,
167 : Draining,
168 : PartialDrainPending,
169 : DrainCompleted,
170 : DrainAborted,
171 : };
172 :
173 0 : class SharedShutdownPromiseHolder : public MozPromiseHolder<ShutdownPromise>
174 : {
175 0 : NS_INLINE_DECL_THREADSAFE_REFCOUNTING(SharedShutdownPromiseHolder)
176 : private:
177 0 : ~SharedShutdownPromiseHolder() { }
178 : };
179 :
180 0 : struct DecoderData
181 : {
182 0 : DecoderData(MediaFormatReader* aOwner,
183 : MediaData::Type aType,
184 : uint32_t aNumOfMaxError)
185 0 : : mOwner(aOwner)
186 : , mType(aType)
187 : , mMutex("DecoderData")
188 : , mDescription("shutdown")
189 : , mUpdateScheduled(false)
190 : , mDemuxEOS(false)
191 : , mWaitingForData(false)
192 : , mWaitingForKey(false)
193 : , mReceivedNewData(false)
194 : , mFlushing(false)
195 : , mFlushed(true)
196 : , mDrainState(DrainState::None)
197 : , mNumOfConsecutiveError(0)
198 : , mMaxConsecutiveError(aNumOfMaxError)
199 0 : , mFirstFrameTime(Some(media::TimeUnit::Zero()))
200 : , mNumSamplesInput(0)
201 : , mNumSamplesOutput(0)
202 : , mNumSamplesOutputTotal(0)
203 : , mNumSamplesSkippedTotal(0)
204 : , mSizeOfQueue(0)
205 : , mIsHardwareAccelerated(false)
206 : , mLastStreamSourceID(UINT32_MAX)
207 0 : , mIsNullDecode(false)
208 : {
209 0 : }
210 :
211 : MediaFormatReader* mOwner;
212 : // Disambiguate Audio vs Video.
213 : MediaData::Type mType;
214 : RefPtr<MediaTrackDemuxer> mTrackDemuxer;
215 : // TaskQueue on which decoder can choose to decode.
216 : // Only non-null up until the decoder is created.
217 : RefPtr<TaskQueue> mTaskQueue;
218 :
219 : // Mutex protecting mDescription and mDecoder.
220 : Mutex mMutex;
221 : // The platform decoder.
222 : RefPtr<MediaDataDecoder> mDecoder;
223 : const char* mDescription;
224 : void ShutdownDecoder();
225 :
226 : // Only accessed from reader's task queue.
227 : bool mUpdateScheduled;
228 : bool mDemuxEOS;
229 : bool mWaitingForData;
230 : bool mWaitingForKey;
231 : bool mReceivedNewData;
232 :
233 : // Pending seek.
234 : MozPromiseRequestHolder<MediaTrackDemuxer::SeekPromise> mSeekRequest;
235 :
236 : // Queued demux samples waiting to be decoded.
237 : nsTArray<RefPtr<MediaRawData>> mQueuedSamples;
238 : MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest;
239 : // A WaitingPromise is pending if the demuxer is waiting for data or
240 : // if the decoder is waiting for a key.
241 : MozPromiseHolder<WaitForDataPromise> mWaitingPromise;
242 0 : bool HasWaitingPromise() const
243 : {
244 0 : MOZ_ASSERT(mOwner->OnTaskQueue());
245 0 : return !mWaitingPromise.IsEmpty();
246 : }
247 0 : bool IsWaiting() const
248 : {
249 0 : MOZ_ASSERT(mOwner->OnTaskQueue());
250 0 : return mWaitingForData || mWaitingForKey;
251 : }
252 :
253 : // MediaDataDecoder handler's variables.
254 : MozPromiseRequestHolder<MediaDataDecoder::DecodePromise> mDecodeRequest;
255 : bool mFlushing; // True if flush is in action.
256 : // Set to true if the last operation run on the decoder was a flush.
257 : bool mFlushed;
258 : RefPtr<SharedShutdownPromiseHolder> mShutdownPromise;
259 :
260 : MozPromiseRequestHolder<MediaDataDecoder::DecodePromise> mDrainRequest;
261 : DrainState mDrainState;
262 0 : bool HasPendingDrain() const
263 : {
264 0 : return mDrainState != DrainState::None;
265 : }
266 0 : bool HasCompletedDrain() const
267 : {
268 0 : return mDrainState == DrainState::DrainCompleted ||
269 0 : mDrainState == DrainState::DrainAborted;
270 : }
271 0 : void RequestDrain()
272 : {
273 0 : MOZ_RELEASE_ASSERT(mDrainState == DrainState::None);
274 0 : mDrainState = DrainState::DrainRequested;
275 0 : }
276 :
277 : uint32_t mNumOfConsecutiveError;
278 : uint32_t mMaxConsecutiveError;
279 : // Set when we haven't yet decoded the first frame.
280 : // Cleared once the first frame has been decoded.
281 : // This is used to determine, upon error, if we should try again to decode
282 : // the frame, or skip to the next keyframe.
283 : Maybe<media::TimeUnit> mFirstFrameTime;
284 :
285 : Maybe<MediaResult> mError;
286 0 : bool HasFatalError() const
287 : {
288 0 : if (!mError.isSome()) {
289 0 : return false;
290 : }
291 0 : if (mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR) {
292 : // Allow decode errors to be non-fatal, but give up
293 : // if we have too many, or if warnings should be treated as errors.
294 0 : return mNumOfConsecutiveError > mMaxConsecutiveError
295 0 : || MediaPrefs::MediaWarningsAsErrors();
296 0 : } else if (mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) {
297 : // If the caller asked for a new decoder we shouldn't treat
298 : // it as fatal.
299 0 : return false;
300 : } else {
301 : // All other error types are fatal
302 0 : return true;
303 : }
304 : }
305 :
306 : // If set, all decoded samples prior mTimeThreshold will be dropped.
307 : // Used for internal seeking when a change of stream is detected or when
308 : // encountering data discontinuity.
309 : Maybe<InternalSeekTarget> mTimeThreshold;
310 : // Time of last decoded sample returned.
311 : Maybe<media::TimeInterval> mLastDecodedSampleTime;
312 :
313 : // Decoded samples returned my mDecoder awaiting being returned to
314 : // state machine upon request.
315 : nsTArray<RefPtr<MediaData>> mOutput;
316 : uint64_t mNumSamplesInput;
317 : uint64_t mNumSamplesOutput;
318 : uint64_t mNumSamplesOutputTotal;
319 : uint64_t mNumSamplesSkippedTotal;
320 :
321 : // These get overridden in the templated concrete class.
322 : // Indicate if we have a pending promise for decoded frame.
323 : // Rejecting the promise will stop the reader from decoding ahead.
324 : virtual bool HasPromise() const = 0;
325 : virtual void RejectPromise(const MediaResult& aError,
326 : const char* aMethodName) = 0;
327 :
328 : // Clear track demuxer related data.
329 0 : void ResetDemuxer()
330 : {
331 0 : mDemuxRequest.DisconnectIfExists();
332 0 : mSeekRequest.DisconnectIfExists();
333 0 : mTrackDemuxer->Reset();
334 0 : mQueuedSamples.Clear();
335 0 : }
336 :
337 : // Flush the decoder if present and reset decoding related data.
338 : // Following a flush, the decoder is ready to accept any new data.
339 : void Flush();
340 :
341 0 : bool CancelWaitingForKey()
342 : {
343 0 : if (!mWaitingForKey) {
344 0 : return false;
345 : }
346 0 : mWaitingForKey = false;
347 0 : if (IsWaiting() || !HasWaitingPromise()) {
348 0 : return false;
349 : }
350 0 : mWaitingPromise.Resolve(mType, __func__);
351 0 : return true;
352 : }
353 :
354 : // Reset the state of the DecoderData, clearing all queued frames
355 : // (pending demuxed and decoded).
356 : // The track demuxer is *not* reset.
357 0 : void ResetState()
358 : {
359 0 : MOZ_ASSERT(mOwner->OnTaskQueue());
360 0 : mDemuxEOS = false;
361 0 : mWaitingForData = false;
362 0 : mQueuedSamples.Clear();
363 0 : mDecodeRequest.DisconnectIfExists();
364 0 : mDrainRequest.DisconnectIfExists();
365 0 : mDrainState = DrainState::None;
366 0 : CancelWaitingForKey();
367 0 : mTimeThreshold.reset();
368 0 : mLastDecodedSampleTime.reset();
369 0 : mOutput.Clear();
370 0 : mNumSamplesInput = 0;
371 0 : mNumSamplesOutput = 0;
372 0 : mSizeOfQueue = 0;
373 0 : mNextStreamSourceID.reset();
374 0 : if (!HasFatalError()) {
375 0 : mError.reset();
376 : }
377 0 : }
378 :
379 0 : bool HasInternalSeekPending() const
380 : {
381 0 : return mTimeThreshold && !mTimeThreshold.ref().mHasSeeked;
382 : }
383 :
384 : // Used by the MDSM for logging purposes.
385 : Atomic<size_t> mSizeOfQueue;
386 : // Used by the MDSM to determine if video decoding is hardware accelerated.
387 : // This value is updated after a frame is successfully decoded.
388 : Atomic<bool> mIsHardwareAccelerated;
389 : // Sample format monitoring.
390 : uint32_t mLastStreamSourceID;
391 : Maybe<uint32_t> mNextStreamSourceID;
392 : media::TimeIntervals mTimeRanges;
393 : Maybe<media::TimeUnit> mLastTimeRangesEnd;
394 : // TrackInfo as first discovered during ReadMetadata.
395 : UniquePtr<TrackInfo> mOriginalInfo;
396 : RefPtr<TrackInfoSharedPtr> mInfo;
397 : Maybe<media::TimeUnit> mFirstDemuxedSampleTime;
398 : // Use NullDecoderModule or not.
399 : bool mIsNullDecode;
400 :
401 : };
402 :
403 : template <typename Type>
404 0 : class DecoderDataWithPromise : public DecoderData
405 : {
406 : public:
407 0 : DecoderDataWithPromise(MediaFormatReader* aOwner,
408 : MediaData::Type aType,
409 : uint32_t aNumOfMaxError)
410 : : DecoderData(aOwner, aType, aNumOfMaxError)
411 0 : , mHasPromise(false)
412 : {
413 0 : }
414 :
415 0 : bool HasPromise() const override
416 : {
417 0 : return mHasPromise;
418 : }
419 :
420 0 : RefPtr<DataPromise<Type>> EnsurePromise(const char* aMethodName)
421 : {
422 0 : MOZ_ASSERT(mOwner->OnTaskQueue());
423 0 : mHasPromise = true;
424 0 : return mPromise.Ensure(aMethodName);
425 : }
426 :
427 0 : void ResolvePromise(Type* aData, const char* aMethodName)
428 : {
429 0 : MOZ_ASSERT(mOwner->OnTaskQueue());
430 0 : mPromise.Resolve(aData, aMethodName);
431 0 : mHasPromise = false;
432 0 : }
433 :
434 0 : void RejectPromise(const MediaResult& aError,
435 : const char* aMethodName) override
436 : {
437 0 : MOZ_ASSERT(mOwner->OnTaskQueue());
438 0 : mPromise.Reject(aError, aMethodName);
439 0 : mHasPromise = false;
440 0 : }
441 :
442 : private:
443 : MozPromiseHolder<DataPromise<Type>> mPromise;
444 : Atomic<bool> mHasPromise;
445 : };
446 :
447 : DecoderDataWithPromise<AudioData> mAudio;
448 : DecoderDataWithPromise<VideoData> mVideo;
449 :
450 : // Returns true when the decoder for this track needs input.
451 : bool NeedInput(DecoderData& aDecoder);
452 :
453 : DecoderData& GetDecoderData(TrackType aTrack);
454 :
455 : // Demuxer objects.
456 : class DemuxerProxy;
457 : UniquePtr<DemuxerProxy> mDemuxer;
458 : bool mDemuxerInitDone;
459 : void OnDemuxerInitDone(const MediaResult& aResult);
460 : void OnDemuxerInitFailed(const MediaResult& aError);
461 : MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest;
462 : MozPromiseRequestHolder<NotifyDataArrivedPromise> mNotifyDataArrivedPromise;
463 : bool mPendingNotifyDataArrived;
464 : void OnDemuxFailed(TrackType aTrack, const MediaResult &aError);
465 :
466 : void DoDemuxVideo();
467 : void OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
468 0 : void OnVideoDemuxFailed(const MediaResult& aError)
469 : {
470 0 : OnDemuxFailed(TrackType::kVideoTrack, aError);
471 0 : }
472 :
473 : void DoDemuxAudio();
474 : void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
475 0 : void OnAudioDemuxFailed(const MediaResult& aError)
476 : {
477 0 : OnDemuxFailed(TrackType::kAudioTrack, aError);
478 0 : }
479 :
480 : void SkipVideoDemuxToNextKeyFrame(media::TimeUnit aTimeThreshold);
481 : MozPromiseRequestHolder<MediaTrackDemuxer::SkipAccessPointPromise> mSkipRequest;
482 : void VideoSkipReset(uint32_t aSkipped);
483 : void OnVideoSkipCompleted(uint32_t aSkipped);
484 : void OnVideoSkipFailed(MediaTrackDemuxer::SkipFailureHolder aFailure);
485 :
486 : // The last number of decoded output frames that we've reported to
487 : // MediaDecoder::NotifyDecoded(). We diff the number of output video
488 : // frames every time that DecodeVideoData() is called, and report the
489 : // delta there.
490 : uint64_t mLastReportedNumDecodedFrames;
491 :
492 : // Timestamp of the previous decoded keyframe, in microseconds.
493 : int64_t mPreviousDecodedKeyframeTime_us;
494 : // Default mLastDecodedKeyframeTime_us value, must be bigger than anything.
495 : static const int64_t sNoPreviousDecodedKeyframe = INT64_MAX;
496 :
497 : RefPtr<layers::KnowsCompositor> mKnowsCompositor;
498 :
499 : // Metadata objects
500 : // True if we've read the streams' metadata.
501 : bool mInitDone;
502 : MozPromiseHolder<MetadataPromise> mMetadataPromise;
503 : bool IsEncrypted() const;
504 :
505 : // Set to true if any of our track buffers may be blocking.
506 : bool mTrackDemuxersMayBlock;
507 :
508 : // Seeking objects.
509 : void SetSeekTarget(const SeekTarget& aTarget);
510 0 : bool IsSeeking() const { return mPendingSeekTime.isSome(); }
511 0 : bool IsVideoSeeking() const
512 : {
513 0 : return IsSeeking() && mOriginalSeekTarget.IsVideoOnly();
514 : }
515 : void ScheduleSeek();
516 : void AttemptSeek();
517 : void OnSeekFailed(TrackType aTrack, const MediaResult& aError);
518 : void DoVideoSeek();
519 : void OnVideoSeekCompleted(media::TimeUnit aTime);
520 : void OnVideoSeekFailed(const MediaResult& aError);
521 : bool mSeekScheduled;
522 :
523 0 : void NotifyCompositorUpdated(RefPtr<layers::KnowsCompositor> aKnowsCompositor)
524 : {
525 0 : mKnowsCompositor = aKnowsCompositor.forget();
526 0 : }
527 :
528 : void DoAudioSeek();
529 : void OnAudioSeekCompleted(media::TimeUnit aTime);
530 : void OnAudioSeekFailed(const MediaResult& aError);
531 : // The SeekTarget that was last given to Seek()
532 : SeekTarget mOriginalSeekTarget;
533 : // Temporary seek information while we wait for the data
534 : Maybe<media::TimeUnit> mFallbackSeekTime;
535 : Maybe<media::TimeUnit> mPendingSeekTime;
536 : MozPromiseHolder<SeekPromise> mSeekPromise;
537 :
538 : RefPtr<VideoFrameContainer> mVideoFrameContainer;
539 : layers::ImageContainer* GetImageContainer();
540 :
541 : RefPtr<CDMProxy> mCDMProxy;
542 :
543 : RefPtr<GMPCrashHelper> mCrashHelper;
544 :
545 : void SetNullDecode(TrackType aTrack, bool aIsNullDecode);
546 :
547 : class DecoderFactory;
548 : UniquePtr<DecoderFactory> mDecoderFactory;
549 :
550 : class ShutdownPromisePool;
551 : UniquePtr<ShutdownPromisePool> mShutdownPromisePool;
552 :
553 : MediaEventListener mCompositorUpdatedListener;
554 : MediaEventListener mOnTrackWaitingForKeyListener;
555 :
556 : void OnFirstDemuxCompleted(TrackInfo::TrackType aType,
557 : RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
558 :
559 : void OnFirstDemuxFailed(TrackInfo::TrackType aType, const MediaResult& aError);
560 :
561 : void MaybeResolveMetadataPromise();
562 :
563 : UniquePtr<MetadataTags> mTags;
564 :
565 : // A flag indicating if the start time is known or not.
566 : bool mHasStartTime = false;
567 :
568 : void ShutdownDecoder(TrackType aTrack);
569 : RefPtr<ShutdownPromise> TearDownDecoders();
570 : };
571 :
572 : } // namespace mozilla
573 :
574 : #endif
|
