Line data Source code
1 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 : /* This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this
5 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #ifdef XP_WIN
8 : // Include Windows headers required for enabling high precision timers.
9 : #include "windows.h"
10 : #include "mmsystem.h"
11 : #endif
12 :
13 : #include <algorithm>
14 : #include <stdint.h>
15 :
16 : #include "gfx2DGlue.h"
17 :
18 : #include "mediasink/AudioSink.h"
19 : #include "mediasink/AudioSinkWrapper.h"
20 : #include "mediasink/DecodedStream.h"
21 : #include "mediasink/OutputStreamManager.h"
22 : #include "mediasink/VideoSink.h"
23 : #include "mozilla/DebugOnly.h"
24 : #include "mozilla/IndexSequence.h"
25 : #include "mozilla/Logging.h"
26 : #include "mozilla/mozalloc.h"
27 : #include "mozilla/MathAlgorithms.h"
28 : #include "mozilla/Preferences.h"
29 : #include "mozilla/SharedThreadPool.h"
30 : #include "mozilla/SizePrintfMacros.h"
31 : #include "mozilla/Sprintf.h"
32 : #include "mozilla/TaskQueue.h"
33 : #include "mozilla/Tuple.h"
34 :
35 : #include "nsComponentManagerUtils.h"
36 : #include "nsContentUtils.h"
37 : #include "nsIEventTarget.h"
38 : #include "nsITimer.h"
39 : #include "nsPrintfCString.h"
40 : #include "nsTArray.h"
41 : #include "nsDeque.h"
42 : #include "prenv.h"
43 :
44 : #include "AudioSegment.h"
45 : #include "DOMMediaStream.h"
46 : #include "ImageContainer.h"
47 : #include "MediaDecoder.h"
48 : #include "MediaDecoderReader.h"
49 : #include "MediaDecoderReaderWrapper.h"
50 : #include "MediaDecoderStateMachine.h"
51 : #include "MediaShutdownManager.h"
52 : #include "MediaPrefs.h"
53 : #include "MediaTimer.h"
54 : #include "TimeUnits.h"
55 : #include "VideoSegment.h"
56 : #include "VideoUtils.h"
57 : #include "gfxPrefs.h"
58 :
59 : namespace mozilla {
60 :
61 : using namespace mozilla::dom;
62 : using namespace mozilla::layers;
63 : using namespace mozilla::media;
64 :
65 : #define NS_DispatchToMainThread(...) CompileError_UseAbstractThreadDispatchInstead
66 :
67 : // avoid redefined macro in unified build
68 : #undef FMT
69 : #undef LOG
70 : #undef LOGV
71 : #undef LOGW
72 : #undef SFMT
73 : #undef SLOG
74 : #undef SLOGW
75 :
76 : #define FMT(x, ...) "Decoder=%p " x, mDecoderID, ##__VA_ARGS__
77 : #define LOG(x, ...) MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, (FMT(x, ##__VA_ARGS__)))
78 : #define LOGV(x, ...) MOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, (FMT(x, ##__VA_ARGS__)))
79 : #define LOGW(x, ...) NS_WARNING(nsPrintfCString(FMT(x, ##__VA_ARGS__)).get())
80 :
81 : // Used by StateObject and its sub-classes
82 : #define SFMT(x, ...) "Decoder=%p state=%s " x, mMaster->mDecoderID, ToStateStr(GetState()), ##__VA_ARGS__
83 : #define SLOG(x, ...) MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, (SFMT(x, ##__VA_ARGS__)))
84 : #define SLOGW(x, ...) NS_WARNING(nsPrintfCString(SFMT(x, ##__VA_ARGS__)).get())
85 :
86 : // Certain constants get stored as member variables and then adjusted by various
87 : // scale factors on a per-decoder basis. We want to make sure to avoid using these
88 : // constants directly, so we put them in a namespace.
89 : namespace detail {
90 :
91 : // Resume a suspended video decoder to the current playback position plus this
92 : // time premium for compensating the seeking delay.
93 : static constexpr auto RESUME_VIDEO_PREMIUM = TimeUnit::FromMicroseconds(125000);
94 :
95 : // If audio queue has less than this much decoded audio, we won't risk
96 : // trying to decode the video, we'll skip decoding video up to the next
97 : // keyframe. We may increase this value for an individual decoder if we
98 : // encounter video frames which take a long time to decode.
99 : static constexpr auto LOW_AUDIO_THRESHOLD = TimeUnit::FromMicroseconds(300000);
100 :
101 : static const int64_t AMPLE_AUDIO_USECS = 2000000;
102 :
103 : // If more than this much decoded audio is queued, we'll hold off
104 : // decoding more audio. If we increase the low audio threshold (see
105 : // LOW_AUDIO_THRESHOLD above) we'll also increase this value to ensure it's not
106 : // less than the low audio threshold.
107 : static constexpr auto AMPLE_AUDIO_THRESHOLD = TimeUnit::FromMicroseconds(AMPLE_AUDIO_USECS);
108 :
109 : } // namespace detail
110 :
111 : // If we have fewer than LOW_VIDEO_FRAMES decoded frames, and
112 : // we're not "prerolling video", we'll skip the video up to the next keyframe
113 : // which is at or after the current playback position.
114 : static const uint32_t LOW_VIDEO_FRAMES = 2;
115 :
116 : // Threshold that used to check if we are low on decoded video.
117 : // If the last video frame's end time |mDecodedVideoEndTime| is more than
118 : // |LOW_VIDEO_THRESHOLD*mPlaybackRate| after the current clock in
119 : // Advanceframe(), the video decode is lagging, and we skip to next keyframe.
120 : static constexpr auto LOW_VIDEO_THRESHOLD = TimeUnit::FromMicroseconds(60000);
121 :
122 : // Arbitrary "frame duration" when playing only audio.
123 : static const int AUDIO_DURATION_USECS = 40000;
124 :
125 : // If we increase our "low audio threshold" (see LOW_AUDIO_THRESHOLD above), we
126 : // use this as a factor in all our calculations. Increasing this will cause
127 : // us to be more likely to increase our low audio threshold, and to
128 : // increase it by more.
129 : static const int THRESHOLD_FACTOR = 2;
130 :
131 : namespace detail {
132 :
133 : // If we have less than this much buffered data available, we'll consider
134 : // ourselves to be running low on buffered data. We determine how much
135 : // buffered data we have remaining using the reader's GetBuffered()
136 : // implementation.
137 : static const int64_t LOW_BUFFER_THRESHOLD_USECS = 5000000;
138 :
139 : static constexpr auto LOW_BUFFER_THRESHOLD = TimeUnit::FromMicroseconds(LOW_BUFFER_THRESHOLD_USECS);
140 :
141 : // LOW_BUFFER_THRESHOLD_USECS needs to be greater than AMPLE_AUDIO_USECS, otherwise
142 : // the skip-to-keyframe logic can activate when we're running low on data.
143 : static_assert(LOW_BUFFER_THRESHOLD_USECS > AMPLE_AUDIO_USECS,
144 : "LOW_BUFFER_THRESHOLD_USECS is too small");
145 :
146 : } // namespace detail
147 :
148 : // Amount of excess data to add in to the "should we buffer" calculation.
149 : static constexpr auto EXHAUSTED_DATA_MARGIN = TimeUnit::FromMicroseconds(100000);
150 :
151 : static const uint32_t MIN_VIDEO_QUEUE_SIZE = 3;
152 : static const uint32_t MAX_VIDEO_QUEUE_SIZE = 10;
153 : #ifdef MOZ_APPLEMEDIA
154 : static const uint32_t HW_VIDEO_QUEUE_SIZE = 10;
155 : #else
156 : static const uint32_t HW_VIDEO_QUEUE_SIZE = 3;
157 : #endif
158 : static const uint32_t VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE = 9999;
159 :
160 : static uint32_t sVideoQueueDefaultSize = MAX_VIDEO_QUEUE_SIZE;
161 : static uint32_t sVideoQueueHWAccelSize = HW_VIDEO_QUEUE_SIZE;
162 : static uint32_t sVideoQueueSendToCompositorSize =
163 : VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE;
164 :
165 0 : static void InitVideoQueuePrefs()
166 : {
167 0 : MOZ_ASSERT(NS_IsMainThread());
168 : static bool sPrefInit = false;
169 0 : if (!sPrefInit) {
170 0 : sPrefInit = true;
171 0 : sVideoQueueDefaultSize = Preferences::GetUint(
172 : "media.video-queue.default-size", MAX_VIDEO_QUEUE_SIZE);
173 0 : sVideoQueueHWAccelSize = Preferences::GetUint(
174 : "media.video-queue.hw-accel-size", HW_VIDEO_QUEUE_SIZE);
175 0 : sVideoQueueSendToCompositorSize = Preferences::GetUint(
176 : "media.video-queue.send-to-compositor-size", VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE);
177 : }
178 0 : }
179 :
180 : // Delay, in milliseconds, that tabs needs to be in background before video
181 : // decoding is suspended.
182 : static TimeDuration
183 0 : SuspendBackgroundVideoDelay()
184 : {
185 : return TimeDuration::FromMilliseconds(
186 0 : MediaPrefs::MDSMSuspendBackgroundVideoDelay());
187 : }
188 :
189 : class MediaDecoderStateMachine::StateObject
190 : {
191 : public:
192 0 : virtual ~StateObject() { }
193 0 : virtual void Exit() { } // Exit action.
194 0 : virtual void Step() { } // Perform a 'cycle' of this state object.
195 : virtual State GetState() const = 0;
196 :
197 : // Event handlers for various events.
198 0 : virtual void HandleCDMProxyReady() { }
199 0 : virtual void HandleAudioCaptured() { }
200 0 : virtual void HandleAudioDecoded(AudioData* aAudio)
201 : {
202 0 : Crash("Unexpected event!", __func__);
203 0 : }
204 0 : virtual void HandleVideoDecoded(VideoData* aVideo, TimeStamp aDecodeStart)
205 : {
206 0 : Crash("Unexpected event!", __func__);
207 0 : }
208 0 : virtual void HandleAudioWaited(MediaData::Type aType)
209 : {
210 0 : Crash("Unexpected event!", __func__);
211 0 : }
212 0 : virtual void HandleVideoWaited(MediaData::Type aType)
213 : {
214 0 : Crash("Unexpected event!", __func__);
215 0 : }
216 0 : virtual void HandleWaitingForAudio()
217 : {
218 0 : Crash("Unexpected event!", __func__);
219 0 : }
220 0 : virtual void HandleAudioCanceled()
221 : {
222 0 : Crash("Unexpected event!", __func__);
223 0 : }
224 0 : virtual void HandleEndOfAudio()
225 : {
226 0 : Crash("Unexpected event!", __func__);
227 0 : }
228 0 : virtual void HandleWaitingForVideo()
229 : {
230 0 : Crash("Unexpected event!", __func__);
231 0 : }
232 0 : virtual void HandleVideoCanceled()
233 : {
234 0 : Crash("Unexpected event!", __func__);
235 0 : }
236 0 : virtual void HandleEndOfVideo()
237 : {
238 0 : Crash("Unexpected event!", __func__);
239 0 : }
240 :
241 : virtual RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget);
242 :
243 : virtual RefPtr<ShutdownPromise> HandleShutdown();
244 :
245 : virtual void HandleVideoSuspendTimeout() = 0;
246 :
247 : virtual void HandleResumeVideoDecoding(const TimeUnit& aTarget);
248 :
249 0 : virtual void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) { }
250 :
251 0 : virtual nsCString GetDebugInfo() { return nsCString(); }
252 :
253 : private:
254 : template <class S, typename R, typename... As>
255 : auto ReturnTypeHelper(R(S::*)(As...)) -> R;
256 :
257 0 : void Crash(const char* aReason, const char* aSite)
258 : {
259 : char buf[1024];
260 0 : SprintfLiteral(buf, "%s state=%s callsite=%s", aReason,
261 0 : ToStateStr(GetState()), aSite);
262 0 : MOZ_ReportAssertionFailure(buf, __FILE__, __LINE__);
263 0 : MOZ_CRASH();
264 : }
265 :
266 : protected:
267 : enum class EventVisibility : int8_t
268 : {
269 : Observable,
270 : Suppressed
271 : };
272 :
273 : using Master = MediaDecoderStateMachine;
274 0 : explicit StateObject(Master* aPtr) : mMaster(aPtr) { }
275 0 : TaskQueue* OwnerThread() const { return mMaster->mTaskQueue; }
276 0 : MediaResource* Resource() const { return mMaster->mResource; }
277 0 : MediaDecoderReaderWrapper* Reader() const { return mMaster->mReader; }
278 0 : const MediaInfo& Info() const { return mMaster->Info(); }
279 0 : bool IsExpectingMoreData() const
280 : {
281 : // We are expecting more data if either the resource states so, or if we
282 : // have a waiting promise pending (such as with non-MSE EME).
283 0 : return Resource()->IsExpectingMoreData()
284 0 : || mMaster->IsWaitingAudioData()
285 0 : || mMaster->IsWaitingVideoData();
286 : }
287 0 : MediaQueue<AudioData>& AudioQueue() const { return mMaster->mAudioQueue; }
288 0 : MediaQueue<VideoData>& VideoQueue() const { return mMaster->mVideoQueue; }
289 :
290 : template <class S, typename... Args, size_t... Indexes>
291 : auto
292 0 : CallEnterMemberFunction(S* aS,
293 : Tuple<Args...>& aTuple,
294 : IndexSequence<Indexes...>)
295 : -> decltype(ReturnTypeHelper(&S::Enter))
296 : {
297 0 : return aS->Enter(Move(Get<Indexes>(aTuple))...);
298 : }
299 :
300 : // Note this function will delete the current state object.
301 : // Don't access members to avoid UAF after this call.
302 : template <class S, typename... Ts>
303 0 : auto SetState(Ts&&... aArgs)
304 : -> decltype(ReturnTypeHelper(&S::Enter))
305 : {
306 : // |aArgs| must be passed by reference to avoid passing MOZ_NON_PARAM class
307 : // SeekJob by value. See bug 1287006 and bug 1338374. But we still *must*
308 : // copy the parameters, because |Exit()| can modify them. See bug 1312321.
309 : // So we 1) pass the parameters by reference, but then 2) immediately copy
310 : // them into a Tuple to be safe against modification, and finally 3) move
311 : // the elements of the Tuple into the final function call.
312 0 : auto copiedArgs = MakeTuple(Forward<Ts>(aArgs)...);
313 :
314 : // keep mMaster in a local object because mMaster will become invalid after
315 : // the current state object is deleted.
316 0 : auto master = mMaster;
317 :
318 0 : auto* s = new S(master);
319 :
320 0 : MOZ_ASSERT(GetState() != s->GetState()
321 : || GetState() == DECODER_STATE_SEEKING);
322 :
323 0 : SLOG("change state to: %s", ToStateStr(s->GetState()));
324 :
325 0 : Exit();
326 :
327 0 : master->mStateObj.reset(s);
328 : return CallEnterMemberFunction(s, copiedArgs,
329 0 : typename IndexSequenceFor<Ts...>::Type());
330 : }
331 :
332 : RefPtr<MediaDecoder::SeekPromise>
333 : SetSeekingState(SeekJob&& aSeekJob, EventVisibility aVisibility);
334 :
335 : // Take a raw pointer in order not to change the life cycle of MDSM.
336 : // It is guaranteed to be valid by MDSM.
337 : Master* mMaster;
338 : };
339 :
340 : /**
341 : * Purpose: decode metadata like duration and dimensions of the media resource.
342 : *
343 : * Transition to other states when decoding metadata is done:
344 : * SHUTDOWN if failing to decode metadata.
345 : * WAIT_FOR_CDM if the media is encrypted and CDM is not available.
346 : * DECODING_FIRSTFRAME otherwise.
347 : */
348 0 : class MediaDecoderStateMachine::DecodeMetadataState
349 : : public MediaDecoderStateMachine::StateObject
350 : {
351 : public:
352 0 : explicit DecodeMetadataState(Master* aPtr) : StateObject(aPtr) { }
353 :
354 0 : void Enter()
355 : {
356 0 : MOZ_ASSERT(!mMaster->mVideoDecodeSuspended);
357 0 : MOZ_ASSERT(!mMetadataRequest.Exists());
358 0 : SLOG("Dispatching AsyncReadMetadata");
359 :
360 : // Set mode to METADATA since we are about to read metadata.
361 0 : Resource()->SetReadMode(MediaCacheStream::MODE_METADATA);
362 :
363 : // We disconnect mMetadataRequest in Exit() so it is fine to capture
364 : // a raw pointer here.
365 0 : Reader()->ReadMetadata()
366 0 : ->Then(OwnerThread(), __func__,
367 0 : [this] (MetadataHolder&& aMetadata) {
368 0 : OnMetadataRead(Move(aMetadata));
369 0 : },
370 0 : [this] (const MediaResult& aError) {
371 0 : OnMetadataNotRead(aError);
372 0 : })
373 0 : ->Track(mMetadataRequest);
374 0 : }
375 :
376 0 : void Exit() override { mMetadataRequest.DisconnectIfExists(); }
377 :
378 0 : State GetState() const override { return DECODER_STATE_DECODING_METADATA; }
379 :
380 0 : RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget) override
381 : {
382 0 : MOZ_DIAGNOSTIC_ASSERT(false, "Can't seek while decoding metadata.");
383 : return MediaDecoder::SeekPromise::CreateAndReject(true, __func__);
384 : }
385 :
386 0 : void HandleVideoSuspendTimeout() override
387 : {
388 : // Do nothing since no decoders are created yet.
389 0 : }
390 :
391 0 : void HandleResumeVideoDecoding(const TimeUnit&) override
392 : {
393 : // We never suspend video decoding in this state.
394 0 : MOZ_ASSERT(false, "Shouldn't have suspended video decoding.");
395 : }
396 :
397 : private:
398 : void OnMetadataRead(MetadataHolder&& aMetadata);
399 :
400 0 : void OnMetadataNotRead(const MediaResult& aError)
401 : {
402 0 : mMetadataRequest.Complete();
403 0 : SLOGW("Decode metadata failed, shutting down decoder");
404 0 : mMaster->DecodeError(aError);
405 0 : }
406 :
407 : MozPromiseRequestHolder<MediaDecoderReader::MetadataPromise> mMetadataRequest;
408 : };
409 :
410 : /**
411 : * Purpose: wait for the CDM to start decoding.
412 : *
413 : * Transition to other states when CDM is ready:
414 : * SEEKING if any pending seek request.
415 : * DECODING_FIRSTFRAME otherwise.
416 : */
417 0 : class MediaDecoderStateMachine::WaitForCDMState
418 : : public MediaDecoderStateMachine::StateObject
419 : {
420 : public:
421 0 : explicit WaitForCDMState(Master* aPtr) : StateObject(aPtr) { }
422 :
423 0 : void Enter() { MOZ_ASSERT(!mMaster->mVideoDecodeSuspended); }
424 :
425 0 : void Exit() override
426 : {
427 : // mPendingSeek is either moved in HandleCDMProxyReady() or should be
428 : // rejected here before transition to SHUTDOWN.
429 0 : mPendingSeek.RejectIfExists(__func__);
430 0 : }
431 :
432 0 : State GetState() const override
433 : {
434 0 : return DECODER_STATE_WAIT_FOR_CDM;
435 : }
436 :
437 : void HandleCDMProxyReady() override;
438 :
439 0 : RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget) override
440 : {
441 0 : SLOG("Not Enough Data to seek at this stage, queuing seek");
442 0 : mPendingSeek.RejectIfExists(__func__);
443 0 : mPendingSeek.mTarget.emplace(aTarget);
444 0 : return mPendingSeek.mPromise.Ensure(__func__);
445 : }
446 :
447 0 : void HandleVideoSuspendTimeout() override
448 : {
449 : // Do nothing since no decoders are created yet.
450 0 : }
451 :
452 0 : void HandleResumeVideoDecoding(const TimeUnit&) override
453 : {
454 : // We never suspend video decoding in this state.
455 0 : MOZ_ASSERT(false, "Shouldn't have suspended video decoding.");
456 : }
457 :
458 : private:
459 : SeekJob mPendingSeek;
460 : };
461 :
462 : /**
463 : * Purpose: release decoder resources to save memory and hardware resources.
464 : *
465 : * Transition to:
466 : * SEEKING if any seek request or play state changes to PLAYING.
467 : */
468 0 : class MediaDecoderStateMachine::DormantState
469 : : public MediaDecoderStateMachine::StateObject
470 : {
471 : public:
472 0 : explicit DormantState(Master* aPtr) : StateObject(aPtr) { }
473 :
474 0 : void Enter()
475 : {
476 0 : if (mMaster->IsPlaying()) {
477 0 : mMaster->StopPlayback();
478 : }
479 :
480 : // Calculate the position to seek to when exiting dormant.
481 0 : auto t = mMaster->mMediaSink->IsStarted()
482 0 : ? mMaster->GetClock() : mMaster->GetMediaTime();
483 0 : mPendingSeek.mTarget.emplace(t, SeekTarget::Accurate);
484 : // SeekJob asserts |mTarget.IsValid() == !mPromise.IsEmpty()| so we
485 : // need to create the promise even it is not used at all.
486 : // The promise may be used when coming out of DormantState into
487 : // SeekingState.
488 : RefPtr<MediaDecoder::SeekPromise> x =
489 0 : mPendingSeek.mPromise.Ensure(__func__);
490 :
491 : // No need to call ResetDecode() and StopMediaSink() here.
492 : // We will do them during seeking when exiting dormant.
493 :
494 : // Ignore WAIT_FOR_DATA since we won't decode in dormant.
495 0 : mMaster->mAudioWaitRequest.DisconnectIfExists();
496 0 : mMaster->mVideoWaitRequest.DisconnectIfExists();
497 :
498 0 : MaybeReleaseResources();
499 0 : }
500 :
501 0 : void Exit() override
502 : {
503 : // mPendingSeek is either moved when exiting dormant or
504 : // should be rejected here before transition to SHUTDOWN.
505 0 : mPendingSeek.RejectIfExists(__func__);
506 0 : }
507 :
508 0 : State GetState() const override { return DECODER_STATE_DORMANT; }
509 :
510 : RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget) override;
511 :
512 0 : void HandleVideoSuspendTimeout() override
513 : {
514 : // Do nothing since we've released decoders in Enter().
515 0 : }
516 :
517 0 : void HandleResumeVideoDecoding(const TimeUnit&) override
518 : {
519 : // Do nothing since we won't resume decoding until exiting dormant.
520 0 : }
521 :
522 : void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override;
523 :
524 0 : void HandleAudioDecoded(AudioData*) override { MaybeReleaseResources(); }
525 0 : void HandleVideoDecoded(VideoData*, TimeStamp) override
526 : {
527 0 : MaybeReleaseResources();
528 0 : }
529 0 : void HandleWaitingForAudio() override { MaybeReleaseResources(); }
530 0 : void HandleWaitingForVideo() override { MaybeReleaseResources(); }
531 0 : void HandleAudioCanceled() override { MaybeReleaseResources(); }
532 0 : void HandleVideoCanceled() override { MaybeReleaseResources(); }
533 0 : void HandleEndOfAudio() override { MaybeReleaseResources(); }
534 0 : void HandleEndOfVideo() override { MaybeReleaseResources(); }
535 :
536 : private:
537 0 : void MaybeReleaseResources()
538 : {
539 0 : if (!mMaster->mAudioDataRequest.Exists() &&
540 0 : !mMaster->mVideoDataRequest.Exists()) {
541 : // Release decoders only when they are idle. Otherwise it might cause
542 : // decode error later when resetting decoders during seeking.
543 0 : mMaster->mReader->ReleaseResources();
544 : }
545 0 : }
546 :
547 : SeekJob mPendingSeek;
548 : };
549 :
550 : /**
551 : * Purpose: decode the 1st audio and video frames to fire the 'loadeddata' event.
552 : *
553 : * Transition to:
554 : * SHUTDOWN if any decode error.
555 : * SEEKING if any seek request.
556 : * DECODING when the 'loadeddata' event is fired.
557 : */
558 0 : class MediaDecoderStateMachine::DecodingFirstFrameState
559 : : public MediaDecoderStateMachine::StateObject
560 : {
561 : public:
562 0 : explicit DecodingFirstFrameState(Master* aPtr) : StateObject(aPtr) { }
563 :
564 : void Enter();
565 :
566 0 : void Exit() override
567 : {
568 : // mPendingSeek is either moved in MaybeFinishDecodeFirstFrame()
569 : // or should be rejected here before transition to SHUTDOWN.
570 0 : mPendingSeek.RejectIfExists(__func__);
571 0 : }
572 :
573 0 : State GetState() const override { return DECODER_STATE_DECODING_FIRSTFRAME; }
574 :
575 0 : void HandleAudioDecoded(AudioData* aAudio) override
576 : {
577 0 : mMaster->PushAudio(aAudio);
578 0 : MaybeFinishDecodeFirstFrame();
579 0 : }
580 :
581 0 : void HandleVideoDecoded(VideoData* aVideo, TimeStamp aDecodeStart) override
582 : {
583 0 : mMaster->PushVideo(aVideo);
584 0 : MaybeFinishDecodeFirstFrame();
585 0 : }
586 :
587 0 : void HandleWaitingForAudio() override
588 : {
589 0 : mMaster->WaitForData(MediaData::AUDIO_DATA);
590 0 : }
591 :
592 0 : void HandleAudioCanceled() override
593 : {
594 0 : mMaster->RequestAudioData();
595 0 : }
596 :
597 0 : void HandleEndOfAudio() override
598 : {
599 0 : AudioQueue().Finish();
600 0 : MaybeFinishDecodeFirstFrame();
601 0 : }
602 :
603 0 : void HandleWaitingForVideo() override
604 : {
605 0 : mMaster->WaitForData(MediaData::VIDEO_DATA);
606 0 : }
607 :
608 0 : void HandleVideoCanceled() override
609 : {
610 0 : mMaster->RequestVideoData(media::TimeUnit());
611 0 : }
612 :
613 0 : void HandleEndOfVideo() override
614 : {
615 0 : VideoQueue().Finish();
616 0 : MaybeFinishDecodeFirstFrame();
617 0 : }
618 :
619 0 : void HandleAudioWaited(MediaData::Type aType) override
620 : {
621 0 : mMaster->RequestAudioData();
622 0 : }
623 :
624 0 : void HandleVideoWaited(MediaData::Type aType) override
625 : {
626 0 : mMaster->RequestVideoData(media::TimeUnit());
627 0 : }
628 :
629 0 : void HandleVideoSuspendTimeout() override
630 : {
631 : // Do nothing for we need to decode the 1st video frame to get the
632 : // dimensions.
633 0 : }
634 :
635 0 : void HandleResumeVideoDecoding(const TimeUnit&) override
636 : {
637 : // We never suspend video decoding in this state.
638 0 : MOZ_ASSERT(false, "Shouldn't have suspended video decoding.");
639 : }
640 :
641 0 : RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget) override
642 : {
643 0 : if (mMaster->mIsMSE) {
644 0 : return StateObject::HandleSeek(aTarget);
645 : }
646 : // Delay seek request until decoding first frames for non-MSE media.
647 0 : SLOG("Not Enough Data to seek at this stage, queuing seek");
648 0 : mPendingSeek.RejectIfExists(__func__);
649 0 : mPendingSeek.mTarget.emplace(aTarget);
650 0 : return mPendingSeek.mPromise.Ensure(__func__);
651 : }
652 :
653 : private:
654 : // Notify FirstFrameLoaded if having decoded first frames and
655 : // transition to SEEKING if there is any pending seek, or DECODING otherwise.
656 : void MaybeFinishDecodeFirstFrame();
657 :
658 : SeekJob mPendingSeek;
659 : };
660 :
661 : /**
662 : * Purpose: decode audio/video data for playback.
663 : *
664 : * Transition to:
665 : * DORMANT if playback is paused for a while.
666 : * SEEKING if any seek request.
667 : * SHUTDOWN if any decode error.
668 : * BUFFERING if playback can't continue due to lack of decoded data.
669 : * COMPLETED when having decoded all audio/video data.
670 : */
671 0 : class MediaDecoderStateMachine::DecodingState
672 : : public MediaDecoderStateMachine::StateObject
673 : {
674 : public:
675 0 : explicit DecodingState(Master* aPtr)
676 0 : : StateObject(aPtr)
677 0 : , mDormantTimer(OwnerThread())
678 : {
679 0 : }
680 :
681 : void Enter();
682 :
683 0 : void Exit() override
684 : {
685 0 : if (!mDecodeStartTime.IsNull()) {
686 0 : TimeDuration decodeDuration = TimeStamp::Now() - mDecodeStartTime;
687 0 : SLOG("Exiting DECODING, decoded for %.3lfs", decodeDuration.ToSeconds());
688 : }
689 0 : mDormantTimer.Reset();
690 0 : mOnAudioPopped.DisconnectIfExists();
691 0 : mOnVideoPopped.DisconnectIfExists();
692 0 : }
693 :
694 0 : void Step() override
695 : {
696 0 : if (mMaster->mPlayState != MediaDecoder::PLAY_STATE_PLAYING
697 0 : && mMaster->IsPlaying()) {
698 : // We're playing, but the element/decoder is in paused state. Stop
699 : // playing!
700 0 : mMaster->StopPlayback();
701 : }
702 :
703 : // Start playback if necessary so that the clock can be properly queried.
704 0 : if (!mIsPrerolling) {
705 0 : mMaster->MaybeStartPlayback();
706 : }
707 :
708 0 : mMaster->UpdatePlaybackPositionPeriodically();
709 :
710 0 : MOZ_ASSERT(!mMaster->IsPlaying()
711 : || mMaster->IsStateMachineScheduled(),
712 : "Must have timer scheduled");
713 :
714 0 : MaybeStartBuffering();
715 0 : }
716 :
717 0 : State GetState() const override
718 : {
719 0 : return DECODER_STATE_DECODING;
720 : }
721 :
722 0 : void HandleAudioDecoded(AudioData* aAudio) override
723 : {
724 0 : mMaster->PushAudio(aAudio);
725 0 : DispatchDecodeTasksIfNeeded();
726 0 : MaybeStopPrerolling();
727 0 : }
728 :
729 0 : void HandleVideoDecoded(VideoData* aVideo, TimeStamp aDecodeStart) override
730 : {
731 0 : mMaster->PushVideo(aVideo);
732 0 : DispatchDecodeTasksIfNeeded();
733 0 : MaybeStopPrerolling();
734 0 : CheckSlowDecoding(aDecodeStart);
735 0 : }
736 :
737 0 : void HandleAudioCanceled() override
738 : {
739 0 : mMaster->RequestAudioData();
740 0 : }
741 :
742 0 : void HandleVideoCanceled() override
743 : {
744 0 : mMaster->RequestVideoData(mMaster->GetMediaTime());
745 0 : }
746 :
747 : void HandleEndOfAudio() override;
748 : void HandleEndOfVideo() override;
749 :
750 0 : void HandleWaitingForAudio() override
751 : {
752 0 : mMaster->WaitForData(MediaData::AUDIO_DATA);
753 0 : MaybeStopPrerolling();
754 0 : }
755 :
756 0 : void HandleWaitingForVideo() override
757 : {
758 0 : mMaster->WaitForData(MediaData::VIDEO_DATA);
759 0 : MaybeStopPrerolling();
760 0 : }
761 :
762 0 : void HandleAudioWaited(MediaData::Type aType) override
763 : {
764 0 : mMaster->RequestAudioData();
765 0 : }
766 :
767 0 : void HandleVideoWaited(MediaData::Type aType) override
768 : {
769 0 : mMaster->RequestVideoData(mMaster->GetMediaTime());
770 0 : }
771 :
772 0 : void HandleAudioCaptured() override
773 : {
774 0 : MaybeStopPrerolling();
775 : // MediaSink is changed. Schedule Step() to check if we can start playback.
776 0 : mMaster->ScheduleStateMachine();
777 0 : }
778 :
779 0 : void HandleVideoSuspendTimeout() override
780 : {
781 : // No video, so nothing to suspend.
782 0 : if (!mMaster->HasVideo()) {
783 0 : return;
784 : }
785 :
786 0 : mMaster->mVideoDecodeSuspended = true;
787 0 : mMaster->mOnPlaybackEvent.Notify(MediaEventType::EnterVideoSuspend);
788 0 : Reader()->SetVideoBlankDecode(true);
789 : }
790 :
791 0 : void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override
792 : {
793 0 : if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) {
794 : // Schedule Step() to check if we can start playback.
795 0 : mMaster->ScheduleStateMachine();
796 : // Try to dispatch decoding tasks for mMinimizePreroll might be reset.
797 0 : DispatchDecodeTasksIfNeeded();
798 : }
799 :
800 0 : if (aPlayState == MediaDecoder::PLAY_STATE_PAUSED) {
801 0 : StartDormantTimer();
802 : } else {
803 0 : mDormantTimer.Reset();
804 : }
805 0 : }
806 :
807 0 : nsCString GetDebugInfo() override
808 : {
809 0 : return nsPrintfCString("mIsPrerolling=%d", mIsPrerolling);
810 : }
811 :
812 : private:
813 : void DispatchDecodeTasksIfNeeded();
814 : void EnsureAudioDecodeTaskQueued();
815 : void EnsureVideoDecodeTaskQueued();
816 : void MaybeStartBuffering();
817 :
818 0 : void CheckSlowDecoding(TimeStamp aDecodeStart)
819 : {
820 : // For non async readers, if the requested video sample was slow to
821 : // arrive, increase the amount of audio we buffer to ensure that we
822 : // don't run out of audio. This is unnecessary for async readers,
823 : // since they decode audio and video on different threads so they
824 : // are unlikely to run out of decoded audio.
825 0 : if (Reader()->IsAsync()) {
826 0 : return;
827 : }
828 :
829 0 : TimeDuration decodeTime = TimeStamp::Now() - aDecodeStart;
830 0 : auto adjusted = TimeUnit::FromTimeDuration(decodeTime * THRESHOLD_FACTOR);
831 0 : if (adjusted > mMaster->mLowAudioThreshold
832 0 : && !mMaster->HasLowBufferedData())
833 : {
834 0 : mMaster->mLowAudioThreshold = std::min(
835 0 : adjusted, mMaster->mAmpleAudioThreshold);
836 :
837 0 : mMaster->mAmpleAudioThreshold = std::max(
838 0 : mMaster->mLowAudioThreshold * THRESHOLD_FACTOR,
839 0 : mMaster->mAmpleAudioThreshold);
840 :
841 0 : SLOG("Slow video decode, set "
842 : "mLowAudioThreshold=%" PRId64
843 : " mAmpleAudioThreshold=%" PRId64,
844 : mMaster->mLowAudioThreshold.ToMicroseconds(),
845 : mMaster->mAmpleAudioThreshold.ToMicroseconds());
846 : }
847 : }
848 :
849 : // At the start of decoding we want to "preroll" the decode until we've
850 : // got a few frames decoded before we consider whether decode is falling
851 : // behind. Otherwise our "we're falling behind" logic will trigger
852 : // unnecessarily if we start playing as soon as the first sample is
853 : // decoded. These two fields store how many video frames and audio
854 : // samples we must consume before are considered to be finished prerolling.
855 0 : TimeUnit AudioPrerollThreshold() const
856 : {
857 0 : return mMaster->mAmpleAudioThreshold / 2;
858 : }
859 :
860 0 : uint32_t VideoPrerollFrames() const
861 : {
862 0 : return mMaster->GetAmpleVideoFrames() / 2;
863 : }
864 :
865 0 : bool DonePrerollingAudio()
866 : {
867 0 : return !mMaster->IsAudioDecoding()
868 0 : || mMaster->GetDecodedAudioDuration()
869 0 : >= AudioPrerollThreshold().MultDouble(mMaster->mPlaybackRate);
870 : }
871 :
872 0 : bool DonePrerollingVideo()
873 : {
874 0 : return !mMaster->IsVideoDecoding()
875 0 : || static_cast<uint32_t>(mMaster->VideoQueue().GetSize())
876 0 : >= VideoPrerollFrames() * mMaster->mPlaybackRate + 1;
877 : }
878 :
879 0 : void MaybeStopPrerolling()
880 : {
881 0 : if (mIsPrerolling
882 0 : && (DonePrerollingAudio() || mMaster->IsWaitingAudioData())
883 0 : && (DonePrerollingVideo() || mMaster->IsWaitingVideoData())) {
884 0 : mIsPrerolling = false;
885 : // Check if we can start playback.
886 0 : mMaster->ScheduleStateMachine();
887 : }
888 0 : }
889 :
890 0 : void StartDormantTimer()
891 : {
892 0 : if (!mMaster->mMediaSeekable) {
893 : // Don't enter dormant if the media is not seekable because we need to
894 : // seek when exiting dormant.
895 0 : return;
896 : }
897 :
898 0 : auto timeout = MediaPrefs::DormantOnPauseTimeout();
899 0 : if (timeout < 0) {
900 : // Disabled when timeout is negative.
901 0 : return;
902 0 : } else if (timeout == 0) {
903 : // Enter dormant immediately without scheduling a timer.
904 0 : SetState<DormantState>();
905 0 : return;
906 : }
907 :
908 0 : if (mMaster->mMinimizePreroll) {
909 0 : SetState<DormantState>();
910 0 : return;
911 : }
912 :
913 0 : TimeStamp target = TimeStamp::Now() +
914 0 : TimeDuration::FromMilliseconds(timeout);
915 :
916 0 : mDormantTimer.Ensure(target,
917 0 : [this] () {
918 0 : mDormantTimer.CompleteRequest();
919 0 : SetState<DormantState>();
920 0 : }, [this] () {
921 0 : mDormantTimer.CompleteRequest();
922 0 : });
923 : }
924 :
925 : // Time at which we started decoding.
926 : TimeStamp mDecodeStartTime;
927 :
928 : // When we start decoding (either for the first time, or after a pause)
929 : // we may be low on decoded data. We don't want our "low data" logic to
930 : // kick in and decide that we're low on decoded data because the download
931 : // can't keep up with the decode, and cause us to pause playback. So we
932 : // have a "preroll" stage, where we ignore the results of our "low data"
933 : // logic during the first few frames of our decode. This occurs during
934 : // playback.
935 : bool mIsPrerolling = true;
936 :
937 : // Fired when playback is paused for a while to enter dormant.
938 : DelayedScheduler mDormantTimer;
939 :
940 : MediaEventListener mOnAudioPopped;
941 : MediaEventListener mOnVideoPopped;
942 : };
943 :
944 : /**
945 : * Purpose: seek to a particular new playback position.
946 : *
947 : * Transition to:
948 : * SEEKING if any new seek request.
949 : * SHUTDOWN if seek failed.
950 : * COMPLETED if the new playback position is the end of the media resource.
951 : * NextFrameSeekingState if completing a NextFrameSeekingFromDormantState.
952 : * DECODING otherwise.
953 : */
954 0 : class MediaDecoderStateMachine::SeekingState
955 : : public MediaDecoderStateMachine::StateObject
956 : {
957 : public:
958 0 : explicit SeekingState(Master* aPtr) : StateObject(aPtr) { }
959 :
960 0 : RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aSeekJob,
961 : EventVisibility aVisibility)
962 : {
963 0 : mSeekJob = Move(aSeekJob);
964 0 : mVisibility = aVisibility;
965 :
966 : // Always switch off the blank decoder otherwise we might become visible
967 : // in the middle of seeking and won't have a valid video frame to show
968 : // when seek is done.
969 0 : if (mMaster->mVideoDecodeSuspended) {
970 0 : mMaster->mVideoDecodeSuspended = false;
971 0 : mMaster->mOnPlaybackEvent.Notify(MediaEventType::ExitVideoSuspend);
972 0 : Reader()->SetVideoBlankDecode(false);
973 : }
974 :
975 : // Suppressed visibility comes from two cases: (1) leaving dormant state,
976 : // and (2) resuming suspended video decoder. We want both cases to be
977 : // transparent to the user. So we only notify the change when the seek
978 : // request is from the user.
979 0 : if (mVisibility == EventVisibility::Observable) {
980 : // Don't stop playback for a video-only seek since we want to keep playing
981 : // audio and we don't need to stop playback while leaving dormant for the
982 : // playback should has been stopped.
983 0 : mMaster->StopPlayback();
984 0 : mMaster->UpdatePlaybackPositionInternal(mSeekJob.mTarget->GetTime());
985 0 : mMaster->mOnPlaybackEvent.Notify(MediaEventType::SeekStarted);
986 0 : mMaster->UpdateNextFrameStatus(
987 0 : MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING);
988 : }
989 :
990 0 : RefPtr<MediaDecoder::SeekPromise> p = mSeekJob.mPromise.Ensure(__func__);
991 :
992 0 : DoSeek();
993 :
994 0 : return p;
995 : }
996 :
997 : virtual void Exit() override = 0;
998 :
999 0 : State GetState() const override
1000 : {
1001 0 : return DECODER_STATE_SEEKING;
1002 : }
1003 :
1004 : void HandleAudioDecoded(AudioData* aAudio) override = 0;
1005 : void HandleVideoDecoded(VideoData* aVideo,
1006 : TimeStamp aDecodeStart) override = 0;
1007 : void HandleAudioWaited(MediaData::Type aType) override = 0;
1008 : void HandleVideoWaited(MediaData::Type aType) override = 0;
1009 :
1010 0 : void HandleVideoSuspendTimeout() override
1011 : {
1012 : // Do nothing since we want a valid video frame to show when seek is done.
1013 0 : }
1014 :
1015 0 : void HandleResumeVideoDecoding(const TimeUnit&) override
1016 : {
1017 : // We set mVideoDecodeSuspended to false in Enter().
1018 0 : MOZ_ASSERT(false, "Shouldn't have suspended video decoding.");
1019 : }
1020 :
1021 : protected:
1022 : SeekJob mSeekJob;
1023 : EventVisibility mVisibility;
1024 :
1025 : virtual void DoSeek() = 0;
1026 : // Transition to the next state (defined by the subclass) when seek is completed.
1027 0 : virtual void GoToNextState() { SetState<DecodingState>(); }
1028 : void SeekCompleted();
1029 : virtual TimeUnit CalculateNewCurrentTime() const = 0;
1030 : };
1031 :
1032 0 : class MediaDecoderStateMachine::AccurateSeekingState
1033 : : public MediaDecoderStateMachine::SeekingState
1034 : {
1035 : public:
1036 0 : explicit AccurateSeekingState(Master* aPtr) : SeekingState(aPtr) { }
1037 :
1038 0 : RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aSeekJob,
1039 : EventVisibility aVisibility)
1040 : {
1041 0 : MOZ_ASSERT(aSeekJob.mTarget->IsAccurate() || aSeekJob.mTarget->IsFast());
1042 0 : mCurrentTimeBeforeSeek = mMaster->GetMediaTime();
1043 0 : return SeekingState::Enter(Move(aSeekJob), aVisibility);
1044 : }
1045 :
1046 0 : void Exit() override
1047 : {
1048 : // Disconnect MediaDecoder.
1049 0 : mSeekJob.RejectIfExists(__func__);
1050 :
1051 : // Disconnect MediaDecoderReaderWrapper.
1052 0 : mSeekRequest.DisconnectIfExists();
1053 :
1054 0 : mWaitRequest.DisconnectIfExists();
1055 0 : }
1056 :
1057 0 : void HandleAudioDecoded(AudioData* aAudio) override
1058 : {
1059 0 : MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking,
1060 : "Seek shouldn't be finished");
1061 0 : MOZ_ASSERT(aAudio);
1062 :
1063 0 : AdjustFastSeekIfNeeded(aAudio);
1064 :
1065 0 : if (mSeekJob.mTarget->IsFast()) {
1066 : // Non-precise seek; we can stop the seek at the first sample.
1067 0 : mMaster->PushAudio(aAudio);
1068 0 : mDoneAudioSeeking = true;
1069 : } else {
1070 0 : nsresult rv = DropAudioUpToSeekTarget(aAudio);
1071 0 : if (NS_FAILED(rv)) {
1072 0 : mMaster->DecodeError(rv);
1073 0 : return;
1074 : }
1075 : }
1076 :
1077 0 : if (!mDoneAudioSeeking) {
1078 0 : RequestAudioData();
1079 0 : return;
1080 : }
1081 0 : MaybeFinishSeek();
1082 : }
1083 :
1084 0 : void HandleVideoDecoded(VideoData* aVideo, TimeStamp aDecodeStart) override
1085 : {
1086 0 : MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking,
1087 : "Seek shouldn't be finished");
1088 0 : MOZ_ASSERT(aVideo);
1089 :
1090 0 : AdjustFastSeekIfNeeded(aVideo);
1091 :
1092 0 : if (mSeekJob.mTarget->IsFast()) {
1093 : // Non-precise seek. We can stop the seek at the first sample.
1094 0 : mMaster->PushVideo(aVideo);
1095 0 : mDoneVideoSeeking = true;
1096 : } else {
1097 0 : nsresult rv = DropVideoUpToSeekTarget(aVideo);
1098 0 : if (NS_FAILED(rv)) {
1099 0 : mMaster->DecodeError(rv);
1100 0 : return;
1101 : }
1102 : }
1103 :
1104 0 : if (!mDoneVideoSeeking) {
1105 0 : RequestVideoData();
1106 0 : return;
1107 : }
1108 0 : MaybeFinishSeek();
1109 : }
1110 :
1111 0 : void HandleWaitingForAudio() override
1112 : {
1113 0 : MOZ_ASSERT(!mDoneAudioSeeking);
1114 0 : mMaster->WaitForData(MediaData::AUDIO_DATA);
1115 0 : }
1116 :
1117 0 : void HandleAudioCanceled() override
1118 : {
1119 0 : MOZ_ASSERT(!mDoneAudioSeeking);
1120 0 : RequestAudioData();
1121 0 : }
1122 :
1123 0 : void HandleEndOfAudio() override
1124 : {
1125 0 : MOZ_ASSERT(!mDoneAudioSeeking);
1126 0 : AudioQueue().Finish();
1127 0 : mDoneAudioSeeking = true;
1128 0 : MaybeFinishSeek();
1129 0 : }
1130 :
1131 0 : void HandleWaitingForVideo() override
1132 : {
1133 0 : MOZ_ASSERT(!mDoneVideoSeeking);
1134 0 : mMaster->WaitForData(MediaData::VIDEO_DATA);
1135 0 : }
1136 :
1137 0 : void HandleVideoCanceled() override
1138 : {
1139 0 : MOZ_ASSERT(!mDoneVideoSeeking);
1140 0 : RequestVideoData();
1141 0 : }
1142 :
1143 0 : void HandleEndOfVideo() override
1144 : {
1145 0 : MOZ_ASSERT(!mDoneVideoSeeking);
1146 0 : if (mFirstVideoFrameAfterSeek) {
1147 : // Hit the end of stream. Move mFirstVideoFrameAfterSeek into
1148 : // mSeekedVideoData so we have something to display after seeking.
1149 0 : mMaster->PushVideo(mFirstVideoFrameAfterSeek);
1150 : }
1151 0 : VideoQueue().Finish();
1152 0 : mDoneVideoSeeking = true;
1153 0 : MaybeFinishSeek();
1154 0 : }
1155 :
1156 0 : void HandleAudioWaited(MediaData::Type aType) override
1157 : {
1158 0 : MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking,
1159 : "Seek shouldn't be finished");
1160 :
1161 0 : RequestAudioData();
1162 0 : }
1163 :
1164 0 : void HandleVideoWaited(MediaData::Type aType) override
1165 : {
1166 0 : MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking,
1167 : "Seek shouldn't be finished");
1168 :
1169 0 : RequestVideoData();
1170 0 : }
1171 :
1172 0 : void DoSeek() override
1173 : {
1174 0 : mDoneAudioSeeking = !Info().HasAudio();
1175 0 : mDoneVideoSeeking = !Info().HasVideo();
1176 :
1177 0 : mMaster->ResetDecode();
1178 0 : mMaster->StopMediaSink();
1179 :
1180 0 : DemuxerSeek();
1181 0 : }
1182 :
1183 0 : TimeUnit CalculateNewCurrentTime() const override
1184 : {
1185 0 : const auto seekTime = mSeekJob.mTarget->GetTime();
1186 :
1187 : // For the accurate seek, we always set the newCurrentTime = seekTime so
1188 : // that the updated HTMLMediaElement.currentTime will always be the seek
1189 : // target; we rely on the MediaSink to handles the gap between the
1190 : // newCurrentTime and the real decoded samples' start time.
1191 0 : if (mSeekJob.mTarget->IsAccurate()) {
1192 0 : return seekTime;
1193 : }
1194 :
1195 : // For the fast seek, we update the newCurrentTime with the decoded audio
1196 : // and video samples, set it to be the one which is closet to the seekTime.
1197 0 : if (mSeekJob.mTarget->IsFast()) {
1198 0 : RefPtr<AudioData> audio = AudioQueue().PeekFront();
1199 0 : RefPtr<VideoData> video = VideoQueue().PeekFront();
1200 :
1201 : // A situation that both audio and video approaches the end.
1202 0 : if (!audio && !video) {
1203 0 : return seekTime;
1204 : }
1205 :
1206 : const int64_t audioStart =
1207 0 : audio ? audio->mTime.ToMicroseconds() : INT64_MAX;
1208 : const int64_t videoStart =
1209 0 : video ? video->mTime.ToMicroseconds() : INT64_MAX;
1210 0 : const int64_t audioGap = std::abs(audioStart - seekTime.ToMicroseconds());
1211 0 : const int64_t videoGap = std::abs(videoStart - seekTime.ToMicroseconds());
1212 : return TimeUnit::FromMicroseconds(
1213 0 : audioGap <= videoGap ? audioStart : videoStart);
1214 : }
1215 :
1216 0 : MOZ_ASSERT(false, "AccurateSeekTask doesn't handle other seek types.");
1217 : return TimeUnit::Zero();
1218 : }
1219 :
1220 : protected:
1221 0 : void DemuxerSeek()
1222 : {
1223 : // Request the demuxer to perform seek.
1224 0 : Reader()->Seek(mSeekJob.mTarget.ref())
1225 0 : ->Then(OwnerThread(), __func__,
1226 0 : [this] (const media::TimeUnit& aUnit) {
1227 0 : OnSeekResolved(aUnit);
1228 0 : },
1229 0 : [this] (const SeekRejectValue& aReject) {
1230 0 : OnSeekRejected(aReject);
1231 0 : })
1232 0 : ->Track(mSeekRequest);
1233 0 : }
1234 :
1235 0 : void OnSeekResolved(media::TimeUnit)
1236 : {
1237 0 : mSeekRequest.Complete();
1238 :
1239 : // We must decode the first samples of active streams, so we can determine
1240 : // the new stream time. So dispatch tasks to do that.
1241 0 : if (!mDoneVideoSeeking) {
1242 0 : RequestVideoData();
1243 : }
1244 0 : if (!mDoneAudioSeeking) {
1245 0 : RequestAudioData();
1246 : }
1247 0 : }
1248 :
1249 0 : void OnSeekRejected(const SeekRejectValue& aReject)
1250 : {
1251 0 : mSeekRequest.Complete();
1252 :
1253 0 : if (aReject.mError == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
1254 0 : SLOG("OnSeekRejected reason=WAITING_FOR_DATA type=%d", aReject.mType);
1255 0 : MOZ_ASSERT_IF(aReject.mType == MediaData::AUDIO_DATA, !mMaster->IsRequestingAudioData());
1256 0 : MOZ_ASSERT_IF(aReject.mType == MediaData::VIDEO_DATA, !mMaster->IsRequestingVideoData());
1257 0 : MOZ_ASSERT_IF(aReject.mType == MediaData::AUDIO_DATA, !mMaster->IsWaitingAudioData());
1258 0 : MOZ_ASSERT_IF(aReject.mType == MediaData::VIDEO_DATA, !mMaster->IsWaitingVideoData());
1259 :
1260 : // Fire 'waiting' to notify the player that we are waiting for data.
1261 0 : mMaster->UpdateNextFrameStatus(
1262 0 : MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING);
1263 : Reader()
1264 0 : ->WaitForData(aReject.mType)
1265 0 : ->Then(OwnerThread(), __func__,
1266 0 : [this](MediaData::Type aType) {
1267 0 : SLOG("OnSeekRejected wait promise resolved");
1268 0 : mWaitRequest.Complete();
1269 0 : DemuxerSeek();
1270 0 : },
1271 0 : [this](const WaitForDataRejectValue& aRejection) {
1272 0 : SLOG("OnSeekRejected wait promise rejected");
1273 0 : mWaitRequest.Complete();
1274 0 : mMaster->DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA);
1275 0 : })
1276 0 : ->Track(mWaitRequest);
1277 0 : return;
1278 : }
1279 :
1280 0 : if (aReject.mError == NS_ERROR_DOM_MEDIA_END_OF_STREAM) {
1281 0 : HandleEndOfAudio();
1282 0 : HandleEndOfVideo();
1283 0 : return;
1284 : }
1285 :
1286 0 : MOZ_ASSERT(NS_FAILED(aReject.mError),
1287 : "Cancels should also disconnect mSeekRequest");
1288 0 : mMaster->DecodeError(aReject.mError);
1289 : }
1290 :
1291 0 : void RequestAudioData()
1292 : {
1293 0 : MOZ_ASSERT(!mDoneAudioSeeking);
1294 0 : mMaster->RequestAudioData();
1295 0 : }
1296 :
1297 0 : void RequestVideoData()
1298 : {
1299 0 : MOZ_ASSERT(!mDoneVideoSeeking);
1300 0 : mMaster->RequestVideoData(media::TimeUnit());
1301 0 : }
1302 :
1303 0 : void AdjustFastSeekIfNeeded(MediaData* aSample)
1304 : {
1305 0 : if (mSeekJob.mTarget->IsFast()
1306 0 : && mSeekJob.mTarget->GetTime() > mCurrentTimeBeforeSeek
1307 0 : && aSample->mTime < mCurrentTimeBeforeSeek) {
1308 : // We are doing a fastSeek, but we ended up *before* the previous
1309 : // playback position. This is surprising UX, so switch to an accurate
1310 : // seek and decode to the seek target. This is not conformant to the
1311 : // spec, fastSeek should always be fast, but until we get the time to
1312 : // change all Readers to seek to the keyframe after the currentTime
1313 : // in this case, we'll just decode forward. Bug 1026330.
1314 0 : mSeekJob.mTarget->SetType(SeekTarget::Accurate);
1315 : }
1316 0 : }
1317 :
1318 0 : nsresult DropAudioUpToSeekTarget(AudioData* aAudio)
1319 : {
1320 0 : MOZ_ASSERT(aAudio && mSeekJob.mTarget->IsAccurate());
1321 :
1322 : auto sampleDuration = FramesToTimeUnit(
1323 0 : aAudio->mFrames, Info().mAudio.mRate);
1324 0 : if (!sampleDuration.IsValid()) {
1325 0 : return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
1326 : }
1327 :
1328 0 : auto audioTime = aAudio->mTime;
1329 0 : if (audioTime + sampleDuration <= mSeekJob.mTarget->GetTime()) {
1330 : // Our seek target lies after the frames in this AudioData. Don't
1331 : // push it onto the audio queue, and keep decoding forwards.
1332 0 : return NS_OK;
1333 : }
1334 :
1335 0 : if (audioTime > mSeekJob.mTarget->GetTime()) {
1336 : // The seek target doesn't lie in the audio block just after the last
1337 : // audio frames we've seen which were before the seek target. This
1338 : // could have been the first audio data we've seen after seek, i.e. the
1339 : // seek terminated after the seek target in the audio stream. Just
1340 : // abort the audio decode-to-target, the state machine will play
1341 : // silence to cover the gap. Typically this happens in poorly muxed
1342 : // files.
1343 0 : SLOGW("Audio not synced after seek, maybe a poorly muxed file?");
1344 0 : mMaster->PushAudio(aAudio);
1345 0 : mDoneAudioSeeking = true;
1346 0 : return NS_OK;
1347 : }
1348 :
1349 : // The seek target lies somewhere in this AudioData's frames, strip off
1350 : // any frames which lie before the seek target, so we'll begin playback
1351 : // exactly at the seek target.
1352 0 : NS_ASSERTION(mSeekJob.mTarget->GetTime() >= audioTime,
1353 : "Target must at or be after data start.");
1354 0 : NS_ASSERTION(mSeekJob.mTarget->GetTime() < audioTime + sampleDuration,
1355 : "Data must end after target.");
1356 :
1357 : CheckedInt64 framesToPrune = TimeUnitToFrames(
1358 0 : mSeekJob.mTarget->GetTime() - audioTime, Info().mAudio.mRate);
1359 0 : if (!framesToPrune.isValid()) {
1360 0 : return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
1361 : }
1362 0 : if (framesToPrune.value() > aAudio->mFrames) {
1363 : // We've messed up somehow. Don't try to trim frames, the |frames|
1364 : // variable below will overflow.
1365 0 : SLOGW("Can't prune more frames that we have!");
1366 0 : return NS_ERROR_FAILURE;
1367 : }
1368 0 : uint32_t frames = aAudio->mFrames - uint32_t(framesToPrune.value());
1369 0 : uint32_t channels = aAudio->mChannels;
1370 0 : AlignedAudioBuffer audioData(frames * channels);
1371 0 : if (!audioData) {
1372 0 : return NS_ERROR_OUT_OF_MEMORY;
1373 : }
1374 :
1375 0 : memcpy(audioData.get(),
1376 0 : aAudio->mAudioData.get() + (framesToPrune.value() * channels),
1377 0 : frames * channels * sizeof(AudioDataValue));
1378 0 : auto duration = FramesToTimeUnit(frames, Info().mAudio.mRate);
1379 0 : if (!duration.IsValid()) {
1380 0 : return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
1381 : }
1382 : RefPtr<AudioData> data(new AudioData(
1383 0 : aAudio->mOffset, mSeekJob.mTarget->GetTime(),
1384 : duration, frames, Move(audioData), channels,
1385 0 : aAudio->mRate));
1386 0 : MOZ_ASSERT(AudioQueue().GetSize() == 0,
1387 : "Should be the 1st sample after seeking");
1388 0 : mMaster->PushAudio(data);
1389 0 : mDoneAudioSeeking = true;
1390 :
1391 0 : return NS_OK;
1392 : }
1393 :
1394 0 : nsresult DropVideoUpToSeekTarget(VideoData* aVideo)
1395 : {
1396 0 : MOZ_ASSERT(aVideo);
1397 0 : SLOG("DropVideoUpToSeekTarget() frame [%" PRId64 ", %" PRId64 "]",
1398 : aVideo->mTime.ToMicroseconds(), aVideo->GetEndTime().ToMicroseconds());
1399 0 : const auto target = mSeekJob.mTarget->GetTime();
1400 :
1401 : // If the frame end time is less than the seek target, we won't want
1402 : // to display this frame after the seek, so discard it.
1403 0 : if (target >= aVideo->GetEndTime()) {
1404 0 : SLOG("DropVideoUpToSeekTarget() pop video frame [%" PRId64 ", %" PRId64
1405 : "] target=%" PRId64,
1406 : aVideo->mTime.ToMicroseconds(),
1407 : aVideo->GetEndTime().ToMicroseconds(),
1408 : target.ToMicroseconds());
1409 0 : mFirstVideoFrameAfterSeek = aVideo;
1410 : } else {
1411 0 : if (target >= aVideo->mTime &&
1412 0 : aVideo->GetEndTime() >= target) {
1413 : // The seek target lies inside this frame's time slice. Adjust the
1414 : // frame's start time to match the seek target.
1415 0 : aVideo->UpdateTimestamp(target);
1416 : }
1417 0 : mFirstVideoFrameAfterSeek = nullptr;
1418 :
1419 0 : SLOG("DropVideoUpToSeekTarget() found video frame [%" PRId64 ", %" PRId64
1420 : "] containing target=%" PRId64,
1421 : aVideo->mTime.ToMicroseconds(),
1422 : aVideo->GetEndTime().ToMicroseconds(),
1423 : target.ToMicroseconds());
1424 :
1425 0 : MOZ_ASSERT(VideoQueue().GetSize() == 0,
1426 : "Should be the 1st sample after seeking");
1427 0 : mMaster->PushVideo(aVideo);
1428 0 : mDoneVideoSeeking = true;
1429 : }
1430 :
1431 0 : return NS_OK;
1432 : }
1433 :
1434 0 : void MaybeFinishSeek()
1435 : {
1436 0 : if (mDoneAudioSeeking && mDoneVideoSeeking) {
1437 0 : SeekCompleted();
1438 : }
1439 0 : }
1440 :
1441 : /*
1442 : * Track the current seek promise made by the reader.
1443 : */
1444 : MozPromiseRequestHolder<MediaDecoderReader::SeekPromise> mSeekRequest;
1445 :
1446 : /*
1447 : * Internal state.
1448 : */
1449 : media::TimeUnit mCurrentTimeBeforeSeek;
1450 : bool mDoneAudioSeeking = false;
1451 : bool mDoneVideoSeeking = false;
1452 : MozPromiseRequestHolder<WaitForDataPromise> mWaitRequest;
1453 :
1454 : // This temporarily stores the first frame we decode after we seek.
1455 : // This is so that if we hit end of stream while we're decoding to reach
1456 : // the seek target, we will still have a frame that we can display as the
1457 : // last frame in the media.
1458 : RefPtr<VideoData> mFirstVideoFrameAfterSeek;
1459 : };
1460 :
1461 : /*
1462 : * Remove samples from the queue until aCompare() returns false.
1463 : * aCompare A function object with the signature bool(int64_t) which returns
1464 : * true for samples that should be removed.
1465 : */
1466 : template <typename Type, typename Function>
1467 : static void
1468 0 : DiscardFrames(MediaQueue<Type>& aQueue, const Function& aCompare)
1469 : {
1470 0 : while(aQueue.GetSize() > 0) {
1471 0 : if (aCompare(aQueue.PeekFront()->mTime.ToMicroseconds())) {
1472 0 : RefPtr<Type> releaseMe = aQueue.PopFront();
1473 0 : continue;
1474 : }
1475 0 : break;
1476 : }
1477 0 : }
1478 :
1479 0 : class MediaDecoderStateMachine::NextFrameSeekingState
1480 : : public MediaDecoderStateMachine::SeekingState
1481 : {
1482 : public:
1483 0 : explicit NextFrameSeekingState(Master* aPtr) : SeekingState(aPtr) { }
1484 :
1485 0 : RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aSeekJob,
1486 : EventVisibility aVisibility)
1487 : {
1488 0 : MOZ_ASSERT(aSeekJob.mTarget->IsNextFrame());
1489 0 : mCurrentTime = mMaster->GetMediaTime();
1490 0 : mDuration = mMaster->Duration();
1491 0 : return SeekingState::Enter(Move(aSeekJob), aVisibility);
1492 : }
1493 :
1494 0 : void Exit() override
1495 : {
1496 : // Disconnect my async seek operation.
1497 0 : if (mAsyncSeekTask) { mAsyncSeekTask->Cancel(); }
1498 :
1499 : // Disconnect MediaDecoder.
1500 0 : mSeekJob.RejectIfExists(__func__);
1501 0 : }
1502 :
1503 0 : void HandleAudioDecoded(AudioData* aAudio) override
1504 : {
1505 0 : mMaster->PushAudio(aAudio);
1506 0 : }
1507 :
1508 0 : void HandleVideoDecoded(VideoData* aVideo, TimeStamp aDecodeStart) override
1509 : {
1510 0 : MOZ_ASSERT(aVideo);
1511 0 : MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
1512 0 : MOZ_ASSERT(NeedMoreVideo());
1513 :
1514 0 : if (aVideo->mTime > mCurrentTime) {
1515 0 : mMaster->PushVideo(aVideo);
1516 0 : FinishSeek();
1517 : } else {
1518 0 : RequestVideoData();
1519 : }
1520 0 : }
1521 :
1522 0 : void HandleWaitingForAudio() override
1523 : {
1524 0 : MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
1525 : // We don't care about audio decode errors in this state which will be
1526 : // handled by other states after seeking.
1527 0 : }
1528 :
1529 0 : void HandleAudioCanceled() override
1530 : {
1531 0 : MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
1532 : // We don't care about audio decode errors in this state which will be
1533 : // handled by other states after seeking.
1534 0 : }
1535 :
1536 0 : void HandleEndOfAudio() override
1537 : {
1538 0 : MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
1539 : // We don't care about audio decode errors in this state which will be
1540 : // handled by other states after seeking.
1541 0 : }
1542 :
1543 0 : void HandleWaitingForVideo() override
1544 : {
1545 0 : MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
1546 0 : MOZ_ASSERT(NeedMoreVideo());
1547 0 : mMaster->WaitForData(MediaData::VIDEO_DATA);
1548 0 : }
1549 :
1550 0 : void HandleVideoCanceled() override
1551 : {
1552 0 : MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
1553 0 : MOZ_ASSERT(NeedMoreVideo());
1554 0 : RequestVideoData();
1555 0 : }
1556 :
1557 0 : void HandleEndOfVideo() override
1558 : {
1559 0 : MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
1560 0 : MOZ_ASSERT(NeedMoreVideo());
1561 0 : VideoQueue().Finish();
1562 0 : FinishSeek();
1563 0 : }
1564 :
1565 0 : void HandleAudioWaited(MediaData::Type aType) override
1566 : {
1567 : // We don't care about audio in this state.
1568 0 : }
1569 :
1570 0 : void HandleVideoWaited(MediaData::Type aType) override
1571 : {
1572 0 : MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
1573 0 : MOZ_ASSERT(NeedMoreVideo());
1574 0 : RequestVideoData();
1575 0 : }
1576 :
1577 0 : TimeUnit CalculateNewCurrentTime() const override
1578 : {
1579 : // The HTMLMediaElement.currentTime should be updated to the seek target
1580 : // which has been updated to the next frame's time.
1581 0 : return mSeekJob.mTarget->GetTime();
1582 : }
1583 :
1584 0 : void DoSeek() override
1585 : {
1586 0 : auto currentTime = mCurrentTime;
1587 0 : DiscardFrames(VideoQueue(), [currentTime] (int64_t aSampleTime) {
1588 0 : return aSampleTime <= currentTime.ToMicroseconds();
1589 0 : });
1590 :
1591 : // If there is a pending video request, finish the seeking if we don't need
1592 : // more data, or wait for HandleVideoDecoded() to finish seeking.
1593 0 : if (mMaster->IsRequestingVideoData()) {
1594 0 : if (!NeedMoreVideo()) {
1595 0 : FinishSeek();
1596 : }
1597 0 : return;
1598 : }
1599 :
1600 : // Otherwise, we need to do the seek operation asynchronously for a special
1601 : // case (bug504613.ogv) which has no data at all, the 1st seekToNextFrame()
1602 : // operation reaches the end of the media. If we did the seek operation
1603 : // synchronously, we immediately resolve the SeekPromise in mSeekJob and
1604 : // then switch to the CompletedState which dispatches an "ended" event.
1605 : // However, the ThenValue of the SeekPromise has not yet been set, so the
1606 : // promise resolving is postponed and then the JS developer receives the
1607 : // "ended" event before the seek promise is resolved.
1608 : // An asynchronous seek operation helps to solve this issue since while the
1609 : // seek is actually performed, the ThenValue of SeekPromise has already
1610 : // been set so that it won't be postponed.
1611 0 : RefPtr<Runnable> r = mAsyncSeekTask = new AysncNextFrameSeekTask(this);
1612 0 : OwnerThread()->Dispatch(r.forget());
1613 : }
1614 :
1615 : private:
1616 0 : void DoSeekInternal()
1617 : {
1618 : // We don't need to discard frames to the mCurrentTime here because we have
1619 : // done it at DoSeek() and any video data received in between either
1620 : // finishes the seek operation or be discarded, see HandleVideoDecoded().
1621 :
1622 0 : if (!NeedMoreVideo()) {
1623 0 : FinishSeek();
1624 0 : } else if (!mMaster->IsRequestingVideoData()
1625 0 : && !mMaster->IsWaitingVideoData()) {
1626 0 : RequestVideoData();
1627 : }
1628 0 : }
1629 :
1630 0 : class AysncNextFrameSeekTask : public Runnable
1631 : {
1632 : public:
1633 0 : explicit AysncNextFrameSeekTask(NextFrameSeekingState* aStateObject)
1634 0 : : Runnable("MediaDecoderStateMachine::NextFrameSeekingState::"
1635 : "AysncNextFrameSeekTask")
1636 0 : , mStateObj(aStateObject)
1637 : {
1638 0 : }
1639 :
1640 0 : void Cancel() { mStateObj = nullptr; }
1641 :
1642 0 : NS_IMETHOD Run() override
1643 : {
1644 0 : if (mStateObj) {
1645 0 : mStateObj->DoSeekInternal();
1646 : }
1647 0 : return NS_OK;
1648 : }
1649 :
1650 : private:
1651 : NextFrameSeekingState* mStateObj;
1652 : };
1653 :
1654 0 : void RequestVideoData()
1655 : {
1656 0 : mMaster->RequestVideoData(media::TimeUnit());
1657 0 : }
1658 :
1659 0 : bool NeedMoreVideo() const
1660 : {
1661 : // Need to request video when we have none and video queue is not finished.
1662 0 : return VideoQueue().GetSize() == 0
1663 0 : && !VideoQueue().IsFinished();
1664 : }
1665 :
1666 : // Update the seek target's time before resolving this seek task, the updated
1667 : // time will be used in the MDSM::SeekCompleted() to update the MDSM's
1668 : // position.
1669 0 : void UpdateSeekTargetTime()
1670 : {
1671 0 : RefPtr<VideoData> data = VideoQueue().PeekFront();
1672 0 : if (data) {
1673 0 : mSeekJob.mTarget->SetTime(data->mTime);
1674 : } else {
1675 0 : MOZ_ASSERT(VideoQueue().AtEndOfStream());
1676 0 : mSeekJob.mTarget->SetTime(mDuration);
1677 : }
1678 0 : }
1679 :
1680 0 : void FinishSeek()
1681 : {
1682 0 : MOZ_ASSERT(!NeedMoreVideo());
1683 0 : UpdateSeekTargetTime();
1684 0 : auto time = mSeekJob.mTarget->GetTime().ToMicroseconds();
1685 0 : DiscardFrames(AudioQueue(), [time] (int64_t aSampleTime) {
1686 : return aSampleTime < time;
1687 0 : });
1688 0 : SeekCompleted();
1689 0 : }
1690 :
1691 : /*
1692 : * Internal state.
1693 : */
1694 : TimeUnit mCurrentTime;
1695 : TimeUnit mDuration;
1696 : RefPtr<AysncNextFrameSeekTask> mAsyncSeekTask;
1697 : };
1698 :
1699 0 : class MediaDecoderStateMachine::NextFrameSeekingFromDormantState
1700 : : public MediaDecoderStateMachine::AccurateSeekingState
1701 : {
1702 : public:
1703 0 : explicit NextFrameSeekingFromDormantState(Master* aPtr)
1704 0 : : AccurateSeekingState(aPtr)
1705 : {
1706 0 : }
1707 :
1708 0 : RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aCurrentSeekJob,
1709 : SeekJob&& aFutureSeekJob)
1710 : {
1711 0 : mFutureSeekJob = Move(aFutureSeekJob);
1712 :
1713 0 : AccurateSeekingState::Enter(Move(aCurrentSeekJob),
1714 0 : EventVisibility::Suppressed);
1715 :
1716 0 : return mFutureSeekJob.mPromise.Ensure(__func__);
1717 : }
1718 :
1719 0 : void Exit() override
1720 : {
1721 0 : mFutureSeekJob.RejectIfExists(__func__);
1722 0 : AccurateSeekingState::Exit();
1723 0 : }
1724 :
1725 : private:
1726 : SeekJob mFutureSeekJob;
1727 :
1728 : // We don't want to transition to DecodingState once this seek completes,
1729 : // instead, we transition to NextFrameSeekingState.
1730 0 : void GoToNextState() override
1731 : {
1732 0 : SetState<NextFrameSeekingState>(Move(mFutureSeekJob),
1733 0 : EventVisibility::Observable);
1734 0 : }
1735 : };
1736 :
1737 0 : class MediaDecoderStateMachine::VideoOnlySeekingState
1738 : : public MediaDecoderStateMachine::AccurateSeekingState
1739 : {
1740 : public:
1741 0 : explicit VideoOnlySeekingState(Master* aPtr) : AccurateSeekingState(aPtr) { }
1742 :
1743 0 : RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aSeekJob,
1744 : EventVisibility aVisibility)
1745 : {
1746 0 : MOZ_ASSERT(aSeekJob.mTarget->IsVideoOnly());
1747 0 : MOZ_ASSERT(aVisibility == EventVisibility::Suppressed);
1748 :
1749 : RefPtr<MediaDecoder::SeekPromise> p =
1750 0 : AccurateSeekingState::Enter(Move(aSeekJob), aVisibility);
1751 :
1752 : // Dispatch a mozvideoonlyseekbegin event to indicate UI for corresponding
1753 : // changes.
1754 0 : mMaster->mOnPlaybackEvent.Notify(MediaEventType::VideoOnlySeekBegin);
1755 :
1756 0 : return p.forget();
1757 : }
1758 :
1759 0 : void Exit() override
1760 : {
1761 : // We are completing or discarding this video-only seek operation now,
1762 : // dispatch an event so that the UI can change in response to the end
1763 : // of video-only seek.
1764 0 : mMaster->mOnPlaybackEvent.Notify(MediaEventType::VideoOnlySeekCompleted);
1765 :
1766 0 : AccurateSeekingState::Exit();
1767 0 : }
1768 :
1769 0 : void HandleAudioDecoded(AudioData* aAudio) override
1770 : {
1771 0 : MOZ_ASSERT(mDoneAudioSeeking && !mDoneVideoSeeking,
1772 : "Seek shouldn't be finished");
1773 0 : MOZ_ASSERT(aAudio);
1774 :
1775 : // Video-only seek doesn't reset audio decoder. There might be pending audio
1776 : // requests when AccurateSeekTask::Seek() begins. We will just store the
1777 : // data without checking |mDiscontinuity| or calling
1778 : // DropAudioUpToSeekTarget().
1779 0 : mMaster->PushAudio(aAudio);
1780 0 : }
1781 :
1782 0 : void HandleWaitingForAudio() override { }
1783 :
1784 0 : void HandleAudioCanceled() override { }
1785 :
1786 0 : void HandleEndOfAudio() override { }
1787 :
1788 0 : void HandleAudioWaited(MediaData::Type aType) override
1789 : {
1790 0 : MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking,
1791 : "Seek shouldn't be finished");
1792 :
1793 : // Ignore pending requests from video-only seek.
1794 0 : }
1795 :
1796 0 : void DoSeek() override
1797 : {
1798 : // TODO: keep decoding audio.
1799 0 : mDoneAudioSeeking = true;
1800 0 : mDoneVideoSeeking = !Info().HasVideo();
1801 :
1802 0 : mMaster->ResetDecode(TrackInfo::kVideoTrack);
1803 :
1804 0 : DemuxerSeek();
1805 0 : }
1806 : };
1807 :
1808 : RefPtr<MediaDecoder::SeekPromise>
1809 0 : MediaDecoderStateMachine::DormantState::HandleSeek(SeekTarget aTarget)
1810 : {
1811 0 : if (aTarget.IsNextFrame()) {
1812 : // NextFrameSeekingState doesn't reset the decoder unlike
1813 : // AccurateSeekingState. So we first must come out of dormant by seeking to
1814 : // mPendingSeek and continue later with the NextFrameSeek
1815 0 : SLOG("Changed state to SEEKING (to %" PRId64 ")",
1816 : aTarget.GetTime().ToMicroseconds());
1817 0 : SeekJob seekJob;
1818 0 : seekJob.mTarget = Some(aTarget);
1819 : return StateObject::SetState<NextFrameSeekingFromDormantState>(
1820 0 : Move(mPendingSeek), Move(seekJob));
1821 : }
1822 :
1823 0 : return StateObject::HandleSeek(aTarget);
1824 : }
1825 :
1826 : /**
1827 : * Purpose: stop playback until enough data is decoded to continue playback.
1828 : *
1829 : * Transition to:
1830 : * SEEKING if any seek request.
1831 : * SHUTDOWN if any decode error.
1832 : * COMPLETED when having decoded all audio/video data.
1833 : * DECODING when having decoded enough data to continue playback.
1834 : */
1835 0 : class MediaDecoderStateMachine::BufferingState
1836 : : public MediaDecoderStateMachine::StateObject
1837 : {
1838 : public:
1839 0 : explicit BufferingState(Master* aPtr) : StateObject(aPtr) { }
1840 :
1841 0 : void Enter()
1842 : {
1843 0 : if (mMaster->IsPlaying()) {
1844 0 : mMaster->StopPlayback();
1845 : }
1846 :
1847 0 : mBufferingStart = TimeStamp::Now();
1848 :
1849 0 : MediaStatistics stats = mMaster->GetStatistics();
1850 0 : SLOG("Playback rate: %.1lfKB/s%s download rate: %.1lfKB/s%s",
1851 : stats.mPlaybackRate / 1024,
1852 : stats.mPlaybackRateReliable ? "" : " (unreliable)",
1853 : stats.mDownloadRate / 1024,
1854 : stats.mDownloadRateReliable ? "" : " (unreliable)");
1855 :
1856 0 : mMaster->ScheduleStateMachineIn(TimeUnit::FromMicroseconds(USECS_PER_S));
1857 :
1858 0 : mMaster->UpdateNextFrameStatus(
1859 0 : MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING);
1860 0 : }
1861 :
1862 : void Step() override;
1863 :
1864 0 : State GetState() const override { return DECODER_STATE_BUFFERING; }
1865 :
1866 0 : void HandleAudioDecoded(AudioData* aAudio) override
1867 : {
1868 : // This might be the sample we need to exit buffering.
1869 : // Schedule Step() to check it.
1870 0 : mMaster->PushAudio(aAudio);
1871 0 : mMaster->ScheduleStateMachine();
1872 0 : }
1873 :
1874 0 : void HandleVideoDecoded(VideoData* aVideo, TimeStamp aDecodeStart) override
1875 : {
1876 : // This might be the sample we need to exit buffering.
1877 : // Schedule Step() to check it.
1878 0 : mMaster->PushVideo(aVideo);
1879 0 : mMaster->ScheduleStateMachine();
1880 0 : }
1881 :
1882 0 : void HandleAudioCanceled() override { mMaster->RequestAudioData(); }
1883 :
1884 0 : void HandleVideoCanceled() override
1885 : {
1886 0 : mMaster->RequestVideoData(media::TimeUnit());
1887 0 : }
1888 :
1889 0 : void HandleWaitingForAudio() override
1890 : {
1891 0 : mMaster->WaitForData(MediaData::AUDIO_DATA);
1892 0 : }
1893 :
1894 0 : void HandleWaitingForVideo() override
1895 : {
1896 0 : mMaster->WaitForData(MediaData::VIDEO_DATA);
1897 0 : }
1898 :
1899 0 : void HandleAudioWaited(MediaData::Type aType) override
1900 : {
1901 0 : mMaster->RequestAudioData();
1902 0 : }
1903 :
1904 0 : void HandleVideoWaited(MediaData::Type aType) override
1905 : {
1906 0 : mMaster->RequestVideoData(media::TimeUnit());
1907 0 : }
1908 :
1909 : void HandleEndOfAudio() override;
1910 : void HandleEndOfVideo() override;
1911 :
1912 0 : void HandleVideoSuspendTimeout() override
1913 : {
1914 : // No video, so nothing to suspend.
1915 0 : if (!mMaster->HasVideo()) {
1916 0 : return;
1917 : }
1918 :
1919 0 : mMaster->mVideoDecodeSuspended = true;
1920 0 : mMaster->mOnPlaybackEvent.Notify(MediaEventType::EnterVideoSuspend);
1921 0 : Reader()->SetVideoBlankDecode(true);
1922 : }
1923 :
1924 : private:
1925 : void DispatchDecodeTasksIfNeeded();
1926 :
1927 : TimeStamp mBufferingStart;
1928 :
1929 : // The maximum number of second we spend buffering when we are short on
1930 : // unbuffered data.
1931 : const uint32_t mBufferingWait = 15;
1932 : };
1933 :
1934 : /**
1935 : * Purpose: play all the decoded data and fire the 'ended' event.
1936 : *
1937 : * Transition to:
1938 : * SEEKING if any seek request.
1939 : */
1940 0 : class MediaDecoderStateMachine::CompletedState
1941 : : public MediaDecoderStateMachine::StateObject
1942 : {
1943 : public:
1944 0 : explicit CompletedState(Master* aPtr) : StateObject(aPtr) { }
1945 :
1946 0 : void Enter()
1947 : {
1948 : // TODO : use more approriate way to decide whether need to release
1949 : // resource in bug1367983.
1950 : #ifndef MOZ_WIDGET_ANDROID
1951 0 : if (!mMaster->mLooping) {
1952 : // We've decoded all samples.
1953 : // We don't need decoders anymore if not looping.
1954 0 : Reader()->ReleaseResources();
1955 : }
1956 : #endif
1957 0 : bool hasNextFrame = (!mMaster->HasAudio() || !mMaster->mAudioCompleted)
1958 0 : && (!mMaster->HasVideo() || !mMaster->mVideoCompleted);
1959 :
1960 0 : mMaster->UpdateNextFrameStatus(
1961 : hasNextFrame ? MediaDecoderOwner::NEXT_FRAME_AVAILABLE
1962 0 : : MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE);
1963 :
1964 0 : Step();
1965 0 : }
1966 :
1967 0 : void Exit() override
1968 : {
1969 0 : mSentPlaybackEndedEvent = false;
1970 0 : }
1971 :
1972 0 : void Step() override
1973 : {
1974 0 : if (mMaster->mPlayState != MediaDecoder::PLAY_STATE_PLAYING
1975 0 : && mMaster->IsPlaying()) {
1976 0 : mMaster->StopPlayback();
1977 : }
1978 :
1979 : // Play the remaining media. We want to run AdvanceFrame() at least
1980 : // once to ensure the current playback position is advanced to the
1981 : // end of the media, and so that we update the readyState.
1982 0 : if ((mMaster->HasVideo() && !mMaster->mVideoCompleted)
1983 0 : || (mMaster->HasAudio() && !mMaster->mAudioCompleted)) {
1984 : // Start playback if necessary to play the remaining media.
1985 0 : mMaster->MaybeStartPlayback();
1986 0 : mMaster->UpdatePlaybackPositionPeriodically();
1987 0 : MOZ_ASSERT(!mMaster->IsPlaying()
1988 : || mMaster->IsStateMachineScheduled(),
1989 : "Must have timer scheduled");
1990 0 : return;
1991 : }
1992 :
1993 : // StopPlayback in order to reset the IsPlaying() state so audio
1994 : // is restarted correctly.
1995 0 : mMaster->StopPlayback();
1996 :
1997 0 : if (!mSentPlaybackEndedEvent) {
1998 : auto clockTime =
1999 0 : std::max(mMaster->AudioEndTime(), mMaster->VideoEndTime());
2000 0 : clockTime = std::max(clockTime, mMaster->Duration());
2001 0 : mMaster->UpdatePlaybackPosition(clockTime);
2002 :
2003 : // Ensure readyState is updated before firing the 'ended' event.
2004 0 : mMaster->UpdateNextFrameStatus(MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE);
2005 :
2006 0 : mMaster->mOnPlaybackEvent.Notify(MediaEventType::PlaybackEnded);
2007 :
2008 0 : mSentPlaybackEndedEvent = true;
2009 :
2010 : // MediaSink::GetEndTime() must be called before stopping playback.
2011 0 : mMaster->StopMediaSink();
2012 : }
2013 : }
2014 :
2015 0 : State GetState() const override
2016 : {
2017 0 : return DECODER_STATE_COMPLETED;
2018 : }
2019 :
2020 0 : void HandleAudioCaptured() override
2021 : {
2022 : // MediaSink is changed. Schedule Step() to check if we can start playback.
2023 0 : mMaster->ScheduleStateMachine();
2024 0 : }
2025 :
2026 0 : void HandleVideoSuspendTimeout() override
2027 : {
2028 : // Do nothing since no decoding is going on.
2029 0 : }
2030 :
2031 0 : void HandleResumeVideoDecoding(const TimeUnit&) override
2032 : {
2033 : // Resume the video decoder and seek to the last video frame.
2034 : // This triggers a video-only seek which won't update the playback position.
2035 0 : StateObject::HandleResumeVideoDecoding(mMaster->mDecodedVideoEndTime);
2036 0 : }
2037 :
2038 0 : void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override
2039 : {
2040 0 : if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) {
2041 : // Schedule Step() to check if we can start playback.
2042 0 : mMaster->ScheduleStateMachine();
2043 : }
2044 0 : }
2045 :
2046 : private:
2047 : bool mSentPlaybackEndedEvent = false;
2048 : };
2049 :
2050 : /**
2051 : * Purpose: release all resources allocated by MDSM.
2052 : *
2053 : * Transition to:
2054 : * None since this is the final state.
2055 : *
2056 : * Transition from:
2057 : * Any states other than SHUTDOWN.
2058 : */
2059 0 : class MediaDecoderStateMachine::ShutdownState
2060 : : public MediaDecoderStateMachine::StateObject
2061 : {
2062 : public:
2063 0 : explicit ShutdownState(Master* aPtr) : StateObject(aPtr) { }
2064 :
2065 : RefPtr<ShutdownPromise> Enter();
2066 :
2067 0 : void Exit() override
2068 : {
2069 0 : MOZ_DIAGNOSTIC_ASSERT(false, "Shouldn't escape the SHUTDOWN state.");
2070 : }
2071 :
2072 0 : State GetState() const override
2073 : {
2074 0 : return DECODER_STATE_SHUTDOWN;
2075 : }
2076 :
2077 0 : RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget) override
2078 : {
2079 0 : MOZ_DIAGNOSTIC_ASSERT(false, "Can't seek in shutdown state.");
2080 : return MediaDecoder::SeekPromise::CreateAndReject(true, __func__);
2081 : }
2082 :
2083 0 : RefPtr<ShutdownPromise> HandleShutdown() override
2084 : {
2085 0 : MOZ_DIAGNOSTIC_ASSERT(false, "Already shutting down.");
2086 : return nullptr;
2087 : }
2088 :
2089 0 : void HandleVideoSuspendTimeout() override
2090 : {
2091 0 : MOZ_DIAGNOSTIC_ASSERT(false, "Already shutting down.");
2092 : }
2093 :
2094 0 : void HandleResumeVideoDecoding(const TimeUnit&) override
2095 : {
2096 0 : MOZ_DIAGNOSTIC_ASSERT(false, "Already shutting down.");
2097 : }
2098 : };
2099 :
2100 : RefPtr<MediaDecoder::SeekPromise>
2101 0 : MediaDecoderStateMachine::
2102 : StateObject::HandleSeek(SeekTarget aTarget)
2103 : {
2104 0 : SLOG("Changed state to SEEKING (to %" PRId64 ")", aTarget.GetTime().ToMicroseconds());
2105 0 : SeekJob seekJob;
2106 0 : seekJob.mTarget = Some(aTarget);
2107 0 : return SetSeekingState(Move(seekJob), EventVisibility::Observable);
2108 : }
2109 :
2110 : RefPtr<ShutdownPromise>
2111 0 : MediaDecoderStateMachine::
2112 : StateObject::HandleShutdown()
2113 : {
2114 0 : return SetState<ShutdownState>();
2115 : }
2116 :
2117 : static void
2118 0 : ReportRecoveryTelemetry(const TimeStamp& aRecoveryStart,
2119 : const MediaInfo& aMediaInfo,
2120 : bool aIsHardwareAccelerated)
2121 : {
2122 0 : MOZ_ASSERT(NS_IsMainThread());
2123 0 : if (!aMediaInfo.HasVideo()) {
2124 0 : return;
2125 : }
2126 :
2127 : // Keyed by audio+video or video alone, hardware acceleration,
2128 : // and by a resolution range.
2129 0 : nsCString key(aMediaInfo.HasAudio() ? "AV" : "V");
2130 0 : key.AppendASCII(aIsHardwareAccelerated ? "(hw)," : ",");
2131 : static const struct { int32_t mH; const char* mRes; } sResolutions[] = {
2132 : { 240, "0-240" },
2133 : { 480, "241-480" },
2134 : { 720, "481-720" },
2135 : { 1080, "721-1080" },
2136 : { 2160, "1081-2160" }
2137 : };
2138 0 : const char* resolution = "2161+";
2139 0 : int32_t height = aMediaInfo.mVideo.mImage.height;
2140 0 : for (const auto& res : sResolutions) {
2141 0 : if (height <= res.mH) {
2142 0 : resolution = res.mRes;
2143 0 : break;
2144 : }
2145 : }
2146 0 : key.AppendASCII(resolution);
2147 :
2148 0 : TimeDuration duration = TimeStamp::Now() - aRecoveryStart;
2149 0 : double duration_ms = duration.ToMilliseconds();
2150 0 : Telemetry::Accumulate(Telemetry::VIDEO_SUSPEND_RECOVERY_TIME_MS,
2151 : key,
2152 0 : uint32_t(duration_ms + 0.5));
2153 0 : Telemetry::Accumulate(Telemetry::VIDEO_SUSPEND_RECOVERY_TIME_MS,
2154 0 : NS_LITERAL_CSTRING("All"),
2155 0 : uint32_t(duration_ms + 0.5));
2156 : }
2157 :
2158 : void
2159 0 : MediaDecoderStateMachine::
2160 : StateObject::HandleResumeVideoDecoding(const TimeUnit& aTarget)
2161 : {
2162 0 : MOZ_ASSERT(mMaster->mVideoDecodeSuspended);
2163 :
2164 : // Start counting recovery time from right now.
2165 0 : TimeStamp start = TimeStamp::Now();
2166 :
2167 : // Local reference to mInfo, so that it will be copied in the lambda below.
2168 0 : auto& info = Info();
2169 0 : bool hw = Reader()->VideoIsHardwareAccelerated();
2170 :
2171 : // Start video-only seek to the current time.
2172 0 : SeekJob seekJob;
2173 :
2174 : // We use fastseek to optimize the resuming time.
2175 : // FastSeek is only used for video-only media since we don't need to worry
2176 : // about A/V sync.
2177 : // Don't use fastSeek if we want to seek to the end because it might seek to a
2178 : // keyframe before the last frame (if the last frame itself is not a keyframe)
2179 : // and we always want to present the final frame to the user when seeking to
2180 : // the end.
2181 0 : const auto type = mMaster->HasAudio() || aTarget == mMaster->Duration()
2182 0 : ? SeekTarget::Type::Accurate
2183 0 : : SeekTarget::Type::PrevSyncPoint;
2184 :
2185 0 : seekJob.mTarget.emplace(aTarget, type, true /* aVideoOnly */);
2186 :
2187 : // Hold mMaster->mAbstractMainThread here because this->mMaster will be
2188 : // invalid after the current state object is deleted in SetState();
2189 0 : RefPtr<AbstractThread> mainThread = mMaster->mAbstractMainThread;
2190 :
2191 0 : SetSeekingState(Move(seekJob), EventVisibility::Suppressed)->Then(
2192 : mainThread, __func__,
2193 0 : [start, info, hw](){ ReportRecoveryTelemetry(start, info, hw); },
2194 0 : [](){});
2195 0 : }
2196 :
2197 : RefPtr<MediaDecoder::SeekPromise>
2198 0 : MediaDecoderStateMachine::
2199 : StateObject::SetSeekingState(SeekJob&& aSeekJob, EventVisibility aVisibility)
2200 : {
2201 0 : if (aSeekJob.mTarget->IsAccurate() || aSeekJob.mTarget->IsFast()) {
2202 0 : if (aSeekJob.mTarget->IsVideoOnly()) {
2203 0 : return SetState<VideoOnlySeekingState>(Move(aSeekJob), aVisibility);
2204 : }
2205 0 : return SetState<AccurateSeekingState>(Move(aSeekJob), aVisibility);
2206 : }
2207 :
2208 0 : if (aSeekJob.mTarget->IsNextFrame()) {
2209 0 : return SetState<NextFrameSeekingState>(Move(aSeekJob), aVisibility);
2210 : }
2211 :
2212 0 : MOZ_ASSERT_UNREACHABLE("Unknown SeekTarget::Type.");
2213 : return nullptr;
2214 : }
2215 :
2216 : void
2217 0 : MediaDecoderStateMachine::
2218 : DecodeMetadataState::OnMetadataRead(MetadataHolder&& aMetadata)
2219 : {
2220 0 : mMetadataRequest.Complete();
2221 :
2222 : // Set mode to PLAYBACK after reading metadata.
2223 0 : Resource()->SetReadMode(MediaCacheStream::MODE_PLAYBACK);
2224 :
2225 0 : mMaster->mInfo.emplace(*aMetadata.mInfo);
2226 0 : mMaster->mMediaSeekable = Info().mMediaSeekable;
2227 0 : mMaster->mMediaSeekableOnlyInBufferedRanges =
2228 0 : Info().mMediaSeekableOnlyInBufferedRanges;
2229 :
2230 0 : if (Info().mMetadataDuration.isSome()) {
2231 0 : mMaster->RecomputeDuration();
2232 0 : } else if (Info().mUnadjustedMetadataEndTime.isSome()) {
2233 0 : const TimeUnit unadjusted = Info().mUnadjustedMetadataEndTime.ref();
2234 0 : const TimeUnit adjustment = Info().mStartTime;
2235 0 : mMaster->mInfo->mMetadataDuration.emplace(unadjusted - adjustment);
2236 0 : mMaster->RecomputeDuration();
2237 : }
2238 :
2239 : // If we don't know the duration by this point, we assume infinity, per spec.
2240 0 : if (mMaster->mDuration.Ref().isNothing()) {
2241 0 : mMaster->mDuration = Some(TimeUnit::FromInfinity());
2242 : }
2243 :
2244 0 : if (mMaster->HasVideo()) {
2245 0 : SLOG("Video decode isAsync=%d HWAccel=%d videoQueueSize=%d",
2246 : Reader()->IsAsync(),
2247 : Reader()->VideoIsHardwareAccelerated(),
2248 : mMaster->GetAmpleVideoFrames());
2249 : }
2250 :
2251 0 : MOZ_ASSERT(mMaster->mDuration.Ref().isSome());
2252 :
2253 0 : mMaster->mMetadataLoadedEvent.Notify(
2254 0 : Move(aMetadata.mInfo),
2255 0 : Move(aMetadata.mTags),
2256 0 : MediaDecoderEventVisibility::Observable);
2257 :
2258 0 : if (Info().IsEncrypted() && !mMaster->mCDMProxy) {
2259 : // Metadata parsing was successful but we're still waiting for CDM caps
2260 : // to become available so that we can build the correct decryptor/decoder.
2261 0 : SetState<WaitForCDMState>();
2262 : } else {
2263 0 : SetState<DecodingFirstFrameState>();
2264 : }
2265 0 : }
2266 :
2267 : void
2268 0 : MediaDecoderStateMachine::
2269 : DormantState::HandlePlayStateChanged(MediaDecoder::PlayState aPlayState)
2270 : {
2271 0 : if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) {
2272 : // Exit dormant when the user wants to play.
2273 0 : MOZ_ASSERT(!Info().IsEncrypted() || mMaster->mCDMProxy);
2274 0 : MOZ_ASSERT(mMaster->mSentFirstFrameLoadedEvent);
2275 0 : SetSeekingState(Move(mPendingSeek), EventVisibility::Suppressed);
2276 : }
2277 0 : }
2278 :
2279 : void
2280 0 : MediaDecoderStateMachine::
2281 : WaitForCDMState::HandleCDMProxyReady()
2282 : {
2283 0 : if (mPendingSeek.Exists()) {
2284 0 : SetSeekingState(Move(mPendingSeek), EventVisibility::Observable);
2285 : } else {
2286 0 : SetState<DecodingFirstFrameState>();
2287 : }
2288 0 : }
2289 :
2290 : void
2291 0 : MediaDecoderStateMachine::
2292 : DecodingFirstFrameState::Enter()
2293 : {
2294 : // Transition to DECODING if we've decoded first frames.
2295 0 : if (mMaster->mSentFirstFrameLoadedEvent) {
2296 0 : SetState<DecodingState>();
2297 0 : return;
2298 : }
2299 :
2300 0 : MOZ_ASSERT(!mMaster->mVideoDecodeSuspended);
2301 :
2302 : // Dispatch tasks to decode first frames.
2303 0 : if (mMaster->HasAudio()) {
2304 0 : mMaster->RequestAudioData();
2305 : }
2306 0 : if (mMaster->HasVideo()) {
2307 0 : mMaster->RequestVideoData(media::TimeUnit());
2308 : }
2309 : }
2310 :
2311 : void
2312 0 : MediaDecoderStateMachine::
2313 : DecodingFirstFrameState::MaybeFinishDecodeFirstFrame()
2314 : {
2315 0 : MOZ_ASSERT(!mMaster->mSentFirstFrameLoadedEvent);
2316 :
2317 0 : if ((mMaster->IsAudioDecoding() && AudioQueue().GetSize() == 0)
2318 0 : || (mMaster->IsVideoDecoding() && VideoQueue().GetSize() == 0)) {
2319 0 : return;
2320 : }
2321 :
2322 0 : mMaster->FinishDecodeFirstFrame();
2323 0 : if (mPendingSeek.Exists()) {
2324 0 : SetSeekingState(Move(mPendingSeek), EventVisibility::Observable);
2325 : } else {
2326 0 : SetState<DecodingState>();
2327 : }
2328 : }
2329 :
2330 : void
2331 0 : MediaDecoderStateMachine::
2332 : DecodingState::Enter()
2333 : {
2334 0 : MOZ_ASSERT(mMaster->mSentFirstFrameLoadedEvent);
2335 :
2336 0 : if (mMaster->mVideoDecodeMode == VideoDecodeMode::Suspend
2337 0 : && !mMaster->mVideoDecodeSuspendTimer.IsScheduled()
2338 0 : && !mMaster->mVideoDecodeSuspended) {
2339 : // If the VideoDecodeMode is Suspend and the timer is not schedule, it means
2340 : // the timer has timed out and we should suspend video decoding now if
2341 : // necessary.
2342 0 : HandleVideoSuspendTimeout();
2343 : }
2344 :
2345 0 : if (!mMaster->IsVideoDecoding() && !mMaster->IsAudioDecoding()) {
2346 0 : SetState<CompletedState>();
2347 0 : return;
2348 : }
2349 :
2350 0 : mOnAudioPopped = AudioQueue().PopEvent().Connect(
2351 0 : OwnerThread(), [this] () {
2352 0 : if (mMaster->IsAudioDecoding() && !mMaster->HaveEnoughDecodedAudio()) {
2353 0 : EnsureAudioDecodeTaskQueued();
2354 : }
2355 0 : });
2356 0 : mOnVideoPopped = VideoQueue().PopEvent().Connect(
2357 0 : OwnerThread(), [this] () {
2358 0 : if (mMaster->IsVideoDecoding() && !mMaster->HaveEnoughDecodedVideo()) {
2359 0 : EnsureVideoDecodeTaskQueued();
2360 : }
2361 0 : });
2362 :
2363 0 : mMaster->UpdateNextFrameStatus(MediaDecoderOwner::NEXT_FRAME_AVAILABLE);
2364 :
2365 0 : mDecodeStartTime = TimeStamp::Now();
2366 :
2367 0 : MaybeStopPrerolling();
2368 :
2369 : // Ensure that we've got tasks enqueued to decode data if we need to.
2370 0 : DispatchDecodeTasksIfNeeded();
2371 :
2372 0 : mMaster->ScheduleStateMachine();
2373 :
2374 : // Will enter dormant when playback is paused for a while.
2375 0 : if (mMaster->mPlayState == MediaDecoder::PLAY_STATE_PAUSED) {
2376 0 : StartDormantTimer();
2377 : }
2378 : }
2379 :
2380 : void
2381 0 : MediaDecoderStateMachine::
2382 : DecodingState::HandleEndOfAudio()
2383 : {
2384 0 : AudioQueue().Finish();
2385 0 : if (!mMaster->IsVideoDecoding()) {
2386 0 : SetState<CompletedState>();
2387 : } else {
2388 0 : MaybeStopPrerolling();
2389 : }
2390 0 : }
2391 :
2392 : void
2393 0 : MediaDecoderStateMachine::
2394 : DecodingState::HandleEndOfVideo()
2395 : {
2396 0 : VideoQueue().Finish();
2397 0 : if (!mMaster->IsAudioDecoding()) {
2398 0 : SetState<CompletedState>();
2399 : } else {
2400 0 : MaybeStopPrerolling();
2401 : }
2402 0 : }
2403 :
2404 : void
2405 0 : MediaDecoderStateMachine::
2406 : DecodingState::DispatchDecodeTasksIfNeeded()
2407 : {
2408 0 : if (mMaster->IsAudioDecoding()
2409 0 : && !mMaster->mMinimizePreroll
2410 0 : && !mMaster->HaveEnoughDecodedAudio()) {
2411 0 : EnsureAudioDecodeTaskQueued();
2412 : }
2413 :
2414 0 : if (mMaster->IsVideoDecoding()
2415 0 : && !mMaster->mMinimizePreroll
2416 0 : && !mMaster->HaveEnoughDecodedVideo()) {
2417 0 : EnsureVideoDecodeTaskQueued();
2418 : }
2419 0 : }
2420 :
2421 : void
2422 0 : MediaDecoderStateMachine::
2423 : DecodingState::EnsureAudioDecodeTaskQueued()
2424 : {
2425 0 : if (!mMaster->IsAudioDecoding()
2426 0 : || mMaster->IsRequestingAudioData()
2427 0 : || mMaster->IsWaitingAudioData()) {
2428 0 : return;
2429 : }
2430 0 : mMaster->RequestAudioData();
2431 : }
2432 :
2433 : void
2434 0 : MediaDecoderStateMachine::
2435 : DecodingState::EnsureVideoDecodeTaskQueued()
2436 : {
2437 0 : if (!mMaster->IsVideoDecoding()
2438 0 : || mMaster->IsRequestingVideoData()
2439 0 : || mMaster->IsWaitingVideoData()) {
2440 0 : return;
2441 : }
2442 0 : mMaster->RequestVideoData(mMaster->GetMediaTime());
2443 : }
2444 :
2445 : void
2446 0 : MediaDecoderStateMachine::
2447 : DecodingState::MaybeStartBuffering()
2448 : {
2449 : // Buffering makes senses only after decoding first frames.
2450 0 : MOZ_ASSERT(mMaster->mSentFirstFrameLoadedEvent);
2451 :
2452 : // Don't enter buffering when MediaDecoder is not playing.
2453 0 : if (mMaster->mPlayState != MediaDecoder::PLAY_STATE_PLAYING) {
2454 0 : return;
2455 : }
2456 :
2457 : // Don't enter buffering while prerolling so that the decoder has a chance to
2458 : // enqueue some decoded data before we give up and start buffering.
2459 0 : if (!mMaster->IsPlaying()) {
2460 0 : return;
2461 : }
2462 :
2463 : bool shouldBuffer;
2464 0 : if (Reader()->UseBufferingHeuristics()) {
2465 0 : shouldBuffer = IsExpectingMoreData()
2466 0 : && mMaster->HasLowDecodedData()
2467 0 : && mMaster->HasLowBufferedData();
2468 : } else {
2469 0 : shouldBuffer =
2470 0 : (mMaster->OutOfDecodedAudio() && mMaster->IsWaitingAudioData())
2471 0 : || (mMaster->OutOfDecodedVideo() && mMaster->IsWaitingVideoData());
2472 : }
2473 0 : if (shouldBuffer) {
2474 0 : SetState<BufferingState>();
2475 : }
2476 : }
2477 :
2478 : void
2479 0 : MediaDecoderStateMachine::
2480 : SeekingState::SeekCompleted()
2481 : {
2482 0 : const auto newCurrentTime = CalculateNewCurrentTime();
2483 :
2484 0 : bool isLiveStream = Resource()->IsLiveStream();
2485 0 : if (newCurrentTime == mMaster->Duration() && !isLiveStream) {
2486 : // Seeked to end of media. Explicitly finish the queues so DECODING
2487 : // will transition to COMPLETED immediately. Note we don't do
2488 : // this when playing a live stream, since the end of media will advance
2489 : // once we download more data!
2490 0 : AudioQueue().Finish();
2491 0 : VideoQueue().Finish();
2492 :
2493 : // We won't start MediaSink when paused. m{Audio,Video}Completed will
2494 : // remain false and 'playbackEnded' won't be notified. Therefore we
2495 : // need to set these flags explicitly when seeking to the end.
2496 0 : mMaster->mAudioCompleted = true;
2497 0 : mMaster->mVideoCompleted = true;
2498 :
2499 : // There might still be a pending audio request when doing video-only or
2500 : // next-frame seek. Discard it so we won't break the invariants of the
2501 : // COMPLETED state by adding audio samples to a finished queue.
2502 0 : mMaster->mAudioDataRequest.DisconnectIfExists();
2503 : }
2504 :
2505 : // We want to resolve the seek request prior finishing the first frame
2506 : // to ensure that the seeked event is fired prior loadeded.
2507 : // Note: SeekJob.Resolve() resets SeekJob.mTarget. Don't use mSeekJob anymore
2508 : // hereafter.
2509 0 : mSeekJob.Resolve(__func__);
2510 :
2511 : // Notify FirstFrameLoaded now if we haven't since we've decoded some data
2512 : // for readyState to transition to HAVE_CURRENT_DATA and fire 'loadeddata'.
2513 0 : if (!mMaster->mSentFirstFrameLoadedEvent) {
2514 0 : mMaster->FinishDecodeFirstFrame();
2515 : }
2516 :
2517 : // Ensure timestamps are up to date.
2518 : // Suppressed visibility comes from two cases: (1) leaving dormant state,
2519 : // and (2) resuming suspended video decoder. We want both cases to be
2520 : // transparent to the user. So we only notify the change when the seek
2521 : // request is from the user.
2522 0 : if (mVisibility == EventVisibility::Observable) {
2523 : // Don't update playback position for video-only seek.
2524 : // Otherwise we might have |newCurrentTime > mMediaSink->GetPosition()|
2525 : // and fail the assertion in GetClock() since we didn't stop MediaSink.
2526 0 : mMaster->UpdatePlaybackPositionInternal(newCurrentTime);
2527 : }
2528 :
2529 : // Try to decode another frame to detect if we're at the end...
2530 0 : SLOG("Seek completed, mCurrentPosition=%" PRId64,
2531 : mMaster->mCurrentPosition.Ref().ToMicroseconds());
2532 :
2533 0 : if (mMaster->VideoQueue().PeekFront()) {
2534 0 : mMaster->mMediaSink->Redraw(Info().mVideo);
2535 0 : mMaster->mOnPlaybackEvent.Notify(MediaEventType::Invalidate);
2536 : }
2537 :
2538 0 : GoToNextState();
2539 0 : }
2540 :
2541 : void
2542 0 : MediaDecoderStateMachine::
2543 : BufferingState::DispatchDecodeTasksIfNeeded()
2544 : {
2545 0 : if (mMaster->IsAudioDecoding()
2546 0 : && !mMaster->HaveEnoughDecodedAudio()
2547 0 : && !mMaster->IsRequestingAudioData()
2548 0 : && !mMaster->IsWaitingAudioData()) {
2549 0 : mMaster->RequestAudioData();
2550 : }
2551 :
2552 0 : if (mMaster->IsVideoDecoding()
2553 0 : && !mMaster->HaveEnoughDecodedVideo()
2554 0 : && !mMaster->IsRequestingVideoData()
2555 0 : && !mMaster->IsWaitingVideoData()) {
2556 0 : mMaster->RequestVideoData(media::TimeUnit());
2557 : }
2558 0 : }
2559 :
2560 : void
2561 0 : MediaDecoderStateMachine::
2562 : BufferingState::Step()
2563 : {
2564 0 : TimeStamp now = TimeStamp::Now();
2565 0 : MOZ_ASSERT(!mBufferingStart.IsNull(), "Must know buffering start time.");
2566 :
2567 : // With buffering heuristics we will remain in the buffering state if
2568 : // we've not decoded enough data to begin playback, or if we've not
2569 : // downloaded a reasonable amount of data inside our buffering time.
2570 0 : if (Reader()->UseBufferingHeuristics()) {
2571 0 : TimeDuration elapsed = now - mBufferingStart;
2572 0 : bool isLiveStream = Resource()->IsLiveStream();
2573 0 : if ((isLiveStream || !mMaster->CanPlayThrough())
2574 0 : && elapsed
2575 0 : < TimeDuration::FromSeconds(mBufferingWait * mMaster->mPlaybackRate)
2576 0 : && mMaster->HasLowBufferedData(TimeUnit::FromSeconds(mBufferingWait))
2577 0 : && IsExpectingMoreData()) {
2578 0 : SLOG("Buffering: wait %ds, timeout in %.3lfs",
2579 : mBufferingWait, mBufferingWait - elapsed.ToSeconds());
2580 0 : mMaster->ScheduleStateMachineIn(TimeUnit::FromMicroseconds(USECS_PER_S));
2581 0 : DispatchDecodeTasksIfNeeded();
2582 0 : return;
2583 : }
2584 0 : } else if (mMaster->OutOfDecodedAudio() || mMaster->OutOfDecodedVideo()) {
2585 0 : DispatchDecodeTasksIfNeeded();
2586 0 : MOZ_ASSERT(!mMaster->OutOfDecodedAudio()
2587 : || mMaster->IsRequestingAudioData()
2588 : || mMaster->IsWaitingAudioData());
2589 0 : MOZ_ASSERT(!mMaster->OutOfDecodedVideo()
2590 : || mMaster->IsRequestingVideoData()
2591 : || mMaster->IsWaitingVideoData());
2592 0 : SLOG("In buffering mode, waiting to be notified: outOfAudio: %d, "
2593 : "mAudioStatus: %s, outOfVideo: %d, mVideoStatus: %s",
2594 : mMaster->OutOfDecodedAudio(), mMaster->AudioRequestStatus(),
2595 : mMaster->OutOfDecodedVideo(), mMaster->VideoRequestStatus());
2596 0 : return;
2597 : }
2598 :
2599 0 : SLOG("Buffered for %.3lfs", (now - mBufferingStart).ToSeconds());
2600 0 : SetState<DecodingState>();
2601 : }
2602 :
2603 : void
2604 0 : MediaDecoderStateMachine::
2605 : BufferingState::HandleEndOfAudio()
2606 : {
2607 0 : AudioQueue().Finish();
2608 0 : if (!mMaster->IsVideoDecoding()) {
2609 0 : SetState<CompletedState>();
2610 : } else {
2611 : // Check if we can exit buffering.
2612 0 : mMaster->ScheduleStateMachine();
2613 : }
2614 0 : }
2615 :
2616 : void
2617 0 : MediaDecoderStateMachine::
2618 : BufferingState::HandleEndOfVideo()
2619 : {
2620 0 : VideoQueue().Finish();
2621 0 : if (!mMaster->IsAudioDecoding()) {
2622 0 : SetState<CompletedState>();
2623 : } else {
2624 : // Check if we can exit buffering.
2625 0 : mMaster->ScheduleStateMachine();
2626 : }
2627 0 : }
2628 :
2629 : RefPtr<ShutdownPromise>
2630 0 : MediaDecoderStateMachine::
2631 : ShutdownState::Enter()
2632 : {
2633 0 : auto master = mMaster;
2634 :
2635 0 : master->mDelayedScheduler.Reset();
2636 :
2637 : // Shutdown happens while decode timer is active, we need to disconnect and
2638 : // dispose of the timer.
2639 0 : master->CancelSuspendTimer();
2640 :
2641 0 : master->mCDMProxyPromise.DisconnectIfExists();
2642 :
2643 0 : if (master->IsPlaying()) {
2644 0 : master->StopPlayback();
2645 : }
2646 :
2647 0 : master->mAudioDataRequest.DisconnectIfExists();
2648 0 : master->mVideoDataRequest.DisconnectIfExists();
2649 0 : master->mAudioWaitRequest.DisconnectIfExists();
2650 0 : master->mVideoWaitRequest.DisconnectIfExists();
2651 :
2652 0 : master->ResetDecode();
2653 0 : master->StopMediaSink();
2654 0 : master->mMediaSink->Shutdown();
2655 :
2656 : // Prevent dangling pointers by disconnecting the listeners.
2657 0 : master->mAudioQueueListener.Disconnect();
2658 0 : master->mVideoQueueListener.Disconnect();
2659 0 : master->mMetadataManager.Disconnect();
2660 0 : master->mOnMediaNotSeekable.Disconnect();
2661 :
2662 : // Disconnect canonicals and mirrors before shutting down our task queue.
2663 0 : master->mBuffered.DisconnectIfConnected();
2664 0 : master->mExplicitDuration.DisconnectIfConnected();
2665 0 : master->mPlayState.DisconnectIfConnected();
2666 0 : master->mNextPlayState.DisconnectIfConnected();
2667 0 : master->mVolume.DisconnectIfConnected();
2668 0 : master->mPreservesPitch.DisconnectIfConnected();
2669 0 : master->mLooping.DisconnectIfConnected();
2670 0 : master->mSameOriginMedia.DisconnectIfConnected();
2671 0 : master->mMediaPrincipalHandle.DisconnectIfConnected();
2672 0 : master->mPlaybackBytesPerSecond.DisconnectIfConnected();
2673 0 : master->mPlaybackRateReliable.DisconnectIfConnected();
2674 0 : master->mDecoderPosition.DisconnectIfConnected();
2675 :
2676 0 : master->mDuration.DisconnectAll();
2677 0 : master->mNextFrameStatus.DisconnectAll();
2678 0 : master->mCurrentPosition.DisconnectAll();
2679 0 : master->mPlaybackOffset.DisconnectAll();
2680 0 : master->mIsAudioDataAudible.DisconnectAll();
2681 :
2682 : // Shut down the watch manager to stop further notifications.
2683 0 : master->mWatchManager.Shutdown();
2684 :
2685 0 : return Reader()->Shutdown()->Then(
2686 0 : OwnerThread(), __func__, master,
2687 : &MediaDecoderStateMachine::FinishShutdown,
2688 0 : &MediaDecoderStateMachine::FinishShutdown);
2689 : }
2690 :
2691 : #define INIT_WATCHABLE(name, val) \
2692 : name(val, "MediaDecoderStateMachine::" #name)
2693 : #define INIT_MIRROR(name, val) \
2694 : name(mTaskQueue, val, "MediaDecoderStateMachine::" #name " (Mirror)")
2695 : #define INIT_CANONICAL(name, val) \
2696 : name(mTaskQueue, val, "MediaDecoderStateMachine::" #name " (Canonical)")
2697 :
2698 0 : MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
2699 0 : MediaDecoderReader* aReader) :
2700 : mDecoderID(aDecoder),
2701 : mAbstractMainThread(aDecoder->AbstractMainThread()),
2702 0 : mFrameStats(&aDecoder->GetFrameStatistics()),
2703 : mVideoFrameContainer(aDecoder->GetVideoFrameContainer()),
2704 0 : mAudioChannel(aDecoder->GetAudioChannel()),
2705 : mTaskQueue(new TaskQueue(
2706 0 : GetMediaThreadPool(MediaThreadType::PLAYBACK),
2707 0 : "MDSM::mTaskQueue", /* aSupportsTailDispatch = */ true)),
2708 : mWatchManager(this, mTaskQueue),
2709 : mDispatchedStateMachine(false),
2710 : mDelayedScheduler(mTaskQueue),
2711 : mCurrentFrameID(0),
2712 0 : INIT_WATCHABLE(mObservedDuration, TimeUnit()),
2713 0 : mReader(new MediaDecoderReaderWrapper(mTaskQueue, aReader)),
2714 : mPlaybackRate(1.0),
2715 : mLowAudioThreshold(detail::LOW_AUDIO_THRESHOLD),
2716 : mAmpleAudioThreshold(detail::AMPLE_AUDIO_THRESHOLD),
2717 : mAudioCaptured(false),
2718 0 : mMinimizePreroll(aDecoder->GetMinimizePreroll()),
2719 : mSentFirstFrameLoadedEvent(false),
2720 : mVideoDecodeSuspended(false),
2721 : mVideoDecodeSuspendTimer(mTaskQueue),
2722 0 : mOutputStreamManager(new OutputStreamManager()),
2723 : mResource(aDecoder->GetResource()),
2724 : mVideoDecodeMode(VideoDecodeMode::Normal),
2725 0 : mIsMSE(aDecoder->IsMSE()),
2726 0 : INIT_MIRROR(mBuffered, TimeIntervals()),
2727 0 : INIT_MIRROR(mExplicitDuration, Maybe<double>()),
2728 : INIT_MIRROR(mPlayState, MediaDecoder::PLAY_STATE_LOADING),
2729 : INIT_MIRROR(mNextPlayState, MediaDecoder::PLAY_STATE_PAUSED),
2730 : INIT_MIRROR(mVolume, 1.0),
2731 : INIT_MIRROR(mPreservesPitch, true),
2732 : INIT_MIRROR(mLooping, false),
2733 : INIT_MIRROR(mSameOriginMedia, false),
2734 : INIT_MIRROR(mMediaPrincipalHandle, PRINCIPAL_HANDLE_NONE),
2735 : INIT_MIRROR(mPlaybackBytesPerSecond, 0.0),
2736 : INIT_MIRROR(mPlaybackRateReliable, true),
2737 : INIT_MIRROR(mDecoderPosition, 0),
2738 0 : INIT_CANONICAL(mDuration, NullableTimeUnit()),
2739 : INIT_CANONICAL(mNextFrameStatus, MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE),
2740 0 : INIT_CANONICAL(mCurrentPosition, TimeUnit::Zero()),
2741 : INIT_CANONICAL(mPlaybackOffset, 0),
2742 0 : INIT_CANONICAL(mIsAudioDataAudible, false)
2743 : #ifdef XP_WIN
2744 : , mShouldUseHiResTimers(Preferences::GetBool("media.hi-res-timers.enabled", true))
2745 : #endif
2746 : {
2747 0 : MOZ_COUNT_CTOR(MediaDecoderStateMachine);
2748 0 : NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
2749 :
2750 0 : InitVideoQueuePrefs();
2751 0 : }
2752 :
2753 : #undef INIT_WATCHABLE
2754 : #undef INIT_MIRROR
2755 : #undef INIT_CANONICAL
2756 :
2757 0 : MediaDecoderStateMachine::~MediaDecoderStateMachine()
2758 : {
2759 0 : MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread.");
2760 0 : MOZ_COUNT_DTOR(MediaDecoderStateMachine);
2761 :
2762 : #ifdef XP_WIN
2763 : MOZ_ASSERT(!mHiResTimersRequested);
2764 : #endif
2765 0 : }
2766 :
2767 : void
2768 0 : MediaDecoderStateMachine::InitializationTask(MediaDecoder* aDecoder)
2769 : {
2770 0 : MOZ_ASSERT(OnTaskQueue());
2771 :
2772 : // Connect mirrors.
2773 0 : mBuffered.Connect(mReader->CanonicalBuffered());
2774 0 : mExplicitDuration.Connect(aDecoder->CanonicalExplicitDuration());
2775 0 : mPlayState.Connect(aDecoder->CanonicalPlayState());
2776 0 : mNextPlayState.Connect(aDecoder->CanonicalNextPlayState());
2777 0 : mVolume.Connect(aDecoder->CanonicalVolume());
2778 0 : mPreservesPitch.Connect(aDecoder->CanonicalPreservesPitch());
2779 0 : mLooping.Connect(aDecoder->CanonicalLooping());
2780 0 : mSameOriginMedia.Connect(aDecoder->CanonicalSameOriginMedia());
2781 0 : mMediaPrincipalHandle.Connect(aDecoder->CanonicalMediaPrincipalHandle());
2782 0 : mPlaybackBytesPerSecond.Connect(aDecoder->CanonicalPlaybackBytesPerSecond());
2783 0 : mPlaybackRateReliable.Connect(aDecoder->CanonicalPlaybackRateReliable());
2784 0 : mDecoderPosition.Connect(aDecoder->CanonicalDecoderPosition());
2785 :
2786 : // Initialize watchers.
2787 0 : mWatchManager.Watch(mBuffered,
2788 0 : &MediaDecoderStateMachine::BufferedRangeUpdated);
2789 0 : mWatchManager.Watch(mVolume, &MediaDecoderStateMachine::VolumeChanged);
2790 0 : mWatchManager.Watch(mPreservesPitch,
2791 0 : &MediaDecoderStateMachine::PreservesPitchChanged);
2792 0 : mWatchManager.Watch(mExplicitDuration,
2793 0 : &MediaDecoderStateMachine::RecomputeDuration);
2794 0 : mWatchManager.Watch(mObservedDuration,
2795 0 : &MediaDecoderStateMachine::RecomputeDuration);
2796 0 : mWatchManager.Watch(mPlayState, &MediaDecoderStateMachine::PlayStateChanged);
2797 :
2798 0 : MOZ_ASSERT(!mStateObj);
2799 0 : auto* s = new DecodeMetadataState(this);
2800 0 : mStateObj.reset(s);
2801 0 : s->Enter();
2802 0 : }
2803 :
2804 : void
2805 0 : MediaDecoderStateMachine::AudioAudibleChanged(bool aAudible)
2806 : {
2807 0 : mIsAudioDataAudible = aAudible;
2808 0 : }
2809 :
2810 : media::MediaSink*
2811 0 : MediaDecoderStateMachine::CreateAudioSink()
2812 : {
2813 0 : RefPtr<MediaDecoderStateMachine> self = this;
2814 0 : auto audioSinkCreator = [self] () {
2815 0 : MOZ_ASSERT(self->OnTaskQueue());
2816 : AudioSink* audioSink = new AudioSink(
2817 0 : self->mTaskQueue, self->mAudioQueue,
2818 0 : self->GetMediaTime(),
2819 0 : self->Info().mAudio, self->mAudioChannel);
2820 :
2821 0 : self->mAudibleListener = audioSink->AudibleEvent().Connect(
2822 0 : self->mTaskQueue, self.get(),
2823 0 : &MediaDecoderStateMachine::AudioAudibleChanged);
2824 0 : return audioSink;
2825 0 : };
2826 0 : return new AudioSinkWrapper(mTaskQueue, audioSinkCreator);
2827 : }
2828 :
2829 : already_AddRefed<media::MediaSink>
2830 0 : MediaDecoderStateMachine::CreateMediaSink(bool aAudioCaptured)
2831 : {
2832 : RefPtr<media::MediaSink> audioSink =
2833 : aAudioCaptured
2834 : ? new DecodedStream(mTaskQueue, mAbstractMainThread, mAudioQueue,
2835 : mVideoQueue, mOutputStreamManager,
2836 0 : mSameOriginMedia.Ref(), mMediaPrincipalHandle.Ref())
2837 0 : : CreateAudioSink();
2838 :
2839 : RefPtr<media::MediaSink> mediaSink =
2840 : new VideoSink(mTaskQueue, audioSink, mVideoQueue,
2841 0 : mVideoFrameContainer, *mFrameStats,
2842 0 : sVideoQueueSendToCompositorSize);
2843 0 : return mediaSink.forget();
2844 : }
2845 :
2846 : TimeUnit
2847 0 : MediaDecoderStateMachine::GetDecodedAudioDuration()
2848 : {
2849 0 : MOZ_ASSERT(OnTaskQueue());
2850 0 : if (mMediaSink->IsStarted()) {
2851 : // mDecodedAudioEndTime might be smaller than GetClock() when there is
2852 : // overlap between 2 adjacent audio samples or when we are playing
2853 : // a chained ogg file.
2854 0 : return std::max(mDecodedAudioEndTime - GetClock(), TimeUnit::Zero());
2855 : }
2856 : // MediaSink not started. All audio samples are in the queue.
2857 0 : return TimeUnit::FromMicroseconds(AudioQueue().Duration());
2858 : }
2859 :
2860 : bool
2861 0 : MediaDecoderStateMachine::HaveEnoughDecodedAudio()
2862 : {
2863 0 : MOZ_ASSERT(OnTaskQueue());
2864 0 : auto ampleAudio = mAmpleAudioThreshold.MultDouble(mPlaybackRate);
2865 0 : return AudioQueue().GetSize() > 0
2866 0 : && GetDecodedAudioDuration() >= ampleAudio;
2867 : }
2868 :
2869 0 : bool MediaDecoderStateMachine::HaveEnoughDecodedVideo()
2870 : {
2871 0 : MOZ_ASSERT(OnTaskQueue());
2872 0 : return VideoQueue().GetSize() >= GetAmpleVideoFrames() * mPlaybackRate + 1;
2873 : }
2874 :
2875 : void
2876 0 : MediaDecoderStateMachine::PushAudio(AudioData* aSample)
2877 : {
2878 0 : MOZ_ASSERT(OnTaskQueue());
2879 0 : MOZ_ASSERT(aSample);
2880 0 : AudioQueue().Push(aSample);
2881 0 : }
2882 :
2883 : void
2884 0 : MediaDecoderStateMachine::PushVideo(VideoData* aSample)
2885 : {
2886 0 : MOZ_ASSERT(OnTaskQueue());
2887 0 : MOZ_ASSERT(aSample);
2888 0 : aSample->mFrameID = ++mCurrentFrameID;
2889 0 : VideoQueue().Push(aSample);
2890 0 : }
2891 :
2892 : void
2893 0 : MediaDecoderStateMachine::OnAudioPopped(const RefPtr<AudioData>& aSample)
2894 : {
2895 0 : MOZ_ASSERT(OnTaskQueue());
2896 0 : mPlaybackOffset = std::max(mPlaybackOffset.Ref(), aSample->mOffset);
2897 0 : }
2898 :
2899 : void
2900 0 : MediaDecoderStateMachine::OnVideoPopped(const RefPtr<VideoData>& aSample)
2901 : {
2902 0 : MOZ_ASSERT(OnTaskQueue());
2903 0 : mPlaybackOffset = std::max(mPlaybackOffset.Ref(), aSample->mOffset);
2904 0 : }
2905 :
2906 : bool
2907 0 : MediaDecoderStateMachine::IsAudioDecoding()
2908 : {
2909 0 : MOZ_ASSERT(OnTaskQueue());
2910 0 : return HasAudio() && !AudioQueue().IsFinished();
2911 : }
2912 :
2913 : bool
2914 0 : MediaDecoderStateMachine::IsVideoDecoding()
2915 : {
2916 0 : MOZ_ASSERT(OnTaskQueue());
2917 0 : return HasVideo() && !VideoQueue().IsFinished();
2918 : }
2919 :
2920 0 : bool MediaDecoderStateMachine::IsPlaying() const
2921 : {
2922 0 : MOZ_ASSERT(OnTaskQueue());
2923 0 : return mMediaSink->IsPlaying();
2924 : }
2925 :
2926 0 : void MediaDecoderStateMachine::SetMediaNotSeekable()
2927 : {
2928 0 : mMediaSeekable = false;
2929 0 : }
2930 :
2931 0 : nsresult MediaDecoderStateMachine::Init(MediaDecoder* aDecoder)
2932 : {
2933 0 : MOZ_ASSERT(NS_IsMainThread());
2934 :
2935 : // Dispatch initialization that needs to happen on that task queue.
2936 0 : nsCOMPtr<nsIRunnable> r = NewRunnableMethod<RefPtr<MediaDecoder>>(
2937 : "MediaDecoderStateMachine::InitializationTask",
2938 : this,
2939 : &MediaDecoderStateMachine::InitializationTask,
2940 0 : aDecoder);
2941 0 : mTaskQueue->DispatchStateChange(r.forget());
2942 :
2943 0 : mAudioQueueListener = AudioQueue().PopEvent().Connect(
2944 0 : mTaskQueue, this, &MediaDecoderStateMachine::OnAudioPopped);
2945 0 : mVideoQueueListener = VideoQueue().PopEvent().Connect(
2946 0 : mTaskQueue, this, &MediaDecoderStateMachine::OnVideoPopped);
2947 :
2948 0 : mMetadataManager.Connect(mReader->TimedMetadataEvent(), OwnerThread());
2949 :
2950 0 : mOnMediaNotSeekable = mReader->OnMediaNotSeekable().Connect(
2951 0 : OwnerThread(), this, &MediaDecoderStateMachine::SetMediaNotSeekable);
2952 :
2953 0 : mMediaSink = CreateMediaSink(mAudioCaptured);
2954 :
2955 0 : aDecoder->RequestCDMProxy()->Then(
2956 0 : OwnerThread(), __func__, this,
2957 : &MediaDecoderStateMachine::OnCDMProxyReady,
2958 : &MediaDecoderStateMachine::OnCDMProxyNotReady)
2959 0 : ->Track(mCDMProxyPromise);
2960 :
2961 0 : nsresult rv = mReader->Init();
2962 0 : NS_ENSURE_SUCCESS(rv, rv);
2963 :
2964 0 : mReader->SetCanonicalDuration(&mDuration);
2965 :
2966 0 : return NS_OK;
2967 : }
2968 :
2969 : void
2970 0 : MediaDecoderStateMachine::StopPlayback()
2971 : {
2972 0 : MOZ_ASSERT(OnTaskQueue());
2973 0 : LOG("StopPlayback()");
2974 :
2975 0 : mOnPlaybackEvent.Notify(MediaEventType::PlaybackStopped);
2976 :
2977 0 : if (IsPlaying()) {
2978 0 : mMediaSink->SetPlaying(false);
2979 0 : MOZ_ASSERT(!IsPlaying());
2980 : #ifdef XP_WIN
2981 : if (mHiResTimersRequested) {
2982 : mHiResTimersRequested = false;
2983 : timeEndPeriod(1);
2984 : }
2985 : #endif
2986 : }
2987 0 : }
2988 :
2989 0 : void MediaDecoderStateMachine::MaybeStartPlayback()
2990 : {
2991 0 : MOZ_ASSERT(OnTaskQueue());
2992 : // Should try to start playback only after decoding first frames.
2993 0 : MOZ_ASSERT(mSentFirstFrameLoadedEvent);
2994 :
2995 0 : if (IsPlaying()) {
2996 : // Logging this case is really spammy - don't do it.
2997 0 : return;
2998 : }
2999 :
3000 0 : if (mPlayState != MediaDecoder::PLAY_STATE_PLAYING) {
3001 0 : LOG("Not starting playback [mPlayState=%d]", mPlayState.Ref());
3002 0 : return;
3003 : }
3004 :
3005 0 : LOG("MaybeStartPlayback() starting playback");
3006 0 : mOnPlaybackEvent.Notify(MediaEventType::PlaybackStarted);
3007 0 : StartMediaSink();
3008 :
3009 : #ifdef XP_WIN
3010 : if (!mHiResTimersRequested && mShouldUseHiResTimers) {
3011 : mHiResTimersRequested = true;
3012 : // Ensure high precision timers are enabled on Windows, otherwise the state
3013 : // machine isn't woken up at reliable intervals to set the next frame, and we
3014 : // drop frames while painting. Note that each call must be matched by a
3015 : // corresponding timeEndPeriod() call. Enabling high precision timers causes
3016 : // the CPU to wake up more frequently on Windows 7 and earlier, which causes
3017 : // more CPU load and battery use. So we only enable high precision timers
3018 : // when we're actually playing.
3019 : timeBeginPeriod(1);
3020 : }
3021 : #endif
3022 :
3023 0 : if (!IsPlaying()) {
3024 0 : mMediaSink->SetPlaying(true);
3025 0 : MOZ_ASSERT(IsPlaying());
3026 : }
3027 : }
3028 :
3029 : void
3030 0 : MediaDecoderStateMachine::UpdatePlaybackPositionInternal(const TimeUnit& aTime)
3031 : {
3032 0 : MOZ_ASSERT(OnTaskQueue());
3033 0 : LOGV("UpdatePlaybackPositionInternal(%" PRId64 ")", aTime.ToMicroseconds());
3034 :
3035 0 : mCurrentPosition = aTime;
3036 0 : NS_ASSERTION(mCurrentPosition.Ref() >= TimeUnit::Zero(),
3037 : "CurrentTime should be positive!");
3038 0 : mObservedDuration = std::max(mObservedDuration.Ref(), mCurrentPosition.Ref());
3039 0 : }
3040 :
3041 : void
3042 0 : MediaDecoderStateMachine::UpdatePlaybackPosition(const TimeUnit& aTime)
3043 : {
3044 0 : MOZ_ASSERT(OnTaskQueue());
3045 0 : UpdatePlaybackPositionInternal(aTime);
3046 :
3047 0 : bool fragmentEnded = mFragmentEndTime.IsValid()
3048 0 : && GetMediaTime() >= mFragmentEndTime;
3049 0 : mMetadataManager.DispatchMetadataIfNeeded(aTime);
3050 :
3051 0 : if (fragmentEnded) {
3052 0 : StopPlayback();
3053 : }
3054 0 : }
3055 :
3056 : /* static */ const char*
3057 0 : MediaDecoderStateMachine::ToStateStr(State aState)
3058 : {
3059 0 : switch (aState) {
3060 0 : case DECODER_STATE_DECODING_METADATA: return "DECODING_METADATA";
3061 0 : case DECODER_STATE_WAIT_FOR_CDM: return "WAIT_FOR_CDM";
3062 0 : case DECODER_STATE_DORMANT: return "DORMANT";
3063 0 : case DECODER_STATE_DECODING_FIRSTFRAME: return "DECODING_FIRSTFRAME";
3064 0 : case DECODER_STATE_DECODING: return "DECODING";
3065 0 : case DECODER_STATE_SEEKING: return "SEEKING";
3066 0 : case DECODER_STATE_BUFFERING: return "BUFFERING";
3067 0 : case DECODER_STATE_COMPLETED: return "COMPLETED";
3068 0 : case DECODER_STATE_SHUTDOWN: return "SHUTDOWN";
3069 0 : default: MOZ_ASSERT_UNREACHABLE("Invalid state.");
3070 : }
3071 : return "UNKNOWN";
3072 : }
3073 :
3074 : const char*
3075 0 : MediaDecoderStateMachine::ToStateStr()
3076 : {
3077 0 : MOZ_ASSERT(OnTaskQueue());
3078 0 : return ToStateStr(mStateObj->GetState());
3079 : }
3080 :
3081 0 : void MediaDecoderStateMachine::VolumeChanged()
3082 : {
3083 0 : MOZ_ASSERT(OnTaskQueue());
3084 0 : mMediaSink->SetVolume(mVolume);
3085 0 : }
3086 :
3087 0 : void MediaDecoderStateMachine::RecomputeDuration()
3088 : {
3089 0 : MOZ_ASSERT(OnTaskQueue());
3090 :
3091 0 : TimeUnit duration;
3092 0 : if (mExplicitDuration.Ref().isSome()) {
3093 0 : double d = mExplicitDuration.Ref().ref();
3094 0 : if (IsNaN(d)) {
3095 : // We have an explicit duration (which means that we shouldn't look at
3096 : // any other duration sources), but the duration isn't ready yet.
3097 0 : return;
3098 : }
3099 : // We don't fire duration changed for this case because it should have
3100 : // already been fired on the main thread when the explicit duration was set.
3101 0 : duration = TimeUnit::FromSeconds(d);
3102 0 : } else if (mInfo.isSome() && Info().mMetadataDuration.isSome()) {
3103 : // We need to check mInfo.isSome() because that this method might be invoked
3104 : // while mObservedDuration is changed which might before the metadata been
3105 : // read.
3106 0 : duration = Info().mMetadataDuration.ref();
3107 : } else {
3108 0 : return;
3109 : }
3110 :
3111 : // Only adjust the duration when an explicit duration isn't set (MSE).
3112 : // The duration is always exactly known with MSE and there's no need to adjust
3113 : // it based on what may have been seen in the past; in particular as this data
3114 : // may no longer exist such as when the mediasource duration was reduced.
3115 0 : if (mExplicitDuration.Ref().isNothing()
3116 0 : && duration < mObservedDuration.Ref()) {
3117 0 : duration = mObservedDuration;
3118 : }
3119 :
3120 0 : MOZ_ASSERT(duration >= TimeUnit::Zero());
3121 0 : mDuration = Some(duration);
3122 : }
3123 :
3124 : RefPtr<ShutdownPromise>
3125 0 : MediaDecoderStateMachine::Shutdown()
3126 : {
3127 0 : MOZ_ASSERT(OnTaskQueue());
3128 0 : return mStateObj->HandleShutdown();
3129 : }
3130 :
3131 0 : void MediaDecoderStateMachine::PlayStateChanged()
3132 : {
3133 0 : MOZ_ASSERT(OnTaskQueue());
3134 :
3135 0 : if (mPlayState != MediaDecoder::PLAY_STATE_PLAYING) {
3136 0 : CancelSuspendTimer();
3137 0 : } else if (mMinimizePreroll) {
3138 : // Once we start playing, we don't want to minimize our prerolling, as we
3139 : // assume the user is likely to want to keep playing in future. This needs
3140 : // to happen before we invoke StartDecoding().
3141 0 : mMinimizePreroll = false;
3142 : }
3143 :
3144 0 : mStateObj->HandlePlayStateChanged(mPlayState);
3145 0 : }
3146 :
3147 0 : void MediaDecoderStateMachine::SetVideoDecodeMode(VideoDecodeMode aMode)
3148 : {
3149 0 : nsCOMPtr<nsIRunnable> r = NewRunnableMethod<VideoDecodeMode>(
3150 : "MediaDecoderStateMachine::SetVideoDecodeModeInternal",
3151 : this,
3152 : &MediaDecoderStateMachine::SetVideoDecodeModeInternal,
3153 0 : aMode);
3154 0 : OwnerThread()->DispatchStateChange(r.forget());
3155 0 : }
3156 :
3157 0 : void MediaDecoderStateMachine::SetVideoDecodeModeInternal(VideoDecodeMode aMode)
3158 : {
3159 0 : MOZ_ASSERT(OnTaskQueue());
3160 0 : LOG("VideoDecodeModeChanged: VideoDecodeMode=(%s->%s), mVideoDecodeSuspended=%c",
3161 : mVideoDecodeMode == VideoDecodeMode::Normal ? "Normal" : "Suspend",
3162 : aMode == VideoDecodeMode::Normal ? "Normal" : "Suspend",
3163 : mVideoDecodeSuspended ? 'T' : 'F');
3164 :
3165 0 : if (!MediaPrefs::MDSMSuspendBackgroundVideoEnabled()) {
3166 0 : return;
3167 : }
3168 :
3169 0 : if (aMode == mVideoDecodeMode) {
3170 0 : return;
3171 : }
3172 :
3173 : // Set new video decode mode.
3174 0 : mVideoDecodeMode = aMode;
3175 :
3176 : // Start timer to trigger suspended video decoding.
3177 0 : if (mVideoDecodeMode == VideoDecodeMode::Suspend) {
3178 0 : TimeStamp target = TimeStamp::Now() + SuspendBackgroundVideoDelay();
3179 :
3180 0 : RefPtr<MediaDecoderStateMachine> self = this;
3181 0 : mVideoDecodeSuspendTimer.Ensure(target,
3182 0 : [=]() { self->OnSuspendTimerResolved(); },
3183 0 : [] () { MOZ_DIAGNOSTIC_ASSERT(false); });
3184 0 : mOnPlaybackEvent.Notify(MediaEventType::StartVideoSuspendTimer);
3185 0 : return;
3186 : }
3187 :
3188 : // Resuming from suspended decoding
3189 :
3190 : // If suspend timer exists, destroy it.
3191 0 : CancelSuspendTimer();
3192 :
3193 0 : if (mVideoDecodeSuspended) {
3194 0 : const auto target = mMediaSink->IsStarted() ? GetClock() : GetMediaTime();
3195 0 : mStateObj->HandleResumeVideoDecoding(target + detail::RESUME_VIDEO_PREMIUM);
3196 : }
3197 : }
3198 :
3199 0 : void MediaDecoderStateMachine::BufferedRangeUpdated()
3200 : {
3201 0 : MOZ_ASSERT(OnTaskQueue());
3202 :
3203 : // While playing an unseekable stream of unknown duration, mObservedDuration
3204 : // is updated (in AdvanceFrame()) as we play. But if data is being downloaded
3205 : // faster than played, mObserved won't reflect the end of playable data
3206 : // since we haven't played the frame at the end of buffered data. So update
3207 : // mObservedDuration here as new data is downloaded to prevent such a lag.
3208 0 : if (!mBuffered.Ref().IsInvalid()) {
3209 : bool exists;
3210 0 : media::TimeUnit end{mBuffered.Ref().GetEnd(&exists)};
3211 0 : if (exists) {
3212 0 : mObservedDuration = std::max(mObservedDuration.Ref(), end);
3213 : }
3214 : }
3215 0 : }
3216 :
3217 : RefPtr<MediaDecoder::SeekPromise>
3218 0 : MediaDecoderStateMachine::Seek(const SeekTarget& aTarget)
3219 : {
3220 0 : MOZ_ASSERT(OnTaskQueue());
3221 :
3222 : // We need to be able to seek in some way
3223 0 : if (!mMediaSeekable && !mMediaSeekableOnlyInBufferedRanges) {
3224 0 : LOGW("Seek() should not be called on a non-seekable media");
3225 : return MediaDecoder::SeekPromise::CreateAndReject(/* aIgnored = */ true,
3226 0 : __func__);
3227 : }
3228 :
3229 0 : if (aTarget.IsNextFrame() && !HasVideo()) {
3230 0 : LOGW("Ignore a NextFrameSeekTask on a media file without video track.");
3231 : return MediaDecoder::SeekPromise::CreateAndReject(/* aIgnored = */ true,
3232 0 : __func__);
3233 : }
3234 :
3235 0 : MOZ_ASSERT(mDuration.Ref().isSome(), "We should have got duration already");
3236 :
3237 0 : return mStateObj->HandleSeek(aTarget);
3238 : }
3239 :
3240 : RefPtr<MediaDecoder::SeekPromise>
3241 0 : MediaDecoderStateMachine::InvokeSeek(const SeekTarget& aTarget)
3242 : {
3243 : return InvokeAsync(
3244 0 : OwnerThread(), this, __func__,
3245 0 : &MediaDecoderStateMachine::Seek, aTarget);
3246 : }
3247 :
3248 0 : void MediaDecoderStateMachine::StopMediaSink()
3249 : {
3250 0 : MOZ_ASSERT(OnTaskQueue());
3251 0 : if (mMediaSink->IsStarted()) {
3252 0 : LOG("Stop MediaSink");
3253 0 : mAudibleListener.DisconnectIfExists();
3254 :
3255 0 : mMediaSink->Stop();
3256 0 : mMediaSinkAudioPromise.DisconnectIfExists();
3257 0 : mMediaSinkVideoPromise.DisconnectIfExists();
3258 : }
3259 0 : }
3260 :
3261 : void
3262 0 : MediaDecoderStateMachine::RequestAudioData()
3263 : {
3264 0 : MOZ_ASSERT(OnTaskQueue());
3265 0 : MOZ_ASSERT(IsAudioDecoding());
3266 0 : MOZ_ASSERT(!IsRequestingAudioData());
3267 0 : MOZ_ASSERT(!IsWaitingAudioData());
3268 0 : LOGV("Queueing audio task - queued=%" PRIuSIZE ", decoder-queued=%" PRIuSIZE,
3269 : AudioQueue().GetSize(), mReader->SizeOfAudioQueueInFrames());
3270 :
3271 0 : RefPtr<MediaDecoderStateMachine> self = this;
3272 0 : mReader->RequestAudioData()->Then(
3273 0 : OwnerThread(), __func__,
3274 0 : [this, self] (RefPtr<AudioData> aAudio) {
3275 0 : MOZ_ASSERT(aAudio);
3276 0 : mAudioDataRequest.Complete();
3277 : // audio->GetEndTime() is not always mono-increasing in chained ogg.
3278 0 : mDecodedAudioEndTime = std::max(
3279 0 : aAudio->GetEndTime(), mDecodedAudioEndTime);
3280 0 : LOGV("OnAudioDecoded [%" PRId64 ",%" PRId64 "]",
3281 : aAudio->mTime.ToMicroseconds(),
3282 : aAudio->GetEndTime().ToMicroseconds());
3283 0 : mStateObj->HandleAudioDecoded(aAudio);
3284 0 : },
3285 0 : [this, self] (const MediaResult& aError) {
3286 0 : LOGV("OnAudioNotDecoded aError=%" PRIu32, static_cast<uint32_t>(aError.Code()));
3287 0 : mAudioDataRequest.Complete();
3288 0 : switch (aError.Code()) {
3289 : case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
3290 0 : mStateObj->HandleWaitingForAudio();
3291 0 : break;
3292 : case NS_ERROR_DOM_MEDIA_CANCELED:
3293 0 : mStateObj->HandleAudioCanceled();
3294 0 : break;
3295 : case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
3296 0 : mStateObj->HandleEndOfAudio();
3297 0 : break;
3298 : default:
3299 0 : DecodeError(aError);
3300 : }
3301 0 : })->Track(mAudioDataRequest);
3302 0 : }
3303 :
3304 : void
3305 0 : MediaDecoderStateMachine::RequestVideoData(const media::TimeUnit& aCurrentTime)
3306 : {
3307 0 : MOZ_ASSERT(OnTaskQueue());
3308 0 : MOZ_ASSERT(IsVideoDecoding());
3309 0 : MOZ_ASSERT(!IsRequestingVideoData());
3310 0 : MOZ_ASSERT(!IsWaitingVideoData());
3311 0 : LOGV("Queueing video task - queued=%" PRIuSIZE ", decoder-queued=%" PRIoSIZE
3312 : ", stime=%" PRId64,
3313 : VideoQueue().GetSize(), mReader->SizeOfVideoQueueInFrames(),
3314 : aCurrentTime.ToMicroseconds());
3315 :
3316 0 : TimeStamp videoDecodeStartTime = TimeStamp::Now();
3317 0 : RefPtr<MediaDecoderStateMachine> self = this;
3318 0 : mReader->RequestVideoData(aCurrentTime)->Then(
3319 0 : OwnerThread(), __func__,
3320 0 : [this, self, videoDecodeStartTime] (RefPtr<VideoData> aVideo) {
3321 0 : MOZ_ASSERT(aVideo);
3322 0 : mVideoDataRequest.Complete();
3323 : // Handle abnormal or negative timestamps.
3324 0 : mDecodedVideoEndTime = std::max(
3325 0 : mDecodedVideoEndTime, aVideo->GetEndTime());
3326 0 : LOGV("OnVideoDecoded [%" PRId64 ",%" PRId64 "]",
3327 : aVideo->mTime.ToMicroseconds(),
3328 : aVideo->GetEndTime().ToMicroseconds());
3329 0 : mStateObj->HandleVideoDecoded(aVideo, videoDecodeStartTime);
3330 0 : },
3331 0 : [this, self] (const MediaResult& aError) {
3332 0 : LOGV("OnVideoNotDecoded aError=%" PRIu32 , static_cast<uint32_t>(aError.Code()));
3333 0 : mVideoDataRequest.Complete();
3334 0 : switch (aError.Code()) {
3335 : case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
3336 0 : mStateObj->HandleWaitingForVideo();
3337 0 : break;
3338 : case NS_ERROR_DOM_MEDIA_CANCELED:
3339 0 : mStateObj->HandleVideoCanceled();
3340 0 : break;
3341 : case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
3342 0 : mStateObj->HandleEndOfVideo();
3343 0 : break;
3344 : default:
3345 0 : DecodeError(aError);
3346 : }
3347 0 : })->Track(mVideoDataRequest);
3348 0 : }
3349 :
3350 : void
3351 0 : MediaDecoderStateMachine::WaitForData(MediaData::Type aType)
3352 : {
3353 0 : MOZ_ASSERT(OnTaskQueue());
3354 0 : MOZ_ASSERT(aType == MediaData::AUDIO_DATA || aType == MediaData::VIDEO_DATA);
3355 0 : RefPtr<MediaDecoderStateMachine> self = this;
3356 0 : if (aType == MediaData::AUDIO_DATA) {
3357 0 : mReader->WaitForData(MediaData::AUDIO_DATA)->Then(
3358 0 : OwnerThread(), __func__,
3359 0 : [self] (MediaData::Type aType) {
3360 0 : self->mAudioWaitRequest.Complete();
3361 0 : MOZ_ASSERT(aType == MediaData::AUDIO_DATA);
3362 0 : self->mStateObj->HandleAudioWaited(aType);
3363 0 : },
3364 0 : [self] (const WaitForDataRejectValue& aRejection) {
3365 0 : self->mAudioWaitRequest.Complete();
3366 0 : self->DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA);
3367 0 : })->Track(mAudioWaitRequest);
3368 : } else {
3369 0 : mReader->WaitForData(MediaData::VIDEO_DATA)->Then(
3370 0 : OwnerThread(), __func__,
3371 0 : [self] (MediaData::Type aType) {
3372 0 : self->mVideoWaitRequest.Complete();
3373 0 : MOZ_ASSERT(aType == MediaData::VIDEO_DATA);
3374 0 : self->mStateObj->HandleVideoWaited(aType);
3375 0 : },
3376 0 : [self] (const WaitForDataRejectValue& aRejection) {
3377 0 : self->mVideoWaitRequest.Complete();
3378 0 : self->DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA);
3379 0 : })->Track(mVideoWaitRequest);
3380 : }
3381 0 : }
3382 :
3383 : void
3384 0 : MediaDecoderStateMachine::StartMediaSink()
3385 : {
3386 0 : MOZ_ASSERT(OnTaskQueue());
3387 0 : if (!mMediaSink->IsStarted()) {
3388 0 : mAudioCompleted = false;
3389 0 : mMediaSink->Start(GetMediaTime(), Info());
3390 :
3391 0 : auto videoPromise = mMediaSink->OnEnded(TrackInfo::kVideoTrack);
3392 0 : auto audioPromise = mMediaSink->OnEnded(TrackInfo::kAudioTrack);
3393 :
3394 0 : if (audioPromise) {
3395 0 : audioPromise->Then(
3396 0 : OwnerThread(), __func__, this,
3397 : &MediaDecoderStateMachine::OnMediaSinkAudioComplete,
3398 : &MediaDecoderStateMachine::OnMediaSinkAudioError)
3399 0 : ->Track(mMediaSinkAudioPromise);
3400 : }
3401 0 : if (videoPromise) {
3402 0 : videoPromise->Then(
3403 0 : OwnerThread(), __func__, this,
3404 : &MediaDecoderStateMachine::OnMediaSinkVideoComplete,
3405 : &MediaDecoderStateMachine::OnMediaSinkVideoError)
3406 0 : ->Track(mMediaSinkVideoPromise);
3407 : }
3408 : }
3409 0 : }
3410 :
3411 : bool
3412 0 : MediaDecoderStateMachine::HasLowDecodedAudio()
3413 : {
3414 0 : MOZ_ASSERT(OnTaskQueue());
3415 0 : return IsAudioDecoding()
3416 0 : && GetDecodedAudioDuration()
3417 0 : < EXHAUSTED_DATA_MARGIN.MultDouble(mPlaybackRate);
3418 : }
3419 :
3420 : bool
3421 0 : MediaDecoderStateMachine::HasLowDecodedVideo()
3422 : {
3423 0 : MOZ_ASSERT(OnTaskQueue());
3424 0 : return IsVideoDecoding()
3425 0 : && VideoQueue().GetSize() < LOW_VIDEO_FRAMES * mPlaybackRate;
3426 : }
3427 :
3428 : bool
3429 0 : MediaDecoderStateMachine::HasLowDecodedData()
3430 : {
3431 0 : MOZ_ASSERT(OnTaskQueue());
3432 0 : MOZ_ASSERT(mReader->UseBufferingHeuristics());
3433 0 : return HasLowDecodedAudio() || HasLowDecodedVideo();
3434 : }
3435 :
3436 0 : bool MediaDecoderStateMachine::OutOfDecodedAudio()
3437 : {
3438 0 : MOZ_ASSERT(OnTaskQueue());
3439 0 : return IsAudioDecoding() && !AudioQueue().IsFinished()
3440 0 : && AudioQueue().GetSize() == 0
3441 0 : && !mMediaSink->HasUnplayedFrames(TrackInfo::kAudioTrack);
3442 : }
3443 :
3444 : bool
3445 0 : MediaDecoderStateMachine::HasLowBufferedData()
3446 : {
3447 0 : MOZ_ASSERT(OnTaskQueue());
3448 0 : return HasLowBufferedData(detail::LOW_BUFFER_THRESHOLD);
3449 : }
3450 :
3451 : bool
3452 0 : MediaDecoderStateMachine::HasLowBufferedData(const TimeUnit& aThreshold)
3453 : {
3454 0 : MOZ_ASSERT(OnTaskQueue());
3455 :
3456 : // If we don't have a duration, mBuffered is probably not going to have
3457 : // a useful buffered range. Return false here so that we don't get stuck in
3458 : // buffering mode for live streams.
3459 0 : if (Duration().IsInfinite()) {
3460 0 : return false;
3461 : }
3462 :
3463 0 : if (mBuffered.Ref().IsInvalid()) {
3464 0 : return false;
3465 : }
3466 :
3467 : // We are never low in decoded data when we don't have audio/video or have
3468 : // decoded all audio/video samples.
3469 0 : TimeUnit endOfDecodedVideo = (HasVideo() && !VideoQueue().IsFinished())
3470 0 : ? mDecodedVideoEndTime : TimeUnit::FromInfinity();
3471 0 : TimeUnit endOfDecodedAudio = (HasAudio() && !AudioQueue().IsFinished())
3472 0 : ? mDecodedAudioEndTime : TimeUnit::FromInfinity();
3473 :
3474 0 : auto endOfDecodedData = std::min(endOfDecodedVideo, endOfDecodedAudio);
3475 0 : if (Duration() < endOfDecodedData) {
3476 : // Our duration is not up to date. No point buffering.
3477 0 : return false;
3478 : }
3479 :
3480 0 : if (endOfDecodedData.IsInfinite()) {
3481 : // Have decoded all samples. No point buffering.
3482 0 : return false;
3483 : }
3484 :
3485 0 : auto start = endOfDecodedData;
3486 0 : auto end = std::min(GetMediaTime() + aThreshold, Duration());
3487 0 : if (start >= end) {
3488 : // Duration of decoded samples is greater than our threshold.
3489 0 : return false;
3490 : }
3491 0 : media::TimeInterval interval(start, end);
3492 0 : return !mBuffered.Ref().Contains(interval);
3493 : }
3494 :
3495 : void
3496 0 : MediaDecoderStateMachine::DecodeError(const MediaResult& aError)
3497 : {
3498 0 : MOZ_ASSERT(OnTaskQueue());
3499 0 : LOGW("Decode error");
3500 : // Notify the decode error and MediaDecoder will shut down MDSM.
3501 0 : mOnPlaybackErrorEvent.Notify(aError);
3502 0 : }
3503 :
3504 : void
3505 0 : MediaDecoderStateMachine::EnqueueFirstFrameLoadedEvent()
3506 : {
3507 0 : MOZ_ASSERT(OnTaskQueue());
3508 : // Track value of mSentFirstFrameLoadedEvent from before updating it
3509 0 : bool firstFrameBeenLoaded = mSentFirstFrameLoadedEvent;
3510 0 : mSentFirstFrameLoadedEvent = true;
3511 : MediaDecoderEventVisibility visibility =
3512 0 : firstFrameBeenLoaded ? MediaDecoderEventVisibility::Suppressed
3513 0 : : MediaDecoderEventVisibility::Observable;
3514 0 : mFirstFrameLoadedEvent.Notify(
3515 0 : nsAutoPtr<MediaInfo>(new MediaInfo(Info())), visibility);
3516 0 : }
3517 :
3518 : void
3519 0 : MediaDecoderStateMachine::FinishDecodeFirstFrame()
3520 : {
3521 0 : MOZ_ASSERT(OnTaskQueue());
3522 0 : MOZ_ASSERT(!mSentFirstFrameLoadedEvent);
3523 0 : LOG("FinishDecodeFirstFrame");
3524 :
3525 0 : mMediaSink->Redraw(Info().mVideo);
3526 :
3527 0 : LOG("Media duration %" PRId64 ", "
3528 : "transportSeekable=%d, mediaSeekable=%d",
3529 : Duration().ToMicroseconds(), mResource->IsTransportSeekable(),
3530 : mMediaSeekable);
3531 :
3532 : // Get potentially updated metadata
3533 0 : mReader->ReadUpdatedMetadata(mInfo.ptr());
3534 :
3535 0 : EnqueueFirstFrameLoadedEvent();
3536 0 : }
3537 :
3538 : RefPtr<ShutdownPromise>
3539 0 : MediaDecoderStateMachine::BeginShutdown()
3540 : {
3541 0 : return InvokeAsync(OwnerThread(), this, __func__,
3542 0 : &MediaDecoderStateMachine::Shutdown);
3543 : }
3544 :
3545 : RefPtr<ShutdownPromise>
3546 0 : MediaDecoderStateMachine::FinishShutdown()
3547 : {
3548 0 : MOZ_ASSERT(OnTaskQueue());
3549 0 : LOG("Shutting down state machine task queue");
3550 0 : return OwnerThread()->BeginShutdown();
3551 : }
3552 :
3553 : void
3554 0 : MediaDecoderStateMachine::RunStateMachine()
3555 : {
3556 0 : MOZ_ASSERT(OnTaskQueue());
3557 :
3558 0 : mDelayedScheduler.Reset(); // Must happen on state machine task queue.
3559 0 : mDispatchedStateMachine = false;
3560 0 : mStateObj->Step();
3561 0 : }
3562 :
3563 : void
3564 0 : MediaDecoderStateMachine::ResetDecode(TrackSet aTracks)
3565 : {
3566 0 : MOZ_ASSERT(OnTaskQueue());
3567 0 : LOG("MediaDecoderStateMachine::Reset");
3568 :
3569 : // Assert that aTracks specifies to reset the video track because we
3570 : // don't currently support resetting just the audio track.
3571 0 : MOZ_ASSERT(aTracks.contains(TrackInfo::kVideoTrack));
3572 :
3573 0 : if (aTracks.contains(TrackInfo::kVideoTrack)) {
3574 0 : mDecodedVideoEndTime = TimeUnit::Zero();
3575 0 : mVideoCompleted = false;
3576 0 : VideoQueue().Reset();
3577 0 : mVideoDataRequest.DisconnectIfExists();
3578 0 : mVideoWaitRequest.DisconnectIfExists();
3579 : }
3580 :
3581 0 : if (aTracks.contains(TrackInfo::kAudioTrack)) {
3582 0 : mDecodedAudioEndTime = TimeUnit::Zero();
3583 0 : mAudioCompleted = false;
3584 0 : AudioQueue().Reset();
3585 0 : mAudioDataRequest.DisconnectIfExists();
3586 0 : mAudioWaitRequest.DisconnectIfExists();
3587 : }
3588 :
3589 0 : mPlaybackOffset = 0;
3590 :
3591 0 : mReader->ResetDecode(aTracks);
3592 0 : }
3593 :
3594 : media::TimeUnit
3595 0 : MediaDecoderStateMachine::GetClock(TimeStamp* aTimeStamp) const
3596 : {
3597 0 : MOZ_ASSERT(OnTaskQueue());
3598 0 : auto clockTime = mMediaSink->GetPosition(aTimeStamp);
3599 0 : NS_ASSERTION(GetMediaTime() <= clockTime, "Clock should go forwards.");
3600 0 : return clockTime;
3601 : }
3602 :
3603 : void
3604 0 : MediaDecoderStateMachine::UpdatePlaybackPositionPeriodically()
3605 : {
3606 0 : MOZ_ASSERT(OnTaskQueue());
3607 :
3608 0 : if (!IsPlaying()) {
3609 0 : return;
3610 : }
3611 :
3612 : // Cap the current time to the larger of the audio and video end time.
3613 : // This ensures that if we're running off the system clock, we don't
3614 : // advance the clock to after the media end time.
3615 0 : if (VideoEndTime() > TimeUnit::Zero() || AudioEndTime() > TimeUnit::Zero()) {
3616 :
3617 0 : const auto clockTime = GetClock();
3618 : // Skip frames up to the frame at the playback position, and figure out
3619 : // the time remaining until it's time to display the next frame and drop
3620 : // the current frame.
3621 0 : NS_ASSERTION(clockTime >= TimeUnit::Zero(), "Should have positive clock time.");
3622 :
3623 : // These will be non -1 if we've displayed a video frame, or played an audio
3624 : // frame.
3625 0 : auto maxEndTime = std::max(VideoEndTime(), AudioEndTime());
3626 0 : auto t = std::min(clockTime, maxEndTime);
3627 : // FIXME: Bug 1091422 - chained ogg files hit this assertion.
3628 : //MOZ_ASSERT(t >= GetMediaTime());
3629 0 : if (t > GetMediaTime()) {
3630 0 : UpdatePlaybackPosition(t);
3631 : }
3632 : }
3633 : // Note we have to update playback position before releasing the monitor.
3634 : // Otherwise, MediaDecoder::AddOutputStream could kick in when we are outside
3635 : // the monitor and get a staled value from GetCurrentTimeUs() which hits the
3636 : // assertion in GetClock().
3637 :
3638 0 : int64_t delay = std::max<int64_t>(1, AUDIO_DURATION_USECS / mPlaybackRate);
3639 0 : ScheduleStateMachineIn(TimeUnit::FromMicroseconds(delay));
3640 : }
3641 :
3642 : /* static */ const char*
3643 0 : MediaDecoderStateMachine::ToStr(NextFrameStatus aStatus)
3644 : {
3645 0 : switch (aStatus) {
3646 : case MediaDecoderOwner::NEXT_FRAME_AVAILABLE:
3647 0 : return "NEXT_FRAME_AVAILABLE";
3648 : case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE:
3649 0 : return "NEXT_FRAME_UNAVAILABLE";
3650 : case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING:
3651 0 : return "NEXT_FRAME_UNAVAILABLE_BUFFERING";
3652 : case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING:
3653 0 : return "NEXT_FRAME_UNAVAILABLE_SEEKING";
3654 : case MediaDecoderOwner::NEXT_FRAME_UNINITIALIZED:
3655 0 : return "NEXT_FRAME_UNINITIALIZED";
3656 : }
3657 0 : return "UNKNOWN";
3658 : }
3659 :
3660 : void
3661 0 : MediaDecoderStateMachine::UpdateNextFrameStatus(NextFrameStatus aStatus)
3662 : {
3663 0 : MOZ_ASSERT(OnTaskQueue());
3664 0 : if (aStatus != mNextFrameStatus) {
3665 0 : LOG("Changed mNextFrameStatus to %s", ToStr(aStatus));
3666 0 : mNextFrameStatus = aStatus;
3667 : }
3668 0 : }
3669 :
3670 : bool
3671 0 : MediaDecoderStateMachine::CanPlayThrough()
3672 : {
3673 0 : MOZ_ASSERT(OnTaskQueue());
3674 0 : return GetStatistics().CanPlayThrough();
3675 : }
3676 :
3677 : MediaStatistics
3678 0 : MediaDecoderStateMachine::GetStatistics()
3679 : {
3680 0 : MOZ_ASSERT(OnTaskQueue());
3681 : MediaStatistics result;
3682 0 : result.mDownloadRate =
3683 0 : mResource->GetDownloadRate(&result.mDownloadRateReliable);
3684 0 : result.mDownloadPosition = mResource->GetCachedDataEnd(mDecoderPosition);
3685 0 : result.mTotalBytes = mResource->GetLength();
3686 0 : result.mPlaybackRate = mPlaybackBytesPerSecond;
3687 0 : result.mPlaybackRateReliable = mPlaybackRateReliable;
3688 0 : result.mDecoderPosition = mDecoderPosition;
3689 0 : result.mPlaybackPosition = mPlaybackOffset;
3690 0 : return result;
3691 : }
3692 :
3693 : void
3694 0 : MediaDecoderStateMachine::ScheduleStateMachine()
3695 : {
3696 0 : MOZ_ASSERT(OnTaskQueue());
3697 0 : if (mDispatchedStateMachine) {
3698 0 : return;
3699 : }
3700 0 : mDispatchedStateMachine = true;
3701 :
3702 0 : OwnerThread()->Dispatch(
3703 0 : NewRunnableMethod("MediaDecoderStateMachine::RunStateMachine",
3704 : this,
3705 0 : &MediaDecoderStateMachine::RunStateMachine));
3706 : }
3707 :
3708 : void
3709 0 : MediaDecoderStateMachine::ScheduleStateMachineIn(const TimeUnit& aTime)
3710 : {
3711 0 : MOZ_ASSERT(OnTaskQueue()); // mDelayedScheduler.Ensure() may Disconnect()
3712 : // the promise, which must happen on the state
3713 : // machine task queue.
3714 0 : MOZ_ASSERT(aTime > TimeUnit::Zero());
3715 0 : if (mDispatchedStateMachine) {
3716 0 : return;
3717 : }
3718 :
3719 0 : TimeStamp target = TimeStamp::Now() + aTime.ToTimeDuration();
3720 :
3721 : // It is OK to capture 'this' without causing UAF because the callback
3722 : // always happens before shutdown.
3723 0 : RefPtr<MediaDecoderStateMachine> self = this;
3724 0 : mDelayedScheduler.Ensure(target, [self] () {
3725 0 : self->mDelayedScheduler.CompleteRequest();
3726 0 : self->RunStateMachine();
3727 0 : }, [] () {
3728 0 : MOZ_DIAGNOSTIC_ASSERT(false);
3729 0 : });
3730 : }
3731 :
3732 0 : bool MediaDecoderStateMachine::OnTaskQueue() const
3733 : {
3734 0 : return OwnerThread()->IsCurrentThreadIn();
3735 : }
3736 :
3737 0 : bool MediaDecoderStateMachine::IsStateMachineScheduled() const
3738 : {
3739 0 : MOZ_ASSERT(OnTaskQueue());
3740 0 : return mDispatchedStateMachine || mDelayedScheduler.IsScheduled();
3741 : }
3742 :
3743 : void
3744 0 : MediaDecoderStateMachine::SetPlaybackRate(double aPlaybackRate)
3745 : {
3746 0 : MOZ_ASSERT(OnTaskQueue());
3747 0 : MOZ_ASSERT(aPlaybackRate != 0, "Should be handled by MediaDecoder::Pause()");
3748 :
3749 0 : mPlaybackRate = aPlaybackRate;
3750 0 : mMediaSink->SetPlaybackRate(mPlaybackRate);
3751 :
3752 : // Schedule next cycle to check if we can stop prerolling.
3753 0 : ScheduleStateMachine();
3754 0 : }
3755 :
3756 0 : void MediaDecoderStateMachine::PreservesPitchChanged()
3757 : {
3758 0 : MOZ_ASSERT(OnTaskQueue());
3759 0 : mMediaSink->SetPreservesPitch(mPreservesPitch);
3760 0 : }
3761 :
3762 : TimeUnit
3763 0 : MediaDecoderStateMachine::AudioEndTime() const
3764 : {
3765 0 : MOZ_ASSERT(OnTaskQueue());
3766 0 : if (mMediaSink->IsStarted()) {
3767 0 : return mMediaSink->GetEndTime(TrackInfo::kAudioTrack);
3768 : }
3769 0 : return TimeUnit::Zero();
3770 : }
3771 :
3772 : TimeUnit
3773 0 : MediaDecoderStateMachine::VideoEndTime() const
3774 : {
3775 0 : MOZ_ASSERT(OnTaskQueue());
3776 0 : if (mMediaSink->IsStarted()) {
3777 0 : return mMediaSink->GetEndTime(TrackInfo::kVideoTrack);
3778 : }
3779 0 : return TimeUnit::Zero();
3780 : }
3781 :
3782 : void
3783 0 : MediaDecoderStateMachine::OnMediaSinkVideoComplete()
3784 : {
3785 0 : MOZ_ASSERT(OnTaskQueue());
3786 0 : MOZ_ASSERT(HasVideo());
3787 0 : LOG("[%s]", __func__);
3788 :
3789 0 : mMediaSinkVideoPromise.Complete();
3790 0 : mVideoCompleted = true;
3791 0 : ScheduleStateMachine();
3792 0 : }
3793 :
3794 : void
3795 0 : MediaDecoderStateMachine::OnMediaSinkVideoError()
3796 : {
3797 0 : MOZ_ASSERT(OnTaskQueue());
3798 0 : MOZ_ASSERT(HasVideo());
3799 0 : LOGW("[%s]", __func__);
3800 :
3801 0 : mMediaSinkVideoPromise.Complete();
3802 0 : mVideoCompleted = true;
3803 0 : if (HasAudio()) {
3804 0 : return;
3805 : }
3806 0 : DecodeError(MediaResult(NS_ERROR_DOM_MEDIA_MEDIASINK_ERR, __func__));
3807 : }
3808 :
3809 0 : void MediaDecoderStateMachine::OnMediaSinkAudioComplete()
3810 : {
3811 0 : MOZ_ASSERT(OnTaskQueue());
3812 0 : MOZ_ASSERT(HasAudio());
3813 0 : LOG("[%s]", __func__);
3814 :
3815 0 : mMediaSinkAudioPromise.Complete();
3816 0 : mAudioCompleted = true;
3817 : // To notify PlaybackEnded as soon as possible.
3818 0 : ScheduleStateMachine();
3819 :
3820 : // Report OK to Decoder Doctor (to know if issue may have been resolved).
3821 0 : mOnDecoderDoctorEvent.Notify(
3822 0 : DecoderDoctorEvent{DecoderDoctorEvent::eAudioSinkStartup, NS_OK});
3823 0 : }
3824 :
3825 0 : void MediaDecoderStateMachine::OnMediaSinkAudioError(nsresult aResult)
3826 : {
3827 0 : MOZ_ASSERT(OnTaskQueue());
3828 0 : MOZ_ASSERT(HasAudio());
3829 0 : LOGW("[%s]", __func__);
3830 :
3831 0 : mMediaSinkAudioPromise.Complete();
3832 0 : mAudioCompleted = true;
3833 :
3834 : // Result should never be NS_OK in this *error* handler. Report to Dec-Doc.
3835 0 : MOZ_ASSERT(NS_FAILED(aResult));
3836 0 : mOnDecoderDoctorEvent.Notify(
3837 0 : DecoderDoctorEvent{DecoderDoctorEvent::eAudioSinkStartup, aResult});
3838 :
3839 : // Make the best effort to continue playback when there is video.
3840 0 : if (HasVideo()) {
3841 0 : return;
3842 : }
3843 :
3844 : // Otherwise notify media decoder/element about this error for it makes
3845 : // no sense to play an audio-only file without sound output.
3846 0 : DecodeError(MediaResult(NS_ERROR_DOM_MEDIA_MEDIASINK_ERR, __func__));
3847 : }
3848 :
3849 : void
3850 0 : MediaDecoderStateMachine::OnCDMProxyReady(RefPtr<CDMProxy> aProxy)
3851 : {
3852 0 : MOZ_ASSERT(OnTaskQueue());
3853 0 : mCDMProxyPromise.Complete();
3854 0 : mCDMProxy = aProxy;
3855 0 : mReader->SetCDMProxy(aProxy);
3856 0 : mStateObj->HandleCDMProxyReady();
3857 0 : }
3858 :
3859 : void
3860 0 : MediaDecoderStateMachine::OnCDMProxyNotReady()
3861 : {
3862 0 : MOZ_ASSERT(OnTaskQueue());
3863 0 : mCDMProxyPromise.Complete();
3864 0 : }
3865 :
3866 : void
3867 0 : MediaDecoderStateMachine::SetAudioCaptured(bool aCaptured)
3868 : {
3869 0 : MOZ_ASSERT(OnTaskQueue());
3870 :
3871 0 : if (aCaptured == mAudioCaptured) {
3872 0 : return;
3873 : }
3874 :
3875 : // Rest these flags so they are consistent with the status of the sink.
3876 : // TODO: Move these flags into MediaSink to improve cohesion so we don't need
3877 : // to reset these flags when switching MediaSinks.
3878 0 : mAudioCompleted = false;
3879 0 : mVideoCompleted = false;
3880 :
3881 : // Backup current playback parameters.
3882 0 : MediaSink::PlaybackParams params = mMediaSink->GetPlaybackParams();
3883 :
3884 : // Stop and shut down the existing sink.
3885 0 : StopMediaSink();
3886 0 : mMediaSink->Shutdown();
3887 :
3888 : // Create a new sink according to whether audio is captured.
3889 0 : mMediaSink = CreateMediaSink(aCaptured);
3890 :
3891 : // Restore playback parameters.
3892 0 : mMediaSink->SetPlaybackParams(params);
3893 :
3894 0 : mAudioCaptured = aCaptured;
3895 :
3896 : // Don't buffer as much when audio is captured because we don't need to worry
3897 : // about high latency audio devices.
3898 0 : mAmpleAudioThreshold = mAudioCaptured
3899 0 : ? detail::AMPLE_AUDIO_THRESHOLD / 2 : detail::AMPLE_AUDIO_THRESHOLD;
3900 :
3901 0 : mStateObj->HandleAudioCaptured();
3902 : }
3903 :
3904 0 : uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const
3905 : {
3906 0 : MOZ_ASSERT(OnTaskQueue());
3907 0 : return (mReader->IsAsync() && mReader->VideoIsHardwareAccelerated())
3908 0 : ? std::max<uint32_t>(sVideoQueueHWAccelSize, MIN_VIDEO_QUEUE_SIZE)
3909 0 : : std::max<uint32_t>(sVideoQueueDefaultSize, MIN_VIDEO_QUEUE_SIZE);
3910 : }
3911 :
3912 : nsCString
3913 0 : MediaDecoderStateMachine::GetDebugInfo()
3914 : {
3915 0 : MOZ_ASSERT(OnTaskQueue());
3916 0 : return nsPrintfCString(
3917 : "MediaDecoderStateMachine State: GetMediaTime=%" PRId64 " GetClock="
3918 : "%" PRId64 " mMediaSink=%p state=%s mPlayState=%d "
3919 : "mSentFirstFrameLoadedEvent=%d IsPlaying=%d mAudioStatus=%s "
3920 : "mVideoStatus=%s mDecodedAudioEndTime=%" PRId64
3921 : " mDecodedVideoEndTime=%" PRId64 "mAudioCompleted=%d "
3922 : "mVideoCompleted=%d",
3923 0 : GetMediaTime().ToMicroseconds(),
3924 0 : mMediaSink->IsStarted() ? GetClock().ToMicroseconds() : -1,
3925 0 : mMediaSink.get(), ToStateStr(), mPlayState.Ref(),
3926 0 : mSentFirstFrameLoadedEvent, IsPlaying(), AudioRequestStatus(),
3927 : VideoRequestStatus(), mDecodedAudioEndTime.ToMicroseconds(),
3928 : mDecodedVideoEndTime.ToMicroseconds(),
3929 0 : mAudioCompleted, mVideoCompleted)
3930 0 : + mStateObj->GetDebugInfo() + nsCString("\n")
3931 0 : + mMediaSink->GetDebugInfo();
3932 : }
3933 :
3934 : RefPtr<MediaDecoder::DebugInfoPromise>
3935 0 : MediaDecoderStateMachine::RequestDebugInfo()
3936 : {
3937 : using PromiseType = MediaDecoder::DebugInfoPromise;
3938 0 : RefPtr<PromiseType::Private> p = new PromiseType::Private(__func__);
3939 0 : RefPtr<MediaDecoderStateMachine> self = this;
3940 0 : OwnerThread()->Dispatch(
3941 0 : NS_NewRunnableFunction(
3942 : "MediaDecoderStateMachine::RequestDebugInfo",
3943 0 : [self, p]() { p->Resolve(self->GetDebugInfo(), __func__); }),
3944 : AbstractThread::AssertDispatchSuccess,
3945 0 : AbstractThread::TailDispatch);
3946 0 : return p.forget();
3947 : }
3948 :
3949 0 : void MediaDecoderStateMachine::AddOutputStream(ProcessedMediaStream* aStream,
3950 : bool aFinishWhenEnded)
3951 : {
3952 0 : MOZ_ASSERT(NS_IsMainThread());
3953 0 : LOG("AddOutputStream aStream=%p!", aStream);
3954 0 : mOutputStreamManager->Add(aStream, aFinishWhenEnded);
3955 : nsCOMPtr<nsIRunnable> r =
3956 0 : NewRunnableMethod<bool>("MediaDecoderStateMachine::SetAudioCaptured",
3957 : this,
3958 : &MediaDecoderStateMachine::SetAudioCaptured,
3959 0 : true);
3960 0 : OwnerThread()->Dispatch(r.forget());
3961 0 : }
3962 :
3963 0 : void MediaDecoderStateMachine::RemoveOutputStream(MediaStream* aStream)
3964 : {
3965 0 : MOZ_ASSERT(NS_IsMainThread());
3966 0 : LOG("RemoveOutputStream=%p!", aStream);
3967 0 : mOutputStreamManager->Remove(aStream);
3968 0 : if (mOutputStreamManager->IsEmpty()) {
3969 : nsCOMPtr<nsIRunnable> r =
3970 0 : NewRunnableMethod<bool>("MediaDecoderStateMachine::SetAudioCaptured",
3971 : this,
3972 : &MediaDecoderStateMachine::SetAudioCaptured,
3973 0 : false);
3974 0 : OwnerThread()->Dispatch(r.forget());
3975 : }
3976 0 : }
3977 :
3978 : size_t
3979 0 : MediaDecoderStateMachine::SizeOfVideoQueue() const
3980 : {
3981 0 : return mReader->SizeOfVideoQueueInBytes();
3982 : }
3983 :
3984 : size_t
3985 0 : MediaDecoderStateMachine::SizeOfAudioQueue() const
3986 : {
3987 0 : return mReader->SizeOfAudioQueueInBytes();
3988 : }
3989 :
3990 : AbstractCanonical<media::TimeIntervals>*
3991 0 : MediaDecoderStateMachine::CanonicalBuffered() const
3992 : {
3993 0 : return mReader->CanonicalBuffered();
3994 : }
3995 :
3996 : MediaEventSource<void>&
3997 0 : MediaDecoderStateMachine::OnMediaNotSeekable() const
3998 : {
3999 0 : return mReader->OnMediaNotSeekable();
4000 : }
4001 :
4002 : const char*
4003 0 : MediaDecoderStateMachine::AudioRequestStatus() const
4004 : {
4005 0 : MOZ_ASSERT(OnTaskQueue());
4006 0 : if (IsRequestingAudioData()) {
4007 0 : MOZ_DIAGNOSTIC_ASSERT(!IsWaitingAudioData());
4008 0 : return "pending";
4009 0 : } else if (IsWaitingAudioData()) {
4010 0 : return "waiting";
4011 : }
4012 0 : return "idle";
4013 : }
4014 :
4015 : const char*
4016 0 : MediaDecoderStateMachine::VideoRequestStatus() const
4017 : {
4018 0 : MOZ_ASSERT(OnTaskQueue());
4019 0 : if (IsRequestingVideoData()) {
4020 0 : MOZ_DIAGNOSTIC_ASSERT(!IsWaitingVideoData());
4021 0 : return "pending";
4022 0 : } else if (IsWaitingVideoData()) {
4023 0 : return "waiting";
4024 : }
4025 0 : return "idle";
4026 : }
4027 :
4028 : void
4029 0 : MediaDecoderStateMachine::OnSuspendTimerResolved()
4030 : {
4031 0 : LOG("OnSuspendTimerResolved");
4032 0 : mVideoDecodeSuspendTimer.CompleteRequest();
4033 0 : mStateObj->HandleVideoSuspendTimeout();
4034 0 : }
4035 :
4036 : void
4037 0 : MediaDecoderStateMachine::CancelSuspendTimer()
4038 : {
4039 0 : LOG("CancelSuspendTimer: State: %s, Timer.IsScheduled: %c",
4040 : ToStateStr(mStateObj->GetState()),
4041 : mVideoDecodeSuspendTimer.IsScheduled() ? 'T' : 'F');
4042 0 : MOZ_ASSERT(OnTaskQueue());
4043 0 : if (mVideoDecodeSuspendTimer.IsScheduled()) {
4044 0 : mOnPlaybackEvent.Notify(MediaEventType::CancelVideoSuspendTimer);
4045 : }
4046 0 : mVideoDecodeSuspendTimer.Reset();
4047 0 : }
4048 :
4049 : } // namespace mozilla
4050 :
4051 : // avoid redefined macro in unified build
4052 : #undef LOG
4053 : #undef LOGV
4054 : #undef LOGW
4055 : #undef SLOGW
4056 : #undef NS_DispatchToMainThread
|
