Line data Source code
1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 : /* This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this
5 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #include "MediaQueue.h"
8 : #include "VideoSink.h"
9 : #include "MediaPrefs.h"
10 :
11 : #include "mozilla/IntegerPrintfMacros.h"
12 : #include "mozilla/SizePrintfMacros.h"
13 :
14 : namespace mozilla {
15 :
16 : extern LazyLogModule gMediaDecoderLog;
17 :
18 : #undef FMT
19 :
20 : #define FMT(x, ...) "VideoSink=%p " x, this, ##__VA_ARGS__
21 : #define VSINK_LOG(x, ...) MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, (FMT(x, ##__VA_ARGS__)))
22 : #define VSINK_LOG_V(x, ...) MOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, (FMT(x, ##__VA_ARGS__)))
23 :
24 : using namespace mozilla::layers;
25 :
26 : namespace media {
27 :
28 : // Minimum update frequency is 1/120th of a second, i.e. half the
29 : // duration of a 60-fps frame.
30 : static const int64_t MIN_UPDATE_INTERVAL_US = 1000000 / (60 * 2);
31 :
32 0 : VideoSink::VideoSink(AbstractThread* aThread,
33 : MediaSink* aAudioSink,
34 : MediaQueue<VideoData>& aVideoQueue,
35 : VideoFrameContainer* aContainer,
36 : FrameStatistics& aFrameStats,
37 0 : uint32_t aVQueueSentToCompositerSize)
38 : : mOwnerThread(aThread)
39 : , mAudioSink(aAudioSink)
40 : , mVideoQueue(aVideoQueue)
41 : , mContainer(aContainer)
42 0 : , mProducerID(ImageContainer::AllocateProducerID())
43 : , mFrameStats(aFrameStats)
44 : , mHasVideo(false)
45 : , mUpdateScheduler(aThread)
46 : , mVideoQueueSendToCompositorSize(aVQueueSentToCompositerSize)
47 0 : , mMinVideoQueueSize(MediaPrefs::RuinAvSync() ? 1 : 0)
48 : {
49 0 : MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
50 0 : }
51 :
52 0 : VideoSink::~VideoSink()
53 : {
54 0 : }
55 :
56 : const MediaSink::PlaybackParams&
57 0 : VideoSink::GetPlaybackParams() const
58 : {
59 0 : AssertOwnerThread();
60 :
61 0 : return mAudioSink->GetPlaybackParams();
62 : }
63 :
64 : void
65 0 : VideoSink::SetPlaybackParams(const PlaybackParams& aParams)
66 : {
67 0 : AssertOwnerThread();
68 :
69 0 : mAudioSink->SetPlaybackParams(aParams);
70 0 : }
71 :
72 : RefPtr<GenericPromise>
73 0 : VideoSink::OnEnded(TrackType aType)
74 : {
75 0 : AssertOwnerThread();
76 0 : MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
77 :
78 0 : if (aType == TrackInfo::kAudioTrack) {
79 0 : return mAudioSink->OnEnded(aType);
80 0 : } else if (aType == TrackInfo::kVideoTrack) {
81 0 : return mEndPromise;
82 : }
83 0 : return nullptr;
84 : }
85 :
86 : TimeUnit
87 0 : VideoSink::GetEndTime(TrackType aType) const
88 : {
89 0 : AssertOwnerThread();
90 0 : MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
91 :
92 0 : if (aType == TrackInfo::kVideoTrack) {
93 0 : return mVideoFrameEndTime;
94 0 : } else if (aType == TrackInfo::kAudioTrack) {
95 0 : return mAudioSink->GetEndTime(aType);
96 : }
97 0 : return TimeUnit::Zero();
98 : }
99 :
100 : TimeUnit
101 0 : VideoSink::GetPosition(TimeStamp* aTimeStamp) const
102 : {
103 0 : AssertOwnerThread();
104 0 : return mAudioSink->GetPosition(aTimeStamp);
105 : }
106 :
107 : bool
108 0 : VideoSink::HasUnplayedFrames(TrackType aType) const
109 : {
110 0 : AssertOwnerThread();
111 0 : MOZ_ASSERT(aType == TrackInfo::kAudioTrack, "Not implemented for non audio tracks.");
112 :
113 0 : return mAudioSink->HasUnplayedFrames(aType);
114 : }
115 :
116 : void
117 0 : VideoSink::SetPlaybackRate(double aPlaybackRate)
118 : {
119 0 : AssertOwnerThread();
120 :
121 0 : mAudioSink->SetPlaybackRate(aPlaybackRate);
122 0 : }
123 :
124 : void
125 0 : VideoSink::SetVolume(double aVolume)
126 : {
127 0 : AssertOwnerThread();
128 :
129 0 : mAudioSink->SetVolume(aVolume);
130 0 : }
131 :
132 : void
133 0 : VideoSink::SetPreservesPitch(bool aPreservesPitch)
134 : {
135 0 : AssertOwnerThread();
136 :
137 0 : mAudioSink->SetPreservesPitch(aPreservesPitch);
138 0 : }
139 :
140 : void
141 0 : VideoSink::SetPlaying(bool aPlaying)
142 : {
143 0 : AssertOwnerThread();
144 0 : VSINK_LOG_V(" playing (%d) -> (%d)", mAudioSink->IsPlaying(), aPlaying);
145 :
146 0 : if (!aPlaying) {
147 : // Reset any update timer if paused.
148 0 : mUpdateScheduler.Reset();
149 : // Since playback is paused, tell compositor to render only current frame.
150 0 : RenderVideoFrames(1);
151 0 : if (mContainer) {
152 0 : mContainer->ClearCachedResources();
153 : }
154 : }
155 :
156 0 : mAudioSink->SetPlaying(aPlaying);
157 :
158 0 : if (mHasVideo && aPlaying) {
159 : // There's no thread in VideoSink for pulling video frames, need to trigger
160 : // rendering while becoming playing status. because the VideoQueue may be
161 : // full already.
162 0 : TryUpdateRenderedVideoFrames();
163 : }
164 0 : }
165 :
166 : void
167 0 : VideoSink::Start(const TimeUnit& aStartTime, const MediaInfo& aInfo)
168 : {
169 0 : AssertOwnerThread();
170 0 : VSINK_LOG("[%s]", __func__);
171 :
172 0 : mAudioSink->Start(aStartTime, aInfo);
173 :
174 0 : mHasVideo = aInfo.HasVideo();
175 :
176 0 : if (mHasVideo) {
177 0 : mEndPromise = mEndPromiseHolder.Ensure(__func__);
178 :
179 : // If the underlying MediaSink has an end promise for the video track (which
180 : // happens when mAudioSink refers to a DecodedStream), we must wait for it
181 : // to complete before resolving our own end promise. Otherwise, MDSM might
182 : // stop playback before DecodedStream plays to the end and cause
183 : // test_streams_element_capture.html to time out.
184 0 : RefPtr<GenericPromise> p = mAudioSink->OnEnded(TrackInfo::kVideoTrack);
185 0 : if (p) {
186 0 : RefPtr<VideoSink> self = this;
187 0 : p->Then(mOwnerThread, __func__,
188 0 : [self] () {
189 0 : self->mVideoSinkEndRequest.Complete();
190 0 : self->TryUpdateRenderedVideoFrames();
191 : // It is possible the video queue size is 0 and we have no frames to
192 : // render. However, we need to call MaybeResolveEndPromise() to ensure
193 : // mEndPromiseHolder is resolved.
194 0 : self->MaybeResolveEndPromise();
195 0 : }, [self] () {
196 0 : self->mVideoSinkEndRequest.Complete();
197 0 : self->TryUpdateRenderedVideoFrames();
198 0 : self->MaybeResolveEndPromise();
199 0 : })
200 0 : ->Track(mVideoSinkEndRequest);
201 : }
202 :
203 0 : ConnectListener();
204 : // Run the render loop at least once so we can resolve the end promise
205 : // when video duration is 0.
206 0 : UpdateRenderedVideoFrames();
207 : }
208 0 : }
209 :
210 : void
211 0 : VideoSink::Stop()
212 : {
213 0 : AssertOwnerThread();
214 0 : MOZ_ASSERT(mAudioSink->IsStarted(), "playback not started.");
215 0 : VSINK_LOG("[%s]", __func__);
216 :
217 0 : mAudioSink->Stop();
218 :
219 0 : mUpdateScheduler.Reset();
220 0 : if (mHasVideo) {
221 0 : DisconnectListener();
222 0 : mVideoSinkEndRequest.DisconnectIfExists();
223 0 : mEndPromiseHolder.ResolveIfExists(true, __func__);
224 0 : mEndPromise = nullptr;
225 : }
226 0 : mVideoFrameEndTime = TimeUnit::Zero();
227 0 : }
228 :
229 : bool
230 0 : VideoSink::IsStarted() const
231 : {
232 0 : AssertOwnerThread();
233 :
234 0 : return mAudioSink->IsStarted();
235 : }
236 :
237 : bool
238 0 : VideoSink::IsPlaying() const
239 : {
240 0 : AssertOwnerThread();
241 :
242 0 : return mAudioSink->IsPlaying();
243 : }
244 :
245 : void
246 0 : VideoSink::Shutdown()
247 : {
248 0 : AssertOwnerThread();
249 0 : MOZ_ASSERT(!mAudioSink->IsStarted(), "must be called after playback stops.");
250 0 : VSINK_LOG("[%s]", __func__);
251 :
252 0 : mAudioSink->Shutdown();
253 0 : }
254 :
255 : void
256 0 : VideoSink::OnVideoQueuePushed(RefPtr<VideoData>&& aSample)
257 : {
258 0 : AssertOwnerThread();
259 : // Listen to push event, VideoSink should try rendering ASAP if first frame
260 : // arrives but update scheduler is not triggered yet.
261 0 : if (!aSample->IsSentToCompositor()) {
262 : // Since we push rendered frames back to the queue, we will receive
263 : // push events for them. We only need to trigger render loop
264 : // when this frame is not rendered yet.
265 0 : TryUpdateRenderedVideoFrames();
266 : }
267 0 : }
268 :
269 : void
270 0 : VideoSink::OnVideoQueueFinished()
271 : {
272 0 : AssertOwnerThread();
273 : // Run render loop if the end promise is not resolved yet.
274 0 : if (!mUpdateScheduler.IsScheduled() &&
275 0 : mAudioSink->IsPlaying() &&
276 0 : !mEndPromiseHolder.IsEmpty()) {
277 0 : UpdateRenderedVideoFrames();
278 : }
279 0 : }
280 :
281 : void
282 0 : VideoSink::Redraw(const VideoInfo& aInfo)
283 : {
284 0 : AssertOwnerThread();
285 :
286 : // No video track, nothing to draw.
287 0 : if (!aInfo.IsValid() || !mContainer) {
288 0 : return;
289 : }
290 :
291 0 : RefPtr<VideoData> video = VideoQueue().PeekFront();
292 0 : if (video) {
293 0 : video->MarkSentToCompositor();
294 0 : mContainer->SetCurrentFrame(video->mDisplay, video->mImage, TimeStamp::Now());
295 0 : return;
296 : }
297 :
298 : // When we reach here, it means there are no frames in this video track.
299 : // Draw a blank frame to ensure there is something in the image container
300 : // to fire 'loadeddata'.
301 : RefPtr<Image> blank =
302 0 : mContainer->GetImageContainer()->CreatePlanarYCbCrImage();
303 0 : mContainer->SetCurrentFrame(aInfo.mDisplay, blank, TimeStamp::Now());
304 : }
305 :
306 : void
307 0 : VideoSink::TryUpdateRenderedVideoFrames()
308 : {
309 0 : AssertOwnerThread();
310 0 : if (mUpdateScheduler.IsScheduled() || !mAudioSink->IsPlaying()) {
311 0 : return;
312 : }
313 0 : RefPtr<VideoData> v = VideoQueue().PeekFront();
314 0 : if (!v) {
315 : // No frames to render.
316 0 : return;
317 : }
318 :
319 0 : TimeStamp nowTime;
320 0 : const TimeUnit clockTime = mAudioSink->GetPosition(&nowTime);
321 0 : if (clockTime >= v->mTime) {
322 : // Time to render this frame.
323 0 : UpdateRenderedVideoFrames();
324 0 : return;
325 : }
326 :
327 : // If we send this future frame to the compositor now, it will be rendered
328 : // immediately and break A/V sync. Instead, we schedule a timer to send it
329 : // later.
330 0 : int64_t delta = (v->mTime - clockTime).ToMicroseconds() /
331 0 : mAudioSink->GetPlaybackParams().mPlaybackRate;
332 0 : TimeStamp target = nowTime + TimeDuration::FromMicroseconds(delta);
333 0 : RefPtr<VideoSink> self = this;
334 0 : mUpdateScheduler.Ensure(
335 : target,
336 0 : [self]() { self->UpdateRenderedVideoFramesByTimer(); },
337 0 : [self]() { self->UpdateRenderedVideoFramesByTimer(); });
338 : }
339 :
340 : void
341 0 : VideoSink::UpdateRenderedVideoFramesByTimer()
342 : {
343 0 : AssertOwnerThread();
344 0 : mUpdateScheduler.CompleteRequest();
345 0 : UpdateRenderedVideoFrames();
346 0 : }
347 :
348 : void
349 0 : VideoSink::ConnectListener()
350 : {
351 0 : AssertOwnerThread();
352 0 : mPushListener = VideoQueue().PushEvent().Connect(
353 0 : mOwnerThread, this, &VideoSink::OnVideoQueuePushed);
354 0 : mFinishListener = VideoQueue().FinishEvent().Connect(
355 0 : mOwnerThread, this, &VideoSink::OnVideoQueueFinished);
356 0 : }
357 :
358 : void
359 0 : VideoSink::DisconnectListener()
360 : {
361 0 : AssertOwnerThread();
362 0 : mPushListener.Disconnect();
363 0 : mFinishListener.Disconnect();
364 0 : }
365 :
366 : void
367 0 : VideoSink::RenderVideoFrames(int32_t aMaxFrames,
368 : int64_t aClockTime,
369 : const TimeStamp& aClockTimeStamp)
370 : {
371 0 : AssertOwnerThread();
372 :
373 0 : AutoTArray<RefPtr<VideoData>,16> frames;
374 0 : VideoQueue().GetFirstElements(aMaxFrames, &frames);
375 0 : if (frames.IsEmpty() || !mContainer) {
376 0 : return;
377 : }
378 :
379 0 : AutoTArray<ImageContainer::NonOwningImage,16> images;
380 0 : TimeStamp lastFrameTime;
381 0 : MediaSink::PlaybackParams params = mAudioSink->GetPlaybackParams();
382 0 : for (uint32_t i = 0; i < frames.Length(); ++i) {
383 0 : VideoData* frame = frames[i];
384 :
385 0 : frame->MarkSentToCompositor();
386 :
387 0 : if (!frame->mImage || !frame->mImage->IsValid() ||
388 0 : !frame->mImage->GetSize().width || !frame->mImage->GetSize().height) {
389 0 : continue;
390 : }
391 :
392 0 : if (frame->mTime.IsNegative()) {
393 : // Frame times before the start time are invalid; drop such frames
394 0 : continue;
395 : }
396 :
397 0 : TimeStamp t;
398 0 : if (aMaxFrames > 1) {
399 0 : MOZ_ASSERT(!aClockTimeStamp.IsNull());
400 0 : int64_t delta = frame->mTime.ToMicroseconds() - aClockTime;
401 : t = aClockTimeStamp +
402 0 : TimeDuration::FromMicroseconds(delta / params.mPlaybackRate);
403 0 : if (!lastFrameTime.IsNull() && t <= lastFrameTime) {
404 : // Timestamps out of order; drop the new frame. In theory we should
405 : // probably replace the previous frame with the new frame if the
406 : // timestamps are equal, but this is a corrupt video file already so
407 : // never mind.
408 0 : continue;
409 : }
410 0 : lastFrameTime = t;
411 : }
412 :
413 0 : ImageContainer::NonOwningImage* img = images.AppendElement();
414 0 : img->mTimeStamp = t;
415 0 : img->mImage = frame->mImage;
416 0 : img->mFrameID = frame->mFrameID;
417 0 : img->mProducerID = mProducerID;
418 :
419 0 : VSINK_LOG_V("playing video frame %" PRId64 " (id=%x) (vq-queued=%" PRIuSIZE ")",
420 : frame->mTime.ToMicroseconds(), frame->mFrameID,
421 : VideoQueue().GetSize());
422 : }
423 :
424 0 : if (images.Length() > 0) {
425 0 : mContainer->SetCurrentFrames(frames[0]->mDisplay, images);
426 : }
427 : }
428 :
429 : void
430 0 : VideoSink::UpdateRenderedVideoFrames()
431 : {
432 0 : AssertOwnerThread();
433 0 : MOZ_ASSERT(mAudioSink->IsPlaying(), "should be called while playing.");
434 :
435 : // Get the current playback position.
436 0 : TimeStamp nowTime;
437 0 : const auto clockTime = mAudioSink->GetPosition(&nowTime);
438 0 : MOZ_ASSERT(!clockTime.IsNegative(), "Should have positive clock time.");
439 :
440 : // Skip frames up to the playback position.
441 0 : TimeUnit lastFrameEndTime;
442 0 : while (VideoQueue().GetSize() > mMinVideoQueueSize &&
443 0 : clockTime >= VideoQueue().PeekFront()->GetEndTime()) {
444 0 : RefPtr<VideoData> frame = VideoQueue().PopFront();
445 0 : lastFrameEndTime = frame->GetEndTime();
446 0 : if (frame->IsSentToCompositor()) {
447 0 : mFrameStats.NotifyPresentedFrame();
448 : } else {
449 0 : mFrameStats.NotifyDecodedFrames({ 0, 0, 1 });
450 0 : VSINK_LOG_V("discarding video frame mTime=%" PRId64 " clock_time=%" PRId64,
451 : frame->mTime.ToMicroseconds(), clockTime.ToMicroseconds());
452 : }
453 : }
454 :
455 : // The presentation end time of the last video frame displayed is either
456 : // the end time of the current frame, or if we dropped all frames in the
457 : // queue, the end time of the last frame we removed from the queue.
458 0 : RefPtr<VideoData> currentFrame = VideoQueue().PeekFront();
459 0 : mVideoFrameEndTime = std::max(mVideoFrameEndTime,
460 0 : currentFrame ? currentFrame->GetEndTime() : lastFrameEndTime);
461 :
462 0 : MaybeResolveEndPromise();
463 :
464 0 : RenderVideoFrames(
465 0 : mVideoQueueSendToCompositorSize,
466 0 : clockTime.ToMicroseconds(), nowTime);
467 :
468 : // Get the timestamp of the next frame. Schedule the next update at
469 : // the start time of the next frame. If we don't have a next frame,
470 : // we will run render loops again upon incoming frames.
471 0 : nsTArray<RefPtr<VideoData>> frames;
472 0 : VideoQueue().GetFirstElements(2, &frames);
473 0 : if (frames.Length() < 2) {
474 0 : return;
475 : }
476 :
477 0 : int64_t nextFrameTime = frames[1]->mTime.ToMicroseconds();
478 : int64_t delta = std::max(
479 0 : nextFrameTime - clockTime.ToMicroseconds(), MIN_UPDATE_INTERVAL_US);
480 0 : TimeStamp target = nowTime + TimeDuration::FromMicroseconds(
481 0 : delta / mAudioSink->GetPlaybackParams().mPlaybackRate);
482 :
483 0 : RefPtr<VideoSink> self = this;
484 0 : mUpdateScheduler.Ensure(target, [self] () {
485 0 : self->UpdateRenderedVideoFramesByTimer();
486 0 : }, [self] () {
487 0 : self->UpdateRenderedVideoFramesByTimer();
488 0 : });
489 : }
490 :
491 : void
492 0 : VideoSink::MaybeResolveEndPromise()
493 : {
494 0 : AssertOwnerThread();
495 : // All frames are rendered, Let's resolve the promise.
496 0 : if (VideoQueue().IsFinished() &&
497 0 : VideoQueue().GetSize() <= 1 &&
498 0 : !mVideoSinkEndRequest.Exists()) {
499 0 : mEndPromiseHolder.ResolveIfExists(true, __func__);
500 : }
501 0 : }
502 :
503 : nsCString
504 0 : VideoSink::GetDebugInfo()
505 : {
506 0 : AssertOwnerThread();
507 0 : return nsPrintfCString(
508 : "VideoSink Status: IsStarted=%d IsPlaying=%d VideoQueue(finished=%d "
509 : "size=%" PRIuSIZE ") mVideoFrameEndTime=%" PRId64 " mHasVideo=%d "
510 : "mVideoSinkEndRequest.Exists()=%d mEndPromiseHolder.IsEmpty()=%d\n",
511 0 : IsStarted(), IsPlaying(), VideoQueue().IsFinished(),
512 0 : VideoQueue().GetSize(), mVideoFrameEndTime.ToMicroseconds(), mHasVideo,
513 0 : mVideoSinkEndRequest.Exists(), mEndPromiseHolder.IsEmpty())
514 0 : + mAudioSink->GetDebugInfo();
515 : }
516 :
517 : } // namespace media
518 : } // namespace mozilla
|
