Line data Source code
1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 : /* This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
5 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #include "mozilla/dom/TextTrackManager.h"
8 : #include "mozilla/dom/HTMLMediaElement.h"
9 : #include "mozilla/dom/HTMLTrackElement.h"
10 : #include "mozilla/dom/HTMLVideoElement.h"
11 : #include "mozilla/dom/TextTrack.h"
12 : #include "mozilla/dom/TextTrackCue.h"
13 : #include "mozilla/dom/Event.h"
14 : #include "mozilla/ClearOnShutdown.h"
15 : #include "mozilla/SizePrintfMacros.h"
16 : #include "mozilla/Telemetry.h"
17 : #include "nsComponentManagerUtils.h"
18 : #include "nsGlobalWindow.h"
19 : #include "nsVariant.h"
20 : #include "nsVideoFrame.h"
21 : #include "nsIFrame.h"
22 : #include "nsTArrayHelpers.h"
23 : #include "nsIWebVTTParserWrapper.h"
24 :
25 :
26 : static mozilla::LazyLogModule gTextTrackLog("TextTrackManager");
27 : #define WEBVTT_LOG(...) MOZ_LOG(gTextTrackLog, LogLevel::Debug, (__VA_ARGS__))
28 : #define WEBVTT_LOGV(...) MOZ_LOG(gTextTrackLog, LogLevel::Verbose, (__VA_ARGS__))
29 :
30 : namespace mozilla {
31 : namespace dom {
32 :
33 0 : NS_IMPL_ISUPPORTS(TextTrackManager::ShutdownObserverProxy, nsIObserver);
34 :
35 0 : CompareTextTracks::CompareTextTracks(HTMLMediaElement* aMediaElement)
36 : {
37 0 : mMediaElement = aMediaElement;
38 0 : }
39 :
40 : int32_t
41 0 : CompareTextTracks::TrackChildPosition(TextTrack* aTextTrack) const {
42 0 : MOZ_DIAGNOSTIC_ASSERT(aTextTrack);
43 0 : HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
44 0 : if (!trackElement) {
45 0 : return -1;
46 : }
47 0 : return mMediaElement->IndexOf(trackElement);
48 : }
49 :
50 : bool
51 0 : CompareTextTracks::Equals(TextTrack* aOne, TextTrack* aTwo) const {
52 : // Two tracks can never be equal. If they have corresponding TrackElements
53 : // they would need to occupy the same tree position (impossible) and in the
54 : // case of tracks coming from AddTextTrack source we put the newest at the
55 : // last position, so they won't be equal as well.
56 0 : return false;
57 : }
58 :
59 : bool
60 0 : CompareTextTracks::LessThan(TextTrack* aOne, TextTrack* aTwo) const
61 : {
62 : // Protect against nullptr TextTrack objects; treat them as
63 : // sorting toward the end.
64 0 : if (!aOne) {
65 0 : return false;
66 : }
67 0 : if (!aTwo) {
68 0 : return true;
69 : }
70 0 : TextTrackSource sourceOne = aOne->GetTextTrackSource();
71 0 : TextTrackSource sourceTwo = aTwo->GetTextTrackSource();
72 0 : if (sourceOne != sourceTwo) {
73 0 : return sourceOne == TextTrackSource::Track ||
74 0 : (sourceOne == AddTextTrack && sourceTwo == MediaResourceSpecific);
75 : }
76 0 : switch (sourceOne) {
77 : case Track: {
78 0 : int32_t positionOne = TrackChildPosition(aOne);
79 0 : int32_t positionTwo = TrackChildPosition(aTwo);
80 : // If either position one or positiontwo are -1 then something has gone
81 : // wrong. In this case we should just put them at the back of the list.
82 0 : return positionOne != -1 && positionTwo != -1 &&
83 0 : positionOne < positionTwo;
84 : }
85 : case AddTextTrack:
86 : // For AddTextTrack sources the tracks will already be in the correct relative
87 : // order in the source array. Assume we're called in iteration order and can
88 : // therefore always report aOne < aTwo to maintain the original temporal ordering.
89 0 : return true;
90 : case MediaResourceSpecific:
91 : // No rules for Media Resource Specific tracks yet.
92 0 : break;
93 : }
94 0 : return true;
95 : }
96 :
97 0 : NS_IMPL_CYCLE_COLLECTION(TextTrackManager, mMediaElement, mTextTracks,
98 : mPendingTextTracks, mNewCues,
99 : mLastActiveCues)
100 :
101 0 : NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackManager)
102 0 : NS_INTERFACE_MAP_ENTRY(nsIDOMEventListener)
103 0 : NS_INTERFACE_MAP_END
104 :
105 0 : NS_IMPL_CYCLE_COLLECTING_ADDREF(TextTrackManager)
106 0 : NS_IMPL_CYCLE_COLLECTING_RELEASE(TextTrackManager)
107 :
108 3 : StaticRefPtr<nsIWebVTTParserWrapper> TextTrackManager::sParserWrapper;
109 :
110 0 : TextTrackManager::TextTrackManager(HTMLMediaElement *aMediaElement)
111 : : mMediaElement(aMediaElement)
112 : , mHasSeeked(false)
113 : , mLastTimeMarchesOnCalled(0.0)
114 : , mTimeMarchesOnDispatched(false)
115 : , mUpdateCueDisplayDispatched(false)
116 : , performedTrackSelection(false)
117 : , mCueTelemetryReported(false)
118 0 : , mShutdown(false)
119 : {
120 : nsISupports* parentObject =
121 0 : mMediaElement->OwnerDoc()->GetParentObject();
122 :
123 0 : NS_ENSURE_TRUE_VOID(parentObject);
124 0 : WEBVTT_LOG("%p Create TextTrackManager",this);
125 0 : nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
126 0 : mNewCues = new TextTrackCueList(window);
127 0 : mLastActiveCues = new TextTrackCueList(window);
128 0 : mTextTracks = new TextTrackList(window, this);
129 0 : mPendingTextTracks = new TextTrackList(window, this);
130 :
131 0 : if (!sParserWrapper) {
132 : nsCOMPtr<nsIWebVTTParserWrapper> parserWrapper =
133 0 : do_CreateInstance(NS_WEBVTTPARSERWRAPPER_CONTRACTID);
134 0 : sParserWrapper = parserWrapper;
135 0 : ClearOnShutdown(&sParserWrapper);
136 : }
137 0 : mShutdownProxy = new ShutdownObserverProxy(this);
138 : }
139 :
140 0 : TextTrackManager::~TextTrackManager()
141 : {
142 0 : WEBVTT_LOG("%p ~TextTrackManager",this);
143 0 : nsContentUtils::UnregisterShutdownObserver(mShutdownProxy);
144 0 : }
145 :
146 : TextTrackList*
147 0 : TextTrackManager::GetTextTracks() const
148 : {
149 0 : return mTextTracks;
150 : }
151 :
152 : already_AddRefed<TextTrack>
153 0 : TextTrackManager::AddTextTrack(TextTrackKind aKind, const nsAString& aLabel,
154 : const nsAString& aLanguage,
155 : TextTrackMode aMode,
156 : TextTrackReadyState aReadyState,
157 : TextTrackSource aTextTrackSource)
158 : {
159 0 : if (!mMediaElement || !mTextTracks) {
160 0 : return nullptr;
161 : }
162 0 : WEBVTT_LOG("%p AddTextTrack",this);
163 0 : WEBVTT_LOGV("AddTextTrack kind %" PRIu32 " Label %s Language %s",
164 : static_cast<uint32_t>(aKind),
165 : NS_ConvertUTF16toUTF8(aLabel).get(), NS_ConvertUTF16toUTF8(aLanguage).get());
166 : RefPtr<TextTrack> track =
167 0 : mTextTracks->AddTextTrack(aKind, aLabel, aLanguage, aMode, aReadyState,
168 0 : aTextTrackSource, CompareTextTracks(mMediaElement));
169 0 : AddCues(track);
170 0 : ReportTelemetryForTrack(track);
171 :
172 0 : if (aTextTrackSource == TextTrackSource::Track) {
173 0 : RefPtr<nsIRunnable> task = NewRunnableMethod(
174 : "dom::TextTrackManager::HonorUserPreferencesForTrackSelection",
175 : this,
176 0 : &TextTrackManager::HonorUserPreferencesForTrackSelection);
177 0 : nsContentUtils::RunInStableState(task.forget());
178 : }
179 :
180 0 : return track.forget();
181 : }
182 :
183 : void
184 0 : TextTrackManager::AddTextTrack(TextTrack* aTextTrack)
185 : {
186 0 : if (!mMediaElement || !mTextTracks) {
187 0 : return;
188 : }
189 0 : WEBVTT_LOG("%p AddTextTrack TextTrack %p",this, aTextTrack);
190 0 : mTextTracks->AddTextTrack(aTextTrack, CompareTextTracks(mMediaElement));
191 0 : AddCues(aTextTrack);
192 0 : ReportTelemetryForTrack(aTextTrack);
193 :
194 0 : if (aTextTrack->GetTextTrackSource() == TextTrackSource::Track) {
195 0 : RefPtr<nsIRunnable> task = NewRunnableMethod(
196 : "dom::TextTrackManager::HonorUserPreferencesForTrackSelection",
197 : this,
198 0 : &TextTrackManager::HonorUserPreferencesForTrackSelection);
199 0 : nsContentUtils::RunInStableState(task.forget());
200 : }
201 : }
202 :
203 : void
204 0 : TextTrackManager::AddCues(TextTrack* aTextTrack)
205 : {
206 0 : if (!mNewCues) {
207 0 : WEBVTT_LOG("AddCues mNewCues is null");
208 0 : return;
209 : }
210 :
211 0 : TextTrackCueList* cueList = aTextTrack->GetCues();
212 0 : if (cueList) {
213 : bool dummy;
214 0 : WEBVTT_LOGV("AddCues cueList->Length() %d",cueList->Length());
215 0 : for (uint32_t i = 0; i < cueList->Length(); ++i) {
216 0 : mNewCues->AddCue(*cueList->IndexedGetter(i, dummy));
217 : }
218 0 : DispatchTimeMarchesOn();
219 : }
220 : }
221 :
222 : void
223 0 : TextTrackManager::RemoveTextTrack(TextTrack* aTextTrack, bool aPendingListOnly)
224 : {
225 0 : if (!mPendingTextTracks || !mTextTracks) {
226 0 : return;
227 : }
228 :
229 0 : WEBVTT_LOG("%p RemoveTextTrack TextTrack %p", this, aTextTrack);
230 0 : mPendingTextTracks->RemoveTextTrack(aTextTrack);
231 0 : if (aPendingListOnly) {
232 0 : return;
233 : }
234 :
235 0 : mTextTracks->RemoveTextTrack(aTextTrack);
236 : // Remove the cues in mNewCues belong to aTextTrack.
237 0 : TextTrackCueList* removeCueList = aTextTrack->GetCues();
238 0 : if (removeCueList) {
239 0 : WEBVTT_LOGV("RemoveTextTrack removeCueList->Length() %d", removeCueList->Length());
240 0 : for (uint32_t i = 0; i < removeCueList->Length(); ++i) {
241 0 : mNewCues->RemoveCue(*((*removeCueList)[i]));
242 : }
243 0 : DispatchTimeMarchesOn();
244 : }
245 : }
246 :
247 : void
248 0 : TextTrackManager::DidSeek()
249 : {
250 0 : WEBVTT_LOG("%p DidSeek",this);
251 0 : if (mTextTracks) {
252 0 : mTextTracks->DidSeek();
253 : }
254 0 : if (mMediaElement) {
255 0 : mLastTimeMarchesOnCalled = mMediaElement->CurrentTime();
256 0 : WEBVTT_LOGV("DidSeek set mLastTimeMarchesOnCalled %lf",mLastTimeMarchesOnCalled);
257 : }
258 0 : mHasSeeked = true;
259 0 : }
260 :
261 : void
262 0 : TextTrackManager::UpdateCueDisplay()
263 : {
264 0 : WEBVTT_LOG("UpdateCueDisplay");
265 0 : mUpdateCueDisplayDispatched = false;
266 :
267 0 : if (!mMediaElement || !mTextTracks) {
268 0 : return;
269 : }
270 :
271 0 : nsIFrame* frame = mMediaElement->GetPrimaryFrame();
272 0 : nsVideoFrame* videoFrame = do_QueryFrame(frame);
273 0 : if (!videoFrame) {
274 0 : return;
275 : }
276 :
277 0 : nsCOMPtr<nsIContent> overlay = videoFrame->GetCaptionOverlay();
278 0 : nsCOMPtr<nsIContent> controls = videoFrame->GetVideoControls();
279 0 : if (!overlay) {
280 0 : return;
281 : }
282 :
283 0 : nsTArray<RefPtr<TextTrackCue> > showingCues;
284 0 : mTextTracks->GetShowingCues(showingCues);
285 :
286 0 : if (showingCues.Length() > 0) {
287 0 : WEBVTT_LOG("UpdateCueDisplay ProcessCues");
288 0 : WEBVTT_LOGV("UpdateCueDisplay showingCues.Length() %" PRIuSIZE, showingCues.Length());
289 0 : RefPtr<nsVariantCC> jsCues = new nsVariantCC();
290 :
291 0 : jsCues->SetAsArray(nsIDataType::VTYPE_INTERFACE,
292 : &NS_GET_IID(nsIDOMEventTarget),
293 0 : showingCues.Length(),
294 0 : static_cast<void*>(showingCues.Elements()));
295 0 : nsPIDOMWindowInner* window = mMediaElement->OwnerDoc()->GetInnerWindow();
296 0 : if (window) {
297 0 : sParserWrapper->ProcessCues(window, jsCues, overlay, controls);
298 : }
299 0 : } else if (overlay->Length() > 0) {
300 0 : WEBVTT_LOG("UpdateCueDisplay EmptyString");
301 0 : nsContentUtils::SetNodeTextContent(overlay, EmptyString(), true);
302 : }
303 : }
304 :
305 : void
306 0 : TextTrackManager::NotifyCueAdded(TextTrackCue& aCue)
307 : {
308 0 : WEBVTT_LOG("NotifyCueAdded");
309 0 : if (mNewCues) {
310 0 : mNewCues->AddCue(aCue);
311 : }
312 0 : DispatchTimeMarchesOn();
313 0 : ReportTelemetryForCue();
314 0 : }
315 :
316 : void
317 0 : TextTrackManager::NotifyCueRemoved(TextTrackCue& aCue)
318 : {
319 0 : WEBVTT_LOG("NotifyCueRemoved");
320 0 : if (mNewCues) {
321 0 : mNewCues->RemoveCue(aCue);
322 : }
323 0 : DispatchTimeMarchesOn();
324 0 : if (aCue.GetActive()) {
325 : // We remove an active cue, need to update the display.
326 0 : DispatchUpdateCueDisplay();
327 : }
328 0 : }
329 :
330 : void
331 0 : TextTrackManager::PopulatePendingList()
332 : {
333 0 : if (!mTextTracks || !mPendingTextTracks || !mMediaElement) {
334 0 : return;
335 : }
336 0 : uint32_t len = mTextTracks->Length();
337 : bool dummy;
338 0 : for (uint32_t index = 0; index < len; ++index) {
339 0 : TextTrack* ttrack = mTextTracks->IndexedGetter(index, dummy);
340 0 : if (ttrack && ttrack->Mode() != TextTrackMode::Disabled &&
341 0 : ttrack->ReadyState() == TextTrackReadyState::Loading) {
342 0 : mPendingTextTracks->AddTextTrack(ttrack,
343 0 : CompareTextTracks(mMediaElement));
344 : }
345 : }
346 : }
347 :
348 : void
349 0 : TextTrackManager::AddListeners()
350 : {
351 0 : if (mMediaElement) {
352 0 : mMediaElement->AddEventListener(NS_LITERAL_STRING("resizevideocontrols"),
353 0 : this, false, false);
354 0 : mMediaElement->AddEventListener(NS_LITERAL_STRING("seeked"),
355 0 : this, false, false);
356 0 : mMediaElement->AddEventListener(NS_LITERAL_STRING("controlbarchange"),
357 0 : this, false, true);
358 : }
359 0 : }
360 :
361 : void
362 0 : TextTrackManager::HonorUserPreferencesForTrackSelection()
363 : {
364 0 : if (performedTrackSelection || !mTextTracks) {
365 0 : return;
366 : }
367 0 : WEBVTT_LOG("HonorUserPreferencesForTrackSelection");
368 : TextTrackKind ttKinds[] = { TextTrackKind::Captions,
369 0 : TextTrackKind::Subtitles };
370 :
371 : // Steps 1 - 3: Perform automatic track selection for different TextTrack
372 : // Kinds.
373 0 : PerformTrackSelection(ttKinds, ArrayLength(ttKinds));
374 0 : PerformTrackSelection(TextTrackKind::Descriptions);
375 0 : PerformTrackSelection(TextTrackKind::Chapters);
376 :
377 : // Step 4: Set all TextTracks with a kind of metadata that are disabled
378 : // to hidden.
379 0 : for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
380 0 : TextTrack* track = (*mTextTracks)[i];
381 0 : if (track->Kind() == TextTrackKind::Metadata && TrackIsDefault(track) &&
382 0 : track->Mode() == TextTrackMode::Disabled) {
383 0 : track->SetMode(TextTrackMode::Hidden);
384 : }
385 : }
386 :
387 0 : performedTrackSelection = true;
388 : }
389 :
390 : bool
391 0 : TextTrackManager::TrackIsDefault(TextTrack* aTextTrack)
392 : {
393 0 : HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
394 0 : if (!trackElement) {
395 0 : return false;
396 : }
397 0 : return trackElement->Default();
398 : }
399 :
400 : void
401 0 : TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKind)
402 : {
403 0 : TextTrackKind ttKinds[] = { aTextTrackKind };
404 0 : PerformTrackSelection(ttKinds, ArrayLength(ttKinds));
405 0 : }
406 :
407 : void
408 0 : TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKinds[],
409 : uint32_t size)
410 : {
411 0 : nsTArray<TextTrack*> candidates;
412 0 : GetTextTracksOfKinds(aTextTrackKinds, size, candidates);
413 :
414 : // Step 3: If any TextTracks in candidates are showing then abort these steps.
415 0 : for (uint32_t i = 0; i < candidates.Length(); i++) {
416 0 : if (candidates[i]->Mode() == TextTrackMode::Showing) {
417 0 : WEBVTT_LOGV("PerformTrackSelection Showing return kind %d",
418 : static_cast<int>(candidates[i]->Kind()));
419 0 : return;
420 : }
421 : }
422 :
423 : // Step 4: Honor user preferences for track selection, otherwise, set the
424 : // first TextTrack in candidates with a default attribute to showing.
425 : // TODO: Bug 981691 - Honor user preferences for text track selection.
426 0 : for (uint32_t i = 0; i < candidates.Length(); i++) {
427 0 : if (TrackIsDefault(candidates[i]) &&
428 0 : candidates[i]->Mode() == TextTrackMode::Disabled) {
429 0 : candidates[i]->SetMode(TextTrackMode::Showing);
430 0 : WEBVTT_LOGV("PerformTrackSelection set Showing kind %d",
431 : static_cast<int>(candidates[i]->Kind()));
432 0 : return;
433 : }
434 : }
435 : }
436 :
437 : void
438 0 : TextTrackManager::GetTextTracksOfKinds(TextTrackKind aTextTrackKinds[],
439 : uint32_t size,
440 : nsTArray<TextTrack*>& aTextTracks)
441 : {
442 0 : for (uint32_t i = 0; i < size; i++) {
443 0 : GetTextTracksOfKind(aTextTrackKinds[i], aTextTracks);
444 : }
445 0 : }
446 :
447 : void
448 0 : TextTrackManager::GetTextTracksOfKind(TextTrackKind aTextTrackKind,
449 : nsTArray<TextTrack*>& aTextTracks)
450 : {
451 0 : if (!mTextTracks) {
452 0 : return;
453 : }
454 0 : for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
455 0 : TextTrack* textTrack = (*mTextTracks)[i];
456 0 : if (textTrack->Kind() == aTextTrackKind) {
457 0 : aTextTracks.AppendElement(textTrack);
458 : }
459 : }
460 : }
461 :
462 : NS_IMETHODIMP
463 0 : TextTrackManager::HandleEvent(nsIDOMEvent* aEvent)
464 : {
465 0 : if (!mTextTracks) {
466 0 : return NS_OK;
467 : }
468 :
469 0 : nsAutoString type;
470 0 : aEvent->GetType(type);
471 0 : if (type.EqualsLiteral("resizevideocontrols") ||
472 0 : type.EqualsLiteral("seeked")) {
473 0 : for (uint32_t i = 0; i< mTextTracks->Length(); i++) {
474 0 : ((*mTextTracks)[i])->SetCuesDirty();
475 : }
476 : }
477 :
478 0 : if (type.EqualsLiteral("controlbarchange")) {
479 0 : UpdateCueDisplay();
480 : }
481 :
482 0 : return NS_OK;
483 : }
484 :
485 :
486 0 : class SimpleTextTrackEvent : public Runnable
487 : {
488 : public:
489 : friend class CompareSimpleTextTrackEvents;
490 0 : SimpleTextTrackEvent(const nsAString& aEventName,
491 : double aTime,
492 : TextTrack* aTrack,
493 : TextTrackCue* aCue)
494 0 : : Runnable("dom::SimpleTextTrackEvent")
495 : , mName(aEventName)
496 : , mTime(aTime)
497 : , mTrack(aTrack)
498 0 : , mCue(aCue)
499 0 : {}
500 :
501 0 : NS_IMETHOD Run() {
502 0 : WEBVTT_LOGV("SimpleTextTrackEvent cue %p mName %s mTime %lf",
503 : mCue.get(), NS_ConvertUTF16toUTF8(mName).get(), mTime);
504 0 : mCue->DispatchTrustedEvent(mName);
505 0 : return NS_OK;
506 : }
507 :
508 : private:
509 : nsString mName;
510 : double mTime;
511 : TextTrack* mTrack;
512 : RefPtr<TextTrackCue> mCue;
513 : };
514 :
515 : class CompareSimpleTextTrackEvents {
516 : private:
517 : int32_t TrackChildPosition(SimpleTextTrackEvent* aEvent) const
518 : {
519 : if (aEvent->mTrack) {
520 : HTMLTrackElement* trackElement = aEvent->mTrack->GetTrackElement();
521 : if (trackElement) {
522 : return mMediaElement->IndexOf(trackElement);
523 : }
524 : }
525 : return -1;
526 : }
527 : HTMLMediaElement* mMediaElement;
528 : public:
529 0 : explicit CompareSimpleTextTrackEvents(HTMLMediaElement* aMediaElement)
530 0 : {
531 0 : mMediaElement = aMediaElement;
532 0 : }
533 :
534 0 : bool Equals(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const
535 : {
536 0 : return false;
537 : }
538 :
539 0 : bool LessThan(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const
540 : {
541 : // TimeMarchesOn step 13.1.
542 0 : if (aOne->mTime < aTwo->mTime) {
543 0 : return true;
544 0 : } else if (aOne->mTime > aTwo->mTime) {
545 0 : return false;
546 : }
547 :
548 : // TimeMarchesOn step 13.2 text track cue order.
549 : // TextTrack position in TextTrackList
550 0 : TextTrack* t1 = aOne->mTrack;
551 0 : TextTrack* t2 = aTwo->mTrack;
552 0 : MOZ_ASSERT(t1, "CompareSimpleTextTrackEvents t1 is null");
553 0 : MOZ_ASSERT(t2, "CompareSimpleTextTrackEvents t2 is null");
554 0 : if (t1 != t2) {
555 0 : TextTrackList* tList= t1->GetTextTrackList();
556 0 : MOZ_ASSERT(tList, "CompareSimpleTextTrackEvents tList is null");
557 0 : nsTArray<RefPtr<TextTrack>>& textTracks = tList->GetTextTrackArray();
558 0 : auto index1 = textTracks.IndexOf(t1);
559 0 : auto index2 = textTracks.IndexOf(t2);
560 0 : if (index1 < index2) {
561 0 : return true;
562 0 : } else if (index1 > index2) {
563 0 : return false;
564 : }
565 : }
566 :
567 0 : MOZ_ASSERT(t1 == t2, "CompareSimpleTextTrackEvents t1 != t2");
568 : // c1 and c2 are both belongs to t1.
569 0 : TextTrackCue* c1 = aOne->mCue;
570 0 : TextTrackCue* c2 = aTwo->mCue;
571 0 : if (c1 != c2) {
572 0 : if (c1->StartTime() < c2->StartTime()) {
573 0 : return true;
574 0 : } else if (c1->StartTime() > c2->StartTime()) {
575 0 : return false;
576 : }
577 0 : if (c1->EndTime() < c2->EndTime()) {
578 0 : return true;
579 0 : } else if (c1->EndTime() > c2->EndTime()) {
580 0 : return false;
581 : }
582 :
583 0 : TextTrackCueList* cueList = t1->GetCues();
584 0 : nsTArray<RefPtr<TextTrackCue>>& cues = cueList->GetCuesArray();
585 0 : auto index1 = cues.IndexOf(c1);
586 0 : auto index2 = cues.IndexOf(c2);
587 0 : if (index1 < index2) {
588 0 : return true;
589 0 : } else if (index1 > index2) {
590 0 : return false;
591 : }
592 : }
593 :
594 : // TimeMarchesOn step 13.3.
595 0 : if (aOne->mName.EqualsLiteral("enter") ||
596 0 : aTwo->mName.EqualsLiteral("exit")) {
597 0 : return true;
598 : }
599 0 : return false;
600 : }
601 : };
602 :
603 0 : class TextTrackListInternal
604 : {
605 : public:
606 0 : void AddTextTrack(TextTrack* aTextTrack,
607 : const CompareTextTracks& aCompareTT)
608 : {
609 0 : if (!mTextTracks.Contains(aTextTrack)) {
610 0 : mTextTracks.InsertElementSorted(aTextTrack, aCompareTT);
611 : }
612 0 : }
613 0 : uint32_t Length() const
614 : {
615 0 : return mTextTracks.Length();
616 : }
617 0 : TextTrack* operator[](uint32_t aIndex)
618 : {
619 0 : return mTextTracks.SafeElementAt(aIndex, nullptr);
620 : }
621 : private:
622 : nsTArray<RefPtr<TextTrack>> mTextTracks;
623 : };
624 :
625 : void
626 0 : TextTrackManager::DispatchUpdateCueDisplay()
627 : {
628 0 : if (!mUpdateCueDisplayDispatched && !mShutdown &&
629 0 : (mMediaElement->GetHasUserInteraction() || mMediaElement->IsCurrentlyPlaying())) {
630 0 : WEBVTT_LOG("DispatchUpdateCueDisplay");
631 0 : nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow();
632 0 : if (win) {
633 0 : nsGlobalWindow::Cast(win)->Dispatch(
634 : "TextTrackManager::UpdateCueDisplay",
635 : TaskCategory::Other,
636 0 : NewRunnableMethod("dom::TextTrackManager::UpdateCueDisplay",
637 : this,
638 0 : &TextTrackManager::UpdateCueDisplay));
639 0 : mUpdateCueDisplayDispatched = true;
640 : }
641 : }
642 0 : }
643 :
644 : void
645 0 : TextTrackManager::DispatchTimeMarchesOn()
646 : {
647 : // Run the algorithm if no previous instance is still running, otherwise
648 : // enqueue the current playback position and whether only that changed
649 : // through its usual monotonic increase during normal playback; current
650 : // executing call upon completion will check queue for further 'work'.
651 0 : if (!mTimeMarchesOnDispatched && !mShutdown &&
652 0 : (mMediaElement->GetHasUserInteraction() || mMediaElement->IsCurrentlyPlaying())) {
653 0 : WEBVTT_LOG("DispatchTimeMarchesOn");
654 0 : nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow();
655 0 : if (win) {
656 0 : nsGlobalWindow::Cast(win)->Dispatch(
657 : "TextTrackManager::TimeMarchesOn",
658 : TaskCategory::Other,
659 0 : NewRunnableMethod("dom::TextTrackManager::TimeMarchesOn",
660 : this,
661 0 : &TextTrackManager::TimeMarchesOn));
662 0 : mTimeMarchesOnDispatched = true;
663 : }
664 : }
665 0 : }
666 :
667 : // https://html.spec.whatwg.org/multipage/embedded-content.html#time-marches-on
668 : void
669 0 : TextTrackManager::TimeMarchesOn()
670 : {
671 0 : NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
672 0 : WEBVTT_LOG("TimeMarchesOn");
673 0 : mTimeMarchesOnDispatched = false;
674 :
675 : // Early return if we don't have any TextTracks or shutting down.
676 0 : if (!mTextTracks || mTextTracks->Length() == 0 || mShutdown) {
677 0 : return;
678 : }
679 :
680 : nsISupports* parentObject =
681 0 : mMediaElement->OwnerDoc()->GetParentObject();
682 0 : if (NS_WARN_IF(!parentObject)) {
683 0 : return;
684 : }
685 0 : nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
686 :
687 0 : if (mMediaElement &&
688 0 : (!(mMediaElement->GetPlayedOrSeeked()) || mMediaElement->Seeking())) {
689 0 : WEBVTT_LOG("TimeMarchesOn seeking or post return");
690 0 : return;
691 : }
692 :
693 : // Step 3.
694 0 : double currentPlaybackTime = mMediaElement->CurrentTime();
695 0 : bool hasNormalPlayback = !mHasSeeked;
696 0 : mHasSeeked = false;
697 0 : WEBVTT_LOG("TimeMarchesOn mLastTimeMarchesOnCalled %lf currentPlaybackTime %lf hasNormalPlayback %d"
698 : , mLastTimeMarchesOnCalled, currentPlaybackTime, hasNormalPlayback);
699 :
700 : // Step 1, 2.
701 : RefPtr<TextTrackCueList> currentCues =
702 0 : new TextTrackCueList(window);
703 : RefPtr<TextTrackCueList> otherCues =
704 0 : new TextTrackCueList(window);
705 : bool dummy;
706 0 : for (uint32_t index = 0; index < mTextTracks->Length(); ++index) {
707 0 : TextTrack* ttrack = mTextTracks->IndexedGetter(index, dummy);
708 0 : if (ttrack && dummy) {
709 : // TODO: call GetCueListByTimeInterval on mNewCues?
710 0 : ttrack->UpdateActiveCueList();
711 0 : TextTrackCueList* activeCueList = ttrack->GetActiveCues();
712 0 : if (activeCueList) {
713 0 : for (uint32_t i = 0; i < activeCueList->Length(); ++i) {
714 0 : currentCues->AddCue(*((*activeCueList)[i]));
715 : }
716 : }
717 : }
718 : }
719 0 : WEBVTT_LOGV("TimeMarchesOn currentCues %d", currentCues->Length());
720 : // Populate otherCues with 'non-active" cues.
721 0 : if (hasNormalPlayback) {
722 0 : if (currentPlaybackTime < mLastTimeMarchesOnCalled) {
723 : // TODO: Add log and find the root cause why the
724 : // playback position goes backward.
725 0 : mLastTimeMarchesOnCalled = currentPlaybackTime;
726 : }
727 : media::Interval<double> interval(mLastTimeMarchesOnCalled,
728 0 : currentPlaybackTime);
729 0 : otherCues = mNewCues->GetCueListByTimeInterval(interval);;
730 : } else {
731 : // Seek case. Put the mLastActiveCues into otherCues.
732 0 : otherCues = mLastActiveCues;
733 : }
734 0 : for (uint32_t i = 0; i < currentCues->Length(); ++i) {
735 0 : TextTrackCue* cue = (*currentCues)[i];
736 0 : otherCues->RemoveCue(*cue);
737 : }
738 0 : WEBVTT_LOGV("TimeMarchesOn otherCues %d", otherCues->Length());
739 : // Step 4.
740 0 : RefPtr<TextTrackCueList> missedCues = new TextTrackCueList(window);
741 0 : if (hasNormalPlayback) {
742 0 : for (uint32_t i = 0; i < otherCues->Length(); ++i) {
743 0 : TextTrackCue* cue = (*otherCues)[i];
744 0 : if (cue->StartTime() >= mLastTimeMarchesOnCalled &&
745 0 : cue->EndTime() <= currentPlaybackTime) {
746 0 : missedCues->AddCue(*cue);
747 : }
748 : }
749 : }
750 0 : WEBVTT_LOGV("TimeMarchesOn missedCues %d", missedCues->Length());
751 : // Step 5. Empty now.
752 : // TODO: Step 6: fire timeupdate?
753 :
754 : // Step 7. Abort steps if condition 1, 2, 3 are satisfied.
755 : // 1. All of the cues in current cues have their active flag set.
756 : // 2. None of the cues in other cues have their active flag set.
757 : // 3. Missed cues is empty.
758 0 : bool c1 = true;
759 0 : for (uint32_t i = 0; i < currentCues->Length(); ++i) {
760 0 : if (!(*currentCues)[i]->GetActive()) {
761 0 : c1 = false;
762 0 : break;
763 : }
764 : }
765 0 : bool c2 = true;
766 0 : for (uint32_t i = 0; i < otherCues->Length(); ++i) {
767 0 : if ((*otherCues)[i]->GetActive()) {
768 0 : c2 = false;
769 0 : break;
770 : }
771 : }
772 0 : bool c3 = (missedCues->Length() == 0);
773 0 : if (c1 && c2 && c3) {
774 0 : mLastTimeMarchesOnCalled = currentPlaybackTime;
775 0 : WEBVTT_LOG("TimeMarchesOn step 7 return, mLastTimeMarchesOnCalled %lf", mLastTimeMarchesOnCalled);
776 0 : return;
777 : }
778 :
779 : // Step 8. Respect PauseOnExit flag if not seek.
780 0 : if (hasNormalPlayback) {
781 0 : for (uint32_t i = 0; i < otherCues->Length(); ++i) {
782 0 : TextTrackCue* cue = (*otherCues)[i];
783 0 : if (cue && cue->PauseOnExit() && cue->GetActive()) {
784 0 : WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
785 0 : mMediaElement->Pause();
786 0 : break;
787 : }
788 : }
789 0 : for (uint32_t i = 0; i < missedCues->Length(); ++i) {
790 0 : TextTrackCue* cue = (*missedCues)[i];
791 0 : if (cue && cue->PauseOnExit()) {
792 0 : WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
793 0 : mMediaElement->Pause();
794 0 : break;
795 : }
796 : }
797 : }
798 :
799 : // Step 15.
800 : // Sort text tracks in the same order as the text tracks appear
801 : // in the media element's list of text tracks, and remove
802 : // duplicates.
803 0 : TextTrackListInternal affectedTracks;
804 : // Step 13, 14.
805 0 : nsTArray<RefPtr<SimpleTextTrackEvent>> eventList;
806 : // Step 9, 10.
807 : // For each text track cue in missed cues, prepare an event named
808 : // enter for the TextTrackCue object with the cue start time.
809 0 : for (uint32_t i = 0; i < missedCues->Length(); ++i) {
810 0 : TextTrackCue* cue = (*missedCues)[i];
811 0 : if (cue) {
812 : SimpleTextTrackEvent* event =
813 0 : new SimpleTextTrackEvent(NS_LITERAL_STRING("enter"),
814 0 : cue->StartTime(), cue->GetTrack(),
815 0 : cue);
816 : eventList.InsertElementSorted(event,
817 0 : CompareSimpleTextTrackEvents(mMediaElement));
818 0 : affectedTracks.AddTextTrack(cue->GetTrack(), CompareTextTracks(mMediaElement));
819 : }
820 : }
821 :
822 : // Step 11, 17.
823 0 : for (uint32_t i = 0; i < otherCues->Length(); ++i) {
824 0 : TextTrackCue* cue = (*otherCues)[i];
825 0 : if (cue->GetActive() || missedCues->IsCueExist(cue)) {
826 0 : double time = cue->StartTime() > cue->EndTime() ? cue->StartTime()
827 0 : : cue->EndTime();
828 : SimpleTextTrackEvent* event =
829 0 : new SimpleTextTrackEvent(NS_LITERAL_STRING("exit"), time,
830 0 : cue->GetTrack(), cue);
831 : eventList.InsertElementSorted(event,
832 0 : CompareSimpleTextTrackEvents(mMediaElement));
833 0 : affectedTracks.AddTextTrack(cue->GetTrack(), CompareTextTracks(mMediaElement));
834 : }
835 0 : cue->SetActive(false);
836 : }
837 :
838 : // Step 12, 17.
839 0 : for (uint32_t i = 0; i < currentCues->Length(); ++i) {
840 0 : TextTrackCue* cue = (*currentCues)[i];
841 0 : if (!cue->GetActive()) {
842 : SimpleTextTrackEvent* event =
843 0 : new SimpleTextTrackEvent(NS_LITERAL_STRING("enter"),
844 0 : cue->StartTime(), cue->GetTrack(),
845 0 : cue);
846 : eventList.InsertElementSorted(event,
847 0 : CompareSimpleTextTrackEvents(mMediaElement));
848 0 : affectedTracks.AddTextTrack(cue->GetTrack(), CompareTextTracks(mMediaElement));
849 : }
850 0 : cue->SetActive(true);
851 : }
852 :
853 : // Fire the eventList
854 0 : for (uint32_t i = 0; i < eventList.Length(); ++i) {
855 0 : NS_DispatchToMainThread(eventList[i].forget());
856 : }
857 :
858 : // Step 16.
859 0 : for (uint32_t i = 0; i < affectedTracks.Length(); ++i) {
860 0 : TextTrack* ttrack = affectedTracks[i];
861 0 : if (ttrack) {
862 0 : ttrack->DispatchAsyncTrustedEvent(NS_LITERAL_STRING("cuechange"));
863 0 : HTMLTrackElement* trackElement = ttrack->GetTrackElement();
864 0 : if (trackElement) {
865 0 : trackElement->DispatchTrackRunnable(NS_LITERAL_STRING("cuechange"));
866 : }
867 : }
868 : }
869 :
870 0 : mLastTimeMarchesOnCalled = currentPlaybackTime;
871 0 : mLastActiveCues = currentCues;
872 :
873 : // Step 18.
874 0 : UpdateCueDisplay();
875 : }
876 :
877 : void
878 0 : TextTrackManager::NotifyCueUpdated(TextTrackCue *aCue)
879 : {
880 : // TODO: Add/Reorder the cue to mNewCues if we have some optimization?
881 0 : WEBVTT_LOG("NotifyCueUpdated");
882 0 : DispatchTimeMarchesOn();
883 : // For the case "Texttrack.mode = hidden/showing", if the mode
884 : // changing between showing and hidden, TimeMarchesOn
885 : // doesn't render the cue. Call DispatchUpdateCueDisplay() explicitly.
886 0 : DispatchUpdateCueDisplay();
887 0 : }
888 :
889 : void
890 0 : TextTrackManager::NotifyReset()
891 : {
892 0 : WEBVTT_LOG("NotifyReset");
893 0 : mLastTimeMarchesOnCalled = 0.0;
894 0 : }
895 :
896 : void
897 0 : TextTrackManager::ReportTelemetryForTrack(TextTrack* aTextTrack) const
898 : {
899 0 : MOZ_ASSERT(NS_IsMainThread());
900 0 : MOZ_ASSERT(aTextTrack);
901 0 : MOZ_ASSERT(mTextTracks->Length() > 0);
902 :
903 0 : TextTrackKind kind = aTextTrack->Kind();
904 0 : Telemetry::Accumulate(Telemetry::WEBVTT_TRACK_KINDS, uint32_t(kind));
905 0 : }
906 :
907 : void
908 0 : TextTrackManager::ReportTelemetryForCue()
909 : {
910 0 : MOZ_ASSERT(NS_IsMainThread());
911 0 : MOZ_ASSERT(!mNewCues->IsEmpty() || !mLastActiveCues->IsEmpty());
912 :
913 0 : if (!mCueTelemetryReported) {
914 0 : Telemetry::Accumulate(Telemetry::WEBVTT_USED_VTT_CUES, 1);
915 0 : mCueTelemetryReported = true;
916 : }
917 0 : }
918 :
919 : bool
920 0 : TextTrackManager::IsLoaded()
921 : {
922 0 : return mTextTracks ? mTextTracks->AreTextTracksLoaded() : true;
923 : }
924 :
925 :
926 : } // namespace dom
927 : } // namespace mozilla
|
