Line data Source code
1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 : /* This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this
5 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #ifndef MOZILLA_TRACKBUFFERSMANAGER_H_
8 : #define MOZILLA_TRACKBUFFERSMANAGER_H_
9 :
10 : #include "mozilla/Atomics.h"
11 : #include "mozilla/Maybe.h"
12 : #include "mozilla/Monitor.h"
13 : #include "AutoTaskQueue.h"
14 : #include "mozilla/dom/SourceBufferBinding.h"
15 :
16 : #include "MediaContainerType.h"
17 : #include "MediaData.h"
18 : #include "MediaDataDemuxer.h"
19 : #include "MediaResult.h"
20 : #include "MediaSourceDecoder.h"
21 : #include "SourceBufferTask.h"
22 : #include "TimeUnits.h"
23 : #include "nsAutoPtr.h"
24 : #include "nsProxyRelease.h"
25 : #include "nsTArray.h"
26 :
27 : namespace mozilla {
28 :
29 : class AbstractThread;
30 : class ContainerParser;
31 : class MediaByteBuffer;
32 : class MediaRawData;
33 : class MediaSourceDemuxer;
34 : class SourceBufferResource;
35 :
36 : class SourceBufferTaskQueue
37 : {
38 : public:
39 0 : SourceBufferTaskQueue()
40 0 : : mMonitor("SourceBufferTaskQueue")
41 0 : {}
42 0 : ~SourceBufferTaskQueue()
43 0 : {
44 0 : MOZ_ASSERT(mQueue.IsEmpty(), "All tasks must have been processed");
45 0 : }
46 :
47 0 : void Push(SourceBufferTask* aTask)
48 : {
49 0 : MonitorAutoLock mon(mMonitor);
50 0 : mQueue.AppendElement(aTask);
51 0 : }
52 :
53 0 : already_AddRefed<SourceBufferTask> Pop()
54 : {
55 0 : MonitorAutoLock mon(mMonitor);
56 0 : if (!mQueue.Length()) {
57 0 : return nullptr;
58 : }
59 0 : RefPtr<SourceBufferTask> task = Move(mQueue[0]);
60 0 : mQueue.RemoveElementAt(0);
61 0 : return task.forget();
62 : }
63 :
64 0 : nsTArray<SourceBufferTask>::size_type Length() const
65 : {
66 0 : MonitorAutoLock mon(mMonitor);
67 0 : return mQueue.Length();
68 : }
69 :
70 : private:
71 : mutable Monitor mMonitor;
72 : nsTArray<RefPtr<SourceBufferTask>> mQueue;
73 : };
74 :
75 : class TrackBuffersManager
76 : {
77 : public:
78 0 : NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackBuffersManager);
79 :
80 : enum class EvictDataResult : int8_t
81 : {
82 : NO_DATA_EVICTED,
83 : CANT_EVICT,
84 : BUFFER_FULL,
85 : };
86 :
87 : typedef TrackInfo::TrackType TrackType;
88 : typedef MediaData::Type MediaType;
89 : typedef nsTArray<RefPtr<MediaRawData>> TrackBuffer;
90 : typedef SourceBufferTask::AppendPromise AppendPromise;
91 : typedef SourceBufferTask::RangeRemovalPromise RangeRemovalPromise;
92 :
93 : // Interface for SourceBuffer
94 : TrackBuffersManager(MediaSourceDecoder* aParentDecoder,
95 : const MediaContainerType& aType);
96 :
97 : // Queue a task to add data to the end of the input buffer and run the MSE
98 : // Buffer Append Algorithm
99 : // 3.5.5 Buffer Append Algorithm.
100 : // http://w3c.github.io/media-source/index.html#sourcebuffer-buffer-append
101 : RefPtr<AppendPromise> AppendData(already_AddRefed<MediaByteBuffer> aData,
102 : const SourceBufferAttributes& aAttributes);
103 :
104 : // Queue a task to abort any pending AppendData.
105 : // Does nothing at this stage.
106 : void AbortAppendData();
107 :
108 : // Queue a task to run MSE Reset Parser State Algorithm.
109 : // 3.5.2 Reset Parser State
110 : void ResetParserState(SourceBufferAttributes& aAttributes);
111 :
112 : // Queue a task to run the MSE range removal algorithm.
113 : // http://w3c.github.io/media-source/#sourcebuffer-coded-frame-removal
114 : RefPtr<RangeRemovalPromise> RangeRemoval(media::TimeUnit aStart,
115 : media::TimeUnit aEnd);
116 :
117 : // Schedule data eviction if necessary as the next call to AppendData will
118 : // add aSize bytes.
119 : // Eviction is done in two steps, first remove data up to aPlaybackTime
120 : // and if still more space is needed remove from the end.
121 : EvictDataResult EvictData(const media::TimeUnit& aPlaybackTime, int64_t aSize);
122 :
123 : // Returns the buffered range currently managed.
124 : // This may be called on any thread.
125 : // Buffered must conform to http://w3c.github.io/media-source/index.html#widl-SourceBuffer-buffered
126 : media::TimeIntervals Buffered() const;
127 : media::TimeUnit HighestStartTime() const;
128 : media::TimeUnit HighestEndTime() const;
129 :
130 : // Return the size of the data managed by this SourceBufferContentManager.
131 : int64_t GetSize() const;
132 :
133 : // Indicate that the MediaSource parent object got into "ended" state.
134 : void Ended();
135 :
136 : // The parent SourceBuffer is about to be destroyed.
137 : void Detach();
138 :
139 : int64_t EvictionThreshold() const;
140 :
141 : // Interface for MediaSourceDemuxer
142 : MediaInfo GetMetadata() const;
143 : const TrackBuffer& GetTrackBuffer(TrackInfo::TrackType aTrack) const;
144 : const media::TimeIntervals& Buffered(TrackInfo::TrackType) const;
145 : const media::TimeUnit& HighestStartTime(TrackInfo::TrackType) const;
146 : media::TimeIntervals SafeBuffered(TrackInfo::TrackType) const;
147 0 : bool IsEnded() const
148 : {
149 0 : return mEnded;
150 : }
151 : uint32_t Evictable(TrackInfo::TrackType aTrack) const;
152 : media::TimeUnit Seek(TrackInfo::TrackType aTrack,
153 : const media::TimeUnit& aTime,
154 : const media::TimeUnit& aFuzz);
155 : uint32_t SkipToNextRandomAccessPoint(TrackInfo::TrackType aTrack,
156 : const media::TimeUnit& aTimeThreadshold,
157 : const media::TimeUnit& aFuzz,
158 : bool& aFound);
159 :
160 : already_AddRefed<MediaRawData> GetSample(TrackInfo::TrackType aTrack,
161 : const media::TimeUnit& aFuzz,
162 : MediaResult& aResult);
163 : int32_t FindCurrentPosition(TrackInfo::TrackType aTrack,
164 : const media::TimeUnit& aFuzz) const;
165 : media::TimeUnit GetNextRandomAccessPoint(TrackInfo::TrackType aTrack,
166 : const media::TimeUnit& aFuzz);
167 :
168 : void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes) const;
169 :
170 : private:
171 : typedef MozPromise<bool, MediaResult, /* IsExclusive = */ true> CodedFrameProcessingPromise;
172 :
173 : // for MediaSourceDemuxer::GetMozDebugReaderData
174 : friend class MediaSourceDemuxer;
175 : ~TrackBuffersManager();
176 : // All following functions run on the taskqueue.
177 : RefPtr<AppendPromise> DoAppendData(already_AddRefed<MediaByteBuffer> aData,
178 : const SourceBufferAttributes& aAttributes);
179 : void ScheduleSegmentParserLoop();
180 : void SegmentParserLoop();
181 : void InitializationSegmentReceived();
182 : void ShutdownDemuxers();
183 : void CreateDemuxerforMIMEType();
184 : void ResetDemuxingState();
185 : void NeedMoreData();
186 : void RejectAppend(const MediaResult& aRejectValue, const char* aName);
187 : // Will return a promise that will be resolved once all frames of the current
188 : // media segment have been processed.
189 : RefPtr<CodedFrameProcessingPromise> CodedFrameProcessing();
190 : void CompleteCodedFrameProcessing();
191 : // Called by ResetParserState.
192 : void CompleteResetParserState();
193 : RefPtr<RangeRemovalPromise>
194 : CodedFrameRemovalWithPromise(media::TimeInterval aInterval);
195 : bool CodedFrameRemoval(media::TimeInterval aInterval);
196 : void SetAppendState(SourceBufferAttributes::AppendState aAppendState);
197 :
198 0 : bool HasVideo() const
199 : {
200 0 : return mVideoTracks.mNumTracks > 0;
201 : }
202 0 : bool HasAudio() const
203 : {
204 0 : return mAudioTracks.mNumTracks > 0;
205 : }
206 :
207 : // The input buffer as per http://w3c.github.io/media-source/index.html#sourcebuffer-input-buffer
208 : RefPtr<MediaByteBuffer> mInputBuffer;
209 : // Buffer full flag as per https://w3c.github.io/media-source/#sourcebuffer-buffer-full-flag.
210 : // Accessed on both the main thread and the task queue.
211 : Atomic<bool> mBufferFull;
212 : bool mFirstInitializationSegmentReceived;
213 : // Set to true once a new segment is started.
214 : bool mNewMediaSegmentStarted;
215 : bool mActiveTrack;
216 : MediaContainerType mType;
217 :
218 : // ContainerParser objects and methods.
219 : // Those are used to parse the incoming input buffer.
220 :
221 : // Recreate the ContainerParser and if aReuseInitData is true then
222 : // feed it with the previous init segment found.
223 : void RecreateParser(bool aReuseInitData);
224 : nsAutoPtr<ContainerParser> mParser;
225 :
226 : // Demuxer objects and methods.
227 : void AppendDataToCurrentInputBuffer(MediaByteBuffer* aData);
228 : RefPtr<MediaByteBuffer> mInitData;
229 : // Temporary input buffer to handle partial media segment header.
230 : // We store the current input buffer content into it should we need to
231 : // reinitialize the demuxer once we have some samples and a discontinuity is
232 : // detected.
233 : RefPtr<MediaByteBuffer> mPendingInputBuffer;
234 : RefPtr<SourceBufferResource> mCurrentInputBuffer;
235 : RefPtr<MediaDataDemuxer> mInputDemuxer;
236 : // Length already processed in current media segment.
237 : uint64_t mProcessedInput;
238 : Maybe<media::TimeUnit> mLastParsedEndTime;
239 :
240 : void OnDemuxerInitDone(const MediaResult& aResult);
241 : void OnDemuxerInitFailed(const MediaResult& aFailure);
242 : void OnDemuxerResetDone(const MediaResult& aResult);
243 : MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest;
244 :
245 : void OnDemuxFailed(TrackType aTrack, const MediaResult& aError);
246 : void DoDemuxVideo();
247 : void OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
248 0 : void OnVideoDemuxFailed(const MediaResult& aError)
249 : {
250 0 : mVideoTracks.mDemuxRequest.Complete();
251 0 : OnDemuxFailed(TrackType::kVideoTrack, aError);
252 0 : }
253 : void DoDemuxAudio();
254 : void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
255 0 : void OnAudioDemuxFailed(const MediaResult& aError)
256 : {
257 0 : mAudioTracks.mDemuxRequest.Complete();
258 0 : OnDemuxFailed(TrackType::kAudioTrack, aError);
259 0 : }
260 :
261 : void DoEvictData(const media::TimeUnit& aPlaybackTime, int64_t aSizeToEvict);
262 :
263 0 : struct TrackData
264 : {
265 0 : TrackData()
266 0 : : mNumTracks(0)
267 : , mNeedRandomAccessPoint(true)
268 0 : , mSizeBuffer(0)
269 0 : {}
270 : uint32_t mNumTracks;
271 : // Definition of variables:
272 : // https://w3c.github.io/media-source/#track-buffers
273 : // Last decode timestamp variable that stores the decode timestamp of the
274 : // last coded frame appended in the current coded frame group.
275 : // The variable is initially unset to indicate that no coded frames have
276 : // been appended yet.
277 : Maybe<media::TimeUnit> mLastDecodeTimestamp;
278 : // Last frame duration variable that stores the coded frame duration of the
279 : // last coded frame appended in the current coded frame group.
280 : // The variable is initially unset to indicate that no coded frames have
281 : // been appended yet.
282 : Maybe<media::TimeUnit> mLastFrameDuration;
283 : // Highest end timestamp variable that stores the highest coded frame end
284 : // timestamp across all coded frames in the current coded frame group that
285 : // were appended to this track buffer.
286 : // The variable is initially unset to indicate that no coded frames have
287 : // been appended yet.
288 : Maybe<media::TimeUnit> mHighestEndTimestamp;
289 : // Highest presentation timestamp in track buffer.
290 : // Protected by global monitor, except when reading on the task queue as it
291 : // is only written there.
292 : media::TimeUnit mHighestStartTimestamp;
293 : // Longest frame duration seen since last random access point.
294 : // Only ever accessed when mLastDecodeTimestamp and mLastFrameDuration are
295 : // set.
296 : media::TimeUnit mLongestFrameDuration;
297 : // Need random access point flag variable that keeps track of whether the
298 : // track buffer is waiting for a random access point coded frame.
299 : // The variable is initially set to true to indicate that random access
300 : // point coded frame is needed before anything can be added to the track
301 : // buffer.
302 : bool mNeedRandomAccessPoint;
303 : RefPtr<MediaTrackDemuxer> mDemuxer;
304 : MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest;
305 : // Highest end timestamp of the last media segment demuxed.
306 : media::TimeUnit mLastParsedEndTime;
307 :
308 : // If set, position where the next contiguous frame will be inserted.
309 : // If a discontinuity is detected, it will be unset and recalculated upon
310 : // the next insertion.
311 : Maybe<uint32_t> mNextInsertionIndex;
312 : // Samples just demuxed, but not yet parsed.
313 : TrackBuffer mQueuedSamples;
314 0 : const TrackBuffer& GetTrackBuffer() const
315 : {
316 0 : MOZ_RELEASE_ASSERT(mBuffers.Length(),
317 : "TrackBuffer must have been created");
318 0 : return mBuffers.LastElement();
319 : }
320 0 : TrackBuffer& GetTrackBuffer()
321 : {
322 0 : MOZ_RELEASE_ASSERT(mBuffers.Length(),
323 : "TrackBuffer must have been created");
324 0 : return mBuffers.LastElement();
325 : }
326 : // We only manage a single track of each type at this time.
327 : nsTArray<TrackBuffer> mBuffers;
328 : // Track buffer ranges variable that represents the presentation time ranges
329 : // occupied by the coded frames currently stored in the track buffer.
330 : media::TimeIntervals mBufferedRanges;
331 : // Sanitized mBufferedRanges with a fuzz of half a sample's duration applied
332 : // This buffered ranges is the basis of what is exposed to the JS.
333 : media::TimeIntervals mSanitizedBufferedRanges;
334 : // Byte size of all samples contained in this track buffer.
335 : uint32_t mSizeBuffer;
336 : // TrackInfo of the first metadata received.
337 : RefPtr<TrackInfoSharedPtr> mInfo;
338 : // TrackInfo of the last metadata parsed (updated with each init segment.
339 : RefPtr<TrackInfoSharedPtr> mLastInfo;
340 :
341 : // If set, position of the next sample to be retrieved by GetSample().
342 : // If the position is equal to the TrackBuffer's length, it indicates that
343 : // we've reached EOS.
344 : Maybe<uint32_t> mNextGetSampleIndex;
345 : // Approximation of the next sample's decode timestamp.
346 : media::TimeUnit mNextSampleTimecode;
347 : // Approximation of the next sample's presentation timestamp.
348 : media::TimeUnit mNextSampleTime;
349 :
350 : struct EvictionIndex
351 : {
352 0 : EvictionIndex() { Reset(); }
353 0 : void Reset()
354 : {
355 0 : mEvictable = 0;
356 0 : mLastIndex = 0;
357 0 : }
358 : uint32_t mEvictable;
359 : uint32_t mLastIndex;
360 : };
361 : // Size of data that can be safely evicted during the next eviction
362 : // cycle.
363 : // We consider as evictable all frames up to the last keyframe prior to
364 : // mNextGetSampleIndex. If mNextGetSampleIndex isn't set, then we assume
365 : // that we can't yet evict data.
366 : // Protected by global monitor, except when reading on the task queue as it
367 : // is only written there.
368 : EvictionIndex mEvictionIndex;
369 :
370 0 : void ResetAppendState()
371 : {
372 0 : mLastDecodeTimestamp.reset();
373 0 : mLastFrameDuration.reset();
374 0 : mHighestEndTimestamp.reset();
375 0 : mNeedRandomAccessPoint = true;
376 :
377 0 : mNextInsertionIndex.reset();
378 0 : }
379 :
380 : void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes) const;
381 : };
382 :
383 : void CheckSequenceDiscontinuity(const media::TimeUnit& aPresentationTime);
384 : void ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData);
385 : media::TimeInterval PresentationInterval(const TrackBuffer& aSamples) const;
386 : bool CheckNextInsertionIndex(TrackData& aTrackData,
387 : const media::TimeUnit& aSampleTime);
388 : void InsertFrames(TrackBuffer& aSamples,
389 : const media::TimeIntervals& aIntervals,
390 : TrackData& aTrackData);
391 : void UpdateHighestTimestamp(TrackData& aTrackData,
392 : const media::TimeUnit& aHighestTime);
393 : // Remove all frames and their dependencies contained in aIntervals.
394 : // Return the index at which frames were first removed or 0 if no frames
395 : // removed.
396 : uint32_t RemoveFrames(const media::TimeIntervals& aIntervals,
397 : TrackData& aTrackData,
398 : uint32_t aStartIndex);
399 : // Recalculate track's evictable amount.
400 : void ResetEvictionIndex(TrackData& aTrackData);
401 : void UpdateEvictionIndex(TrackData& aTrackData, uint32_t aCurrentIndex);
402 : // Find index of sample. Return a negative value if not found.
403 : uint32_t FindSampleIndex(const TrackBuffer& aTrackBuffer,
404 : const media::TimeInterval& aInterval);
405 : const MediaRawData* GetSample(TrackInfo::TrackType aTrack,
406 : uint32_t aIndex,
407 : const media::TimeUnit& aExpectedDts,
408 : const media::TimeUnit& aExpectedPts,
409 : const media::TimeUnit& aFuzz);
410 : void UpdateBufferedRanges();
411 : void RejectProcessing(const MediaResult& aRejectValue, const char* aName);
412 : void ResolveProcessing(bool aResolveValue, const char* aName);
413 : MozPromiseRequestHolder<CodedFrameProcessingPromise> mProcessingRequest;
414 : MozPromiseHolder<CodedFrameProcessingPromise> mProcessingPromise;
415 :
416 : // Trackbuffers definition.
417 : nsTArray<const TrackData*> GetTracksList() const;
418 : nsTArray<TrackData*> GetTracksList();
419 0 : TrackData& GetTracksData(TrackType aTrack)
420 : {
421 0 : switch(aTrack) {
422 : case TrackType::kVideoTrack:
423 0 : return mVideoTracks;
424 : case TrackType::kAudioTrack:
425 : default:
426 0 : return mAudioTracks;
427 : }
428 : }
429 0 : const TrackData& GetTracksData(TrackType aTrack) const
430 : {
431 0 : switch(aTrack) {
432 : case TrackType::kVideoTrack:
433 0 : return mVideoTracks;
434 : case TrackType::kAudioTrack:
435 : default:
436 0 : return mAudioTracks;
437 : }
438 : }
439 : TrackData mVideoTracks;
440 : TrackData mAudioTracks;
441 :
442 : // TaskQueue methods and objects.
443 0 : AbstractThread* GetTaskQueue() const
444 : {
445 0 : return mTaskQueue;
446 : }
447 0 : bool OnTaskQueue() const
448 : {
449 0 : return !GetTaskQueue() || GetTaskQueue()->IsCurrentThreadIn();
450 : }
451 : RefPtr<AutoTaskQueue> mTaskQueue;
452 :
453 : // SourceBuffer Queues and running context.
454 : SourceBufferTaskQueue mQueue;
455 : void QueueTask(SourceBufferTask* aTask);
456 : void ProcessTasks();
457 : // Set if the TrackBuffersManager is currently processing a task.
458 : // At this stage, this task is always a AppendBufferTask.
459 : RefPtr<SourceBufferTask> mCurrentTask;
460 : // Current SourceBuffer state for ongoing task.
461 : // Its content is returned to the SourceBuffer once the AppendBufferTask has
462 : // completed.
463 : UniquePtr<SourceBufferAttributes> mSourceBufferAttributes;
464 : // The current sourcebuffer append window. It's content is equivalent to
465 : // mSourceBufferAttributes.mAppendWindowStart/End
466 : media::TimeInterval mAppendWindow;
467 :
468 : // Strong references to external objects.
469 : nsMainThreadPtrHandle<MediaSourceDecoder> mParentDecoder;
470 :
471 : const RefPtr<AbstractThread> mAbstractMainThread;
472 :
473 : // Return public highest end time across all aTracks.
474 : // Monitor must be held.
475 : media::TimeUnit HighestEndTime(nsTArray<const media::TimeIntervals*>& aTracks) const;
476 :
477 : // Set to true if mediasource state changed to ended.
478 : Atomic<bool> mEnded;
479 :
480 : // Global size of this source buffer content.
481 : Atomic<int64_t> mSizeSourceBuffer;
482 : const int64_t mVideoEvictionThreshold;
483 : const int64_t mAudioEvictionThreshold;
484 : enum class EvictionState
485 : {
486 : NO_EVICTION_NEEDED,
487 : EVICTION_NEEDED,
488 : EVICTION_COMPLETED,
489 : };
490 : Atomic<EvictionState> mEvictionState;
491 :
492 : // Monitor to protect following objects accessed across multiple threads.
493 : mutable Monitor mMonitor;
494 : // Stable audio and video track time ranges.
495 : media::TimeIntervals mVideoBufferedRanges;
496 : media::TimeIntervals mAudioBufferedRanges;
497 : // MediaInfo of the first init segment read.
498 : MediaInfo mInfo;
499 : };
500 :
501 : } // namespace mozilla
502 :
503 : #endif /* MOZILLA_TRACKBUFFERSMANAGER_H_ */
|
