Line data Source code
1 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 : /* This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this
5 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #include "H264Converter.h"
8 :
9 : #include "DecoderDoctorDiagnostics.h"
10 : #include "ImageContainer.h"
11 : #include "MediaInfo.h"
12 : #include "MediaPrefs.h"
13 : #include "PDMFactory.h"
14 : #include "mozilla/TaskQueue.h"
15 : #include "mp4_demuxer/AnnexB.h"
16 : #include "mp4_demuxer/H264.h"
17 :
18 : namespace mozilla
19 : {
20 :
21 0 : H264Converter::H264Converter(PlatformDecoderModule* aPDM,
22 0 : const CreateDecoderParams& aParams)
23 : : mPDM(aPDM)
24 : , mOriginalConfig(aParams.VideoConfig())
25 : , mCurrentConfig(aParams.VideoConfig())
26 : , mKnowsCompositor(aParams.mKnowsCompositor)
27 0 : , mImageContainer(aParams.mImageContainer)
28 0 : , mTaskQueue(aParams.mTaskQueue)
29 : , mDecoder(nullptr)
30 : , mGMPCrashHelper(aParams.mCrashHelper)
31 : , mLastError(NS_OK)
32 0 : , mType(aParams.mType)
33 0 : , mOnWaitingForKeyEvent(aParams.mOnWaitingForKeyEvent)
34 0 : , mDecoderOptions(aParams.mOptions)
35 : {
36 0 : CreateDecoder(mOriginalConfig, aParams.mDiagnostics);
37 0 : if (mDecoder) {
38 0 : MOZ_ASSERT(mp4_demuxer::H264::HasSPS(mOriginalConfig.mExtraData));
39 : // The video metadata contains out of band SPS/PPS (AVC1) store it.
40 0 : mOriginalExtraData = mOriginalConfig.mExtraData;
41 : }
42 0 : }
43 :
44 0 : H264Converter::~H264Converter()
45 : {
46 0 : }
47 :
48 : RefPtr<MediaDataDecoder::InitPromise>
49 0 : H264Converter::Init()
50 : {
51 0 : if (mDecoder) {
52 0 : return mDecoder->Init();
53 : }
54 :
55 : // We haven't been able to initialize a decoder due to a missing SPS/PPS.
56 : return MediaDataDecoder::InitPromise::CreateAndResolve(
57 0 : TrackType::kVideoTrack, __func__);
58 : }
59 :
60 : RefPtr<MediaDataDecoder::DecodePromise>
61 0 : H264Converter::Decode(MediaRawData* aSample)
62 : {
63 0 : MOZ_RELEASE_ASSERT(mFlushPromise.IsEmpty(), "Flush operatin didn't complete");
64 :
65 0 : MOZ_RELEASE_ASSERT(!mDecodePromiseRequest.Exists()
66 : && !mInitPromiseRequest.Exists(),
67 : "Can't request a new decode until previous one completed");
68 :
69 0 : if (!mp4_demuxer::AnnexB::ConvertSampleToAVCC(aSample)) {
70 : // We need AVCC content to be able to later parse the SPS.
71 : // This is a no-op if the data is already AVCC.
72 : return DecodePromise::CreateAndReject(
73 0 : MediaResult(NS_ERROR_OUT_OF_MEMORY, RESULT_DETAIL("ConvertSampleToAVCC")),
74 0 : __func__);
75 : }
76 :
77 : nsresult rv;
78 0 : if (!mDecoder) {
79 : // It is not possible to create an AVCC H264 decoder without SPS.
80 : // As such, creation will fail if the extra_data just extracted doesn't
81 : // contain a SPS.
82 0 : rv = CreateDecoderAndInit(aSample);
83 0 : if (rv == NS_ERROR_NOT_INITIALIZED) {
84 : // We are missing the required SPS to create the decoder.
85 : // Ignore for the time being, the MediaRawData will be dropped.
86 0 : return DecodePromise::CreateAndResolve(DecodedData(), __func__);
87 : }
88 : } else {
89 : // Initialize the members that we couldn't if the extradata was given during
90 : // H264Converter's construction.
91 0 : if (!mNeedAVCC) {
92 : mNeedAVCC =
93 0 : Some(mDecoder->NeedsConversion() == ConversionRequired::kNeedAVCC);
94 : }
95 0 : if (!mCanRecycleDecoder) {
96 0 : mCanRecycleDecoder = Some(CanRecycleDecoder());
97 : }
98 0 : rv = CheckForSPSChange(aSample);
99 : }
100 :
101 0 : if (rv == NS_ERROR_DOM_MEDIA_INITIALIZING_DECODER) {
102 : // The decoder is pending initialization.
103 0 : RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__);
104 0 : return p;
105 : }
106 :
107 0 : if (NS_FAILED(rv)) {
108 : return DecodePromise::CreateAndReject(
109 0 : MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
110 0 : RESULT_DETAIL("Unable to create H264 decoder")),
111 0 : __func__);
112 : }
113 :
114 0 : if (mNeedKeyframe && !aSample->mKeyframe) {
115 0 : return DecodePromise::CreateAndResolve(DecodedData(), __func__);
116 : }
117 :
118 0 : if (!*mNeedAVCC
119 0 : && !mp4_demuxer::AnnexB::ConvertSampleToAnnexB(aSample, mNeedKeyframe)) {
120 : return DecodePromise::CreateAndReject(
121 0 : MediaResult(NS_ERROR_OUT_OF_MEMORY,
122 0 : RESULT_DETAIL("ConvertSampleToAnnexB")),
123 0 : __func__);
124 : }
125 :
126 0 : mNeedKeyframe = false;
127 :
128 0 : aSample->mExtraData = mCurrentConfig.mExtraData;
129 :
130 0 : return mDecoder->Decode(aSample);
131 : }
132 :
133 : RefPtr<MediaDataDecoder::FlushPromise>
134 0 : H264Converter::Flush()
135 : {
136 0 : mDecodePromiseRequest.DisconnectIfExists();
137 0 : mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
138 0 : mNeedKeyframe = true;
139 0 : mPendingFrames.Clear();
140 :
141 0 : MOZ_RELEASE_ASSERT(mFlushPromise.IsEmpty(), "Previous flush didn't complete");
142 :
143 : /*
144 : When we detect a change of content in the H264 stream, we first drain the
145 : current decoder (1), flush (2), shut it down (3) create a new decoder and
146 : initialize it (4). It is possible for H264Converter::Flush to be called
147 : during any of those times.
148 : If during (1):
149 : - mDrainRequest will not be empty.
150 : - The old decoder can still be used, with the current extradata as stored
151 : in mCurrentConfig.mExtraData.
152 :
153 : If during (2):
154 : - mFlushRequest will not be empty.
155 : - The old decoder can still be used, with the current extradata as stored
156 : in mCurrentConfig.mExtraData.
157 :
158 : If during (3):
159 : - mShutdownRequest won't be empty.
160 : - mDecoder is empty.
161 : - The old decoder is no longer referenced by the H264Converter.
162 :
163 : If during (4):
164 : - mInitPromiseRequest won't be empty.
165 : - mDecoder is set but not usable yet.
166 : */
167 :
168 0 : if (mDrainRequest.Exists() || mFlushRequest.Exists() ||
169 0 : mShutdownRequest.Exists() || mInitPromiseRequest.Exists()) {
170 : // We let the current decoder complete and will resume after.
171 0 : return mFlushPromise.Ensure(__func__);
172 : }
173 0 : if (mDecoder) {
174 0 : return mDecoder->Flush();
175 : }
176 0 : return FlushPromise::CreateAndResolve(true, __func__);
177 : }
178 :
179 : RefPtr<MediaDataDecoder::DecodePromise>
180 0 : H264Converter::Drain()
181 : {
182 0 : MOZ_RELEASE_ASSERT(!mDrainRequest.Exists());
183 0 : mNeedKeyframe = true;
184 0 : if (mDecoder) {
185 0 : return mDecoder->Drain();
186 : }
187 0 : return DecodePromise::CreateAndResolve(DecodedData(), __func__);
188 : }
189 :
190 : RefPtr<ShutdownPromise>
191 0 : H264Converter::Shutdown()
192 : {
193 0 : mInitPromiseRequest.DisconnectIfExists();
194 0 : mDecodePromiseRequest.DisconnectIfExists();
195 0 : mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
196 0 : mDrainRequest.DisconnectIfExists();
197 0 : mFlushRequest.DisconnectIfExists();
198 0 : mFlushPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
199 0 : mShutdownRequest.DisconnectIfExists();
200 :
201 0 : if (mShutdownPromise) {
202 : // We have a shutdown in progress, return that promise instead as we can't
203 : // shutdown a decoder twice.
204 0 : return mShutdownPromise.forget();
205 : }
206 0 : return ShutdownDecoder();
207 : }
208 :
209 : RefPtr<ShutdownPromise>
210 0 : H264Converter::ShutdownDecoder()
211 : {
212 0 : mNeedAVCC.reset();
213 0 : if (mDecoder) {
214 0 : RefPtr<MediaDataDecoder> decoder = mDecoder.forget();
215 0 : return decoder->Shutdown();
216 : }
217 0 : return ShutdownPromise::CreateAndResolve(true, __func__);
218 : }
219 :
220 : bool
221 0 : H264Converter::IsHardwareAccelerated(nsACString& aFailureReason) const
222 : {
223 0 : if (mDecoder) {
224 0 : return mDecoder->IsHardwareAccelerated(aFailureReason);
225 : }
226 0 : return MediaDataDecoder::IsHardwareAccelerated(aFailureReason);
227 : }
228 :
229 : void
230 0 : H264Converter::SetSeekThreshold(const media::TimeUnit& aTime)
231 : {
232 0 : if (mDecoder) {
233 0 : mDecoder->SetSeekThreshold(aTime);
234 : } else {
235 0 : MediaDataDecoder::SetSeekThreshold(aTime);
236 : }
237 0 : }
238 :
239 : nsresult
240 0 : H264Converter::CreateDecoder(const VideoInfo& aConfig,
241 : DecoderDoctorDiagnostics* aDiagnostics)
242 : {
243 0 : if (!mp4_demuxer::H264::HasSPS(aConfig.mExtraData)) {
244 : // nothing found yet, will try again later
245 0 : return NS_ERROR_NOT_INITIALIZED;
246 : }
247 0 : UpdateConfigFromExtraData(aConfig.mExtraData);
248 :
249 0 : mp4_demuxer::SPSData spsdata;
250 0 : if (mp4_demuxer::H264::DecodeSPSFromExtraData(aConfig.mExtraData, spsdata)) {
251 : // Do some format check here.
252 : // WMF H.264 Video Decoder and Apple ATDecoder do not support YUV444 format.
253 0 : if (spsdata.profile_idc == 244 /* Hi444PP */
254 0 : || spsdata.chroma_format_idc == PDMFactory::kYUV444) {
255 0 : mLastError = NS_ERROR_FAILURE;
256 0 : if (aDiagnostics) {
257 0 : aDiagnostics->SetVideoNotSupported();
258 : }
259 0 : return NS_ERROR_FAILURE;
260 : }
261 : } else {
262 : // SPS was invalid.
263 0 : mLastError = NS_ERROR_FAILURE;
264 0 : return NS_ERROR_FAILURE;
265 : }
266 :
267 0 : mDecoder = mPDM->CreateVideoDecoder({
268 : aConfig,
269 : mTaskQueue,
270 : aDiagnostics,
271 : mImageContainer,
272 : mKnowsCompositor,
273 : mGMPCrashHelper,
274 : mType,
275 : mOnWaitingForKeyEvent,
276 : mDecoderOptions
277 0 : });
278 :
279 0 : if (!mDecoder) {
280 0 : mLastError = NS_ERROR_FAILURE;
281 0 : return NS_ERROR_FAILURE;
282 : }
283 :
284 0 : mNeedKeyframe = true;
285 :
286 0 : return NS_OK;
287 : }
288 :
289 : nsresult
290 0 : H264Converter::CreateDecoderAndInit(MediaRawData* aSample)
291 : {
292 : RefPtr<MediaByteBuffer> extra_data =
293 0 : mp4_demuxer::H264::ExtractExtraData(aSample);
294 0 : bool inbandExtradata = mp4_demuxer::H264::HasSPS(extra_data);
295 0 : if (!inbandExtradata &&
296 0 : !mp4_demuxer::H264::HasSPS(mCurrentConfig.mExtraData)) {
297 0 : return NS_ERROR_NOT_INITIALIZED;
298 : }
299 :
300 0 : if (inbandExtradata) {
301 0 : UpdateConfigFromExtraData(extra_data);
302 : }
303 :
304 : nsresult rv =
305 0 : CreateDecoder(mCurrentConfig, /* DecoderDoctorDiagnostics* */ nullptr);
306 :
307 0 : if (NS_SUCCEEDED(rv)) {
308 0 : RefPtr<H264Converter> self = this;
309 0 : RefPtr<MediaRawData> sample = aSample;
310 0 : mDecoder->Init()
311 0 : ->Then(
312 0 : AbstractThread::GetCurrent()->AsTaskQueue(),
313 : __func__,
314 0 : [self, sample, this](const TrackType aTrackType) {
315 0 : mInitPromiseRequest.Complete();
316 : mNeedAVCC =
317 0 : Some(mDecoder->NeedsConversion() == ConversionRequired::kNeedAVCC);
318 0 : mCanRecycleDecoder = Some(CanRecycleDecoder());
319 :
320 0 : if (!mFlushPromise.IsEmpty()) {
321 : // A Flush is pending, abort the current operation.
322 0 : mFlushPromise.Resolve(true, __func__);
323 0 : return;
324 : }
325 :
326 0 : DecodeFirstSample(sample);
327 : },
328 0 : [self, this](const MediaResult& aError) {
329 0 : mInitPromiseRequest.Complete();
330 :
331 0 : if (!mFlushPromise.IsEmpty()) {
332 : // A Flush is pending, abort the current operation.
333 0 : mFlushPromise.Reject(aError, __func__);
334 0 : return;
335 : }
336 :
337 0 : mDecodePromise.Reject(
338 0 : MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
339 0 : RESULT_DETAIL("Unable to initialize H264 decoder")),
340 0 : __func__);
341 : })
342 0 : ->Track(mInitPromiseRequest);
343 0 : return NS_ERROR_DOM_MEDIA_INITIALIZING_DECODER;
344 : }
345 0 : return rv;
346 : }
347 :
348 : bool
349 0 : H264Converter::CanRecycleDecoder() const
350 : {
351 0 : MOZ_ASSERT(mDecoder);
352 0 : return MediaPrefs::MediaDecoderCheckRecycling()
353 0 : && mDecoder->SupportDecoderRecycling();
354 : }
355 :
356 : void
357 0 : H264Converter::DecodeFirstSample(MediaRawData* aSample)
358 : {
359 0 : if (mNeedKeyframe && !aSample->mKeyframe) {
360 0 : mDecodePromise.Resolve(mPendingFrames, __func__);
361 0 : mPendingFrames.Clear();
362 0 : return;
363 : }
364 :
365 0 : if (!*mNeedAVCC
366 0 : && !mp4_demuxer::AnnexB::ConvertSampleToAnnexB(aSample, mNeedKeyframe)) {
367 0 : mDecodePromise.Reject(
368 0 : MediaResult(NS_ERROR_OUT_OF_MEMORY,
369 0 : RESULT_DETAIL("ConvertSampleToAnnexB")),
370 0 : __func__);
371 0 : return;
372 : }
373 :
374 0 : mNeedKeyframe = false;
375 :
376 0 : RefPtr<H264Converter> self = this;
377 0 : mDecoder->Decode(aSample)
378 0 : ->Then(AbstractThread::GetCurrent()->AsTaskQueue(), __func__,
379 0 : [self, this](const MediaDataDecoder::DecodedData& aResults) {
380 0 : mDecodePromiseRequest.Complete();
381 0 : mPendingFrames.AppendElements(aResults);
382 0 : mDecodePromise.Resolve(mPendingFrames, __func__);
383 0 : mPendingFrames.Clear();
384 0 : },
385 0 : [self, this](const MediaResult& aError) {
386 0 : mDecodePromiseRequest.Complete();
387 0 : mDecodePromise.Reject(aError, __func__);
388 0 : })
389 0 : ->Track(mDecodePromiseRequest);
390 : }
391 :
392 : nsresult
393 0 : H264Converter::CheckForSPSChange(MediaRawData* aSample)
394 : {
395 : RefPtr<MediaByteBuffer> extra_data =
396 0 : mp4_demuxer::H264::ExtractExtraData(aSample);
397 0 : if (!mp4_demuxer::H264::HasSPS(extra_data)) {
398 0 : MOZ_ASSERT(mCanRecycleDecoder.isSome());
399 0 : if (!*mCanRecycleDecoder) {
400 : // If the decoder can't be recycled, the out of band extradata will never
401 : // change as the H264Converter will be recreated by the MediaFormatReader
402 : // instead. So there's no point in testing for changes.
403 0 : return NS_OK;
404 : }
405 : // This sample doesn't contain inband SPS/PPS
406 : // We now check if the out of band one has changed.
407 : // This scenario can only occur on Android with devices that can recycle a
408 : // decoder.
409 0 : if (!mp4_demuxer::H264::HasSPS(aSample->mExtraData) ||
410 0 : mp4_demuxer::H264::CompareExtraData(aSample->mExtraData,
411 0 : mOriginalExtraData)) {
412 0 : return NS_OK;
413 : }
414 0 : extra_data = mOriginalExtraData = aSample->mExtraData;
415 : }
416 0 : if (mp4_demuxer::H264::CompareExtraData(extra_data,
417 0 : mCurrentConfig.mExtraData)) {
418 0 : return NS_OK;
419 : }
420 :
421 0 : MOZ_ASSERT(mCanRecycleDecoder.isSome());
422 0 : if (*mCanRecycleDecoder) {
423 : // Do not recreate the decoder, reuse it.
424 0 : UpdateConfigFromExtraData(extra_data);
425 0 : if (!aSample->mTrackInfo) {
426 0 : aSample->mTrackInfo = new TrackInfoSharedPtr(mCurrentConfig, 0);
427 : }
428 0 : mNeedKeyframe = true;
429 0 : return NS_OK;
430 : }
431 :
432 : // The SPS has changed, signal to drain the current decoder and once done
433 : // create a new one.
434 0 : DrainThenFlushDecoder(aSample);
435 0 : return NS_ERROR_DOM_MEDIA_INITIALIZING_DECODER;
436 : }
437 :
438 : void
439 0 : H264Converter::DrainThenFlushDecoder(MediaRawData* aPendingSample)
440 : {
441 0 : RefPtr<MediaRawData> sample = aPendingSample;
442 0 : RefPtr<H264Converter> self = this;
443 0 : mDecoder->Drain()
444 0 : ->Then(AbstractThread::GetCurrent()->AsTaskQueue(),
445 : __func__,
446 0 : [self, sample, this](const MediaDataDecoder::DecodedData& aResults) {
447 0 : mDrainRequest.Complete();
448 0 : if (!mFlushPromise.IsEmpty()) {
449 : // A Flush is pending, abort the current operation.
450 0 : mFlushPromise.Resolve(true, __func__);
451 0 : return;
452 : }
453 0 : if (aResults.Length() > 0) {
454 0 : mPendingFrames.AppendElements(aResults);
455 0 : DrainThenFlushDecoder(sample);
456 0 : return;
457 : }
458 : // We've completed the draining operation, we can now flush the
459 : // decoder.
460 0 : FlushThenShutdownDecoder(sample);
461 : },
462 0 : [self, this](const MediaResult& aError) {
463 0 : mDrainRequest.Complete();
464 0 : if (!mFlushPromise.IsEmpty()) {
465 : // A Flush is pending, abort the current operation.
466 0 : mFlushPromise.Reject(aError, __func__);
467 0 : return;
468 : }
469 0 : mDecodePromise.Reject(aError, __func__);
470 : })
471 0 : ->Track(mDrainRequest);
472 0 : }
473 :
474 0 : void H264Converter::FlushThenShutdownDecoder(MediaRawData* aPendingSample)
475 : {
476 0 : RefPtr<MediaRawData> sample = aPendingSample;
477 0 : RefPtr<H264Converter> self = this;
478 0 : mDecoder->Flush()
479 0 : ->Then(AbstractThread::GetCurrent()->AsTaskQueue(),
480 : __func__,
481 0 : [self, sample, this]() {
482 0 : mFlushRequest.Complete();
483 :
484 0 : if (!mFlushPromise.IsEmpty()) {
485 : // A Flush is pending, abort the current operation.
486 0 : mFlushPromise.Resolve(true, __func__);
487 0 : return;
488 : }
489 :
490 0 : mShutdownPromise = ShutdownDecoder();
491 : mShutdownPromise
492 0 : ->Then(AbstractThread::GetCurrent()->AsTaskQueue(),
493 : __func__,
494 0 : [self, sample, this]() {
495 0 : mShutdownRequest.Complete();
496 0 : mShutdownPromise = nullptr;
497 :
498 0 : if (!mFlushPromise.IsEmpty()) {
499 : // A Flush is pending, abort the current operation.
500 0 : mFlushPromise.Resolve(true, __func__);
501 0 : return;
502 : }
503 :
504 0 : nsresult rv = CreateDecoderAndInit(sample);
505 0 : if (rv == NS_ERROR_DOM_MEDIA_INITIALIZING_DECODER) {
506 : // All good so far, will continue later.
507 0 : return;
508 : }
509 0 : MOZ_ASSERT(NS_FAILED(rv));
510 0 : mDecodePromise.Reject(rv, __func__);
511 0 : return;
512 : },
513 0 : [] { MOZ_CRASH("Can't reach here'"); })
514 0 : ->Track(mShutdownRequest);
515 : },
516 0 : [self, this](const MediaResult& aError) {
517 0 : mFlushRequest.Complete();
518 0 : if (!mFlushPromise.IsEmpty()) {
519 : // A Flush is pending, abort the current operation.
520 0 : mFlushPromise.Reject(aError, __func__);
521 0 : return;
522 : }
523 0 : mDecodePromise.Reject(aError, __func__);
524 : })
525 0 : ->Track(mFlushRequest);
526 0 : }
527 :
528 : void
529 0 : H264Converter::UpdateConfigFromExtraData(MediaByteBuffer* aExtraData)
530 : {
531 0 : mp4_demuxer::SPSData spsdata;
532 0 : if (mp4_demuxer::H264::DecodeSPSFromExtraData(aExtraData, spsdata)
533 0 : && spsdata.pic_width > 0
534 0 : && spsdata.pic_height > 0) {
535 0 : mp4_demuxer::H264::EnsureSPSIsSane(spsdata);
536 0 : mCurrentConfig.mImage.width = spsdata.pic_width;
537 0 : mCurrentConfig.mImage.height = spsdata.pic_height;
538 0 : mCurrentConfig.mDisplay.width = spsdata.display_width;
539 0 : mCurrentConfig.mDisplay.height = spsdata.display_height;
540 : }
541 0 : mCurrentConfig.mExtraData = aExtraData;
542 0 : }
543 :
544 : } // namespace mozilla
|
