Line data Source code
1 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 : /* This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this
5 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #include "FFmpegVideoDecoder.h"
8 : #include "FFmpegLog.h"
9 : #include "ImageContainer.h"
10 : #include "MediaInfo.h"
11 : #include "MP4Decoder.h"
12 : #include "VPXDecoder.h"
13 : #include "mozilla/layers/KnowsCompositor.h"
14 :
15 : #include "libavutil/pixfmt.h"
16 : #if LIBAVCODEC_VERSION_MAJOR < 54
17 : #define AVPixelFormat PixelFormat
18 : #define AV_PIX_FMT_YUV420P PIX_FMT_YUV420P
19 : #define AV_PIX_FMT_YUVJ420P PIX_FMT_YUVJ420P
20 : #define AV_PIX_FMT_YUV444P PIX_FMT_YUV444P
21 : #define AV_PIX_FMT_NONE PIX_FMT_NONE
22 : #endif
23 : #include "mozilla/PodOperations.h"
24 : #include "mozilla/TaskQueue.h"
25 : #include "nsThreadUtils.h"
26 :
27 :
28 : typedef mozilla::layers::Image Image;
29 : typedef mozilla::layers::PlanarYCbCrImage PlanarYCbCrImage;
30 :
31 : namespace mozilla {
32 :
33 : using media::TimeUnit;
34 :
35 : /**
36 : * FFmpeg calls back to this function with a list of pixel formats it supports.
37 : * We choose a pixel format that we support and return it.
38 : * For now, we just look for YUV420P, YUVJ420P and YUV444 as those are the only
39 : * only non-HW accelerated format supported by FFmpeg's H264 and VP9 decoder.
40 : */
41 : static AVPixelFormat
42 0 : ChoosePixelFormat(AVCodecContext* aCodecContext, const AVPixelFormat* aFormats)
43 : {
44 0 : FFMPEG_LOG("Choosing FFmpeg pixel format for video decoding.");
45 0 : for (; *aFormats > -1; aFormats++) {
46 0 : switch (*aFormats) {
47 : case AV_PIX_FMT_YUV444P:
48 0 : FFMPEG_LOG("Requesting pixel format YUV444P.");
49 0 : return AV_PIX_FMT_YUV444P;
50 : case AV_PIX_FMT_YUV420P:
51 0 : FFMPEG_LOG("Requesting pixel format YUV420P.");
52 0 : return AV_PIX_FMT_YUV420P;
53 : case AV_PIX_FMT_YUVJ420P:
54 0 : FFMPEG_LOG("Requesting pixel format YUVJ420P.");
55 0 : return AV_PIX_FMT_YUVJ420P;
56 : default:
57 0 : break;
58 : }
59 : }
60 :
61 0 : NS_WARNING("FFmpeg does not share any supported pixel formats.");
62 0 : return AV_PIX_FMT_NONE;
63 : }
64 :
65 0 : FFmpegVideoDecoder<LIBAV_VER>::PtsCorrectionContext::PtsCorrectionContext()
66 : : mNumFaultyPts(0)
67 : , mNumFaultyDts(0)
68 : , mLastPts(INT64_MIN)
69 0 : , mLastDts(INT64_MIN)
70 : {
71 0 : }
72 :
73 : int64_t
74 0 : FFmpegVideoDecoder<LIBAV_VER>::PtsCorrectionContext::GuessCorrectPts(
75 : int64_t aPts, int64_t aDts)
76 : {
77 0 : int64_t pts = AV_NOPTS_VALUE;
78 :
79 0 : if (aDts != int64_t(AV_NOPTS_VALUE)) {
80 0 : mNumFaultyDts += aDts <= mLastDts;
81 0 : mLastDts = aDts;
82 : }
83 0 : if (aPts != int64_t(AV_NOPTS_VALUE)) {
84 0 : mNumFaultyPts += aPts <= mLastPts;
85 0 : mLastPts = aPts;
86 : }
87 0 : if ((mNumFaultyPts <= mNumFaultyDts || aDts == int64_t(AV_NOPTS_VALUE))
88 0 : && aPts != int64_t(AV_NOPTS_VALUE)) {
89 0 : pts = aPts;
90 : } else {
91 0 : pts = aDts;
92 : }
93 0 : return pts;
94 : }
95 :
96 : void
97 0 : FFmpegVideoDecoder<LIBAV_VER>::PtsCorrectionContext::Reset()
98 : {
99 0 : mNumFaultyPts = 0;
100 0 : mNumFaultyDts = 0;
101 0 : mLastPts = INT64_MIN;
102 0 : mLastDts = INT64_MIN;
103 0 : }
104 :
105 0 : FFmpegVideoDecoder<LIBAV_VER>::FFmpegVideoDecoder(
106 : FFmpegLibWrapper* aLib, TaskQueue* aTaskQueue, const VideoInfo& aConfig,
107 : KnowsCompositor* aAllocator, ImageContainer* aImageContainer,
108 0 : bool aLowLatency)
109 : : FFmpegDataDecoder(aLib, aTaskQueue, GetCodecId(aConfig.mMimeType))
110 : , mImageAllocator(aAllocator)
111 : , mImageContainer(aImageContainer)
112 : , mInfo(aConfig)
113 : , mCodecParser(nullptr)
114 : , mLastInputDts(INT64_MIN)
115 0 : , mLowLatency(aLowLatency)
116 : {
117 0 : MOZ_COUNT_CTOR(FFmpegVideoDecoder);
118 : // Use a new MediaByteBuffer as the object will be modified during
119 : // initialization.
120 0 : mExtraData = new MediaByteBuffer;
121 0 : mExtraData->AppendElements(*aConfig.mExtraData);
122 0 : }
123 :
124 : RefPtr<MediaDataDecoder::InitPromise>
125 0 : FFmpegVideoDecoder<LIBAV_VER>::Init()
126 : {
127 0 : if (NS_FAILED(InitDecoder())) {
128 0 : return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
129 : }
130 :
131 0 : return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__);
132 : }
133 :
134 : void
135 0 : FFmpegVideoDecoder<LIBAV_VER>::InitCodecContext()
136 : {
137 0 : mCodecContext->width = mInfo.mImage.width;
138 0 : mCodecContext->height = mInfo.mImage.height;
139 :
140 : // We use the same logic as libvpx in determining the number of threads to use
141 : // so that we end up behaving in the same fashion when using ffmpeg as
142 : // we would otherwise cause various crashes (see bug 1236167)
143 0 : int decode_threads = 1;
144 0 : if (mInfo.mDisplay.width >= 2048) {
145 0 : decode_threads = 8;
146 0 : } else if (mInfo.mDisplay.width >= 1024) {
147 0 : decode_threads = 4;
148 0 : } else if (mInfo.mDisplay.width >= 320) {
149 0 : decode_threads = 2;
150 : }
151 :
152 0 : if (mLowLatency) {
153 0 : mCodecContext->flags |= CODEC_FLAG_LOW_DELAY;
154 : // ffvp9 and ffvp8 at this stage do not support slice threading, but it may
155 : // help with the h264 decoder if there's ever one.
156 0 : mCodecContext->thread_type = FF_THREAD_SLICE;
157 : } else {
158 0 : decode_threads = std::min(decode_threads, PR_GetNumberOfProcessors() - 1);
159 0 : decode_threads = std::max(decode_threads, 1);
160 0 : mCodecContext->thread_count = decode_threads;
161 0 : if (decode_threads > 1) {
162 0 : mCodecContext->thread_type = FF_THREAD_SLICE | FF_THREAD_FRAME;
163 : }
164 : }
165 :
166 : // FFmpeg will call back to this to negotiate a video pixel format.
167 0 : mCodecContext->get_format = ChoosePixelFormat;
168 :
169 0 : mCodecParser = mLib->av_parser_init(mCodecID);
170 0 : if (mCodecParser) {
171 0 : mCodecParser->flags |= PARSER_FLAG_COMPLETE_FRAMES;
172 : }
173 0 : }
174 :
175 : RefPtr<MediaDataDecoder::DecodePromise>
176 0 : FFmpegVideoDecoder<LIBAV_VER>::ProcessDecode(MediaRawData* aSample)
177 : {
178 0 : bool gotFrame = false;
179 0 : DecodedData results;
180 0 : MediaResult rv = DoDecode(aSample, &gotFrame, results);
181 0 : if (NS_FAILED(rv)) {
182 0 : return DecodePromise::CreateAndReject(rv, __func__);
183 : }
184 0 : return DecodePromise::CreateAndResolve(Move(results), __func__);
185 : }
186 :
187 : MediaResult
188 0 : FFmpegVideoDecoder<LIBAV_VER>::DoDecode(MediaRawData* aSample, bool* aGotFrame,
189 : MediaDataDecoder::DecodedData& aResults)
190 : {
191 0 : uint8_t* inputData = const_cast<uint8_t*>(aSample->Data());
192 0 : size_t inputSize = aSample->Size();
193 :
194 : #if LIBAVCODEC_VERSION_MAJOR >= 54
195 0 : if (inputSize && mCodecParser && (mCodecID == AV_CODEC_ID_VP8
196 : #if LIBAVCODEC_VERSION_MAJOR >= 55
197 0 : || mCodecID == AV_CODEC_ID_VP9
198 : #endif
199 : )) {
200 0 : while (inputSize) {
201 : uint8_t* data;
202 : int size;
203 0 : int len = mLib->av_parser_parse2(
204 : mCodecParser, mCodecContext, &data, &size, inputData, inputSize,
205 : aSample->mTime.ToMicroseconds(), aSample->mTimecode.ToMicroseconds(),
206 0 : aSample->mOffset);
207 0 : if (size_t(len) > inputSize) {
208 0 : return NS_ERROR_DOM_MEDIA_DECODE_ERR;
209 : }
210 0 : inputData += len;
211 0 : inputSize -= len;
212 0 : if (size) {
213 0 : bool gotFrame = false;
214 0 : MediaResult rv = DoDecode(aSample, data, size, &gotFrame, aResults);
215 0 : if (NS_FAILED(rv)) {
216 0 : return rv;
217 : }
218 0 : if (gotFrame && aGotFrame) {
219 0 : *aGotFrame = true;
220 : }
221 : }
222 : }
223 0 : return NS_OK;
224 : }
225 : #endif
226 0 : return DoDecode(aSample, inputData, inputSize, aGotFrame, aResults);
227 : }
228 :
229 : MediaResult
230 0 : FFmpegVideoDecoder<LIBAV_VER>::DoDecode(MediaRawData* aSample,
231 : uint8_t* aData, int aSize,
232 : bool* aGotFrame,
233 : MediaDataDecoder::DecodedData& aResults)
234 : {
235 : AVPacket packet;
236 0 : mLib->av_init_packet(&packet);
237 :
238 0 : packet.data = aData;
239 0 : packet.size = aSize;
240 0 : packet.dts = mLastInputDts = aSample->mTimecode.ToMicroseconds();
241 0 : packet.pts = aSample->mTime.ToMicroseconds();
242 0 : packet.flags = aSample->mKeyframe ? AV_PKT_FLAG_KEY : 0;
243 0 : packet.pos = aSample->mOffset;
244 :
245 : // LibAV provides no API to retrieve the decoded sample's duration.
246 : // (FFmpeg >= 1.0 provides av_frame_get_pkt_duration)
247 : // As such we instead use a map using the dts as key that we will retrieve
248 : // later.
249 : // The map will have a typical size of 16 entry.
250 0 : mDurationMap.Insert(
251 0 : aSample->mTimecode.ToMicroseconds(), aSample->mDuration.ToMicroseconds());
252 :
253 0 : if (!PrepareFrame()) {
254 0 : NS_WARNING("FFmpeg h264 decoder failed to allocate frame.");
255 0 : return MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__);
256 : }
257 :
258 : // Required with old version of FFmpeg/LibAV
259 0 : mFrame->reordered_opaque = AV_NOPTS_VALUE;
260 :
261 : int decoded;
262 : int bytesConsumed =
263 0 : mLib->avcodec_decode_video2(mCodecContext, mFrame, &decoded, &packet);
264 :
265 0 : FFMPEG_LOG("DoDecodeFrame:decode_video: rv=%d decoded=%d "
266 : "(Input: pts(%" PRId64 ") dts(%" PRId64 ") Output: pts(%" PRId64 ") "
267 : "opaque(%" PRId64 ") pkt_pts(%" PRId64 ") pkt_dts(%" PRId64 "))",
268 : bytesConsumed, decoded, packet.pts, packet.dts, mFrame->pts,
269 : mFrame->reordered_opaque, mFrame->pkt_pts, mFrame->pkt_dts);
270 :
271 0 : if (bytesConsumed < 0) {
272 : return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
273 0 : RESULT_DETAIL("FFmpeg video error:%d", bytesConsumed));
274 : }
275 :
276 0 : if (!decoded) {
277 0 : if (aGotFrame) {
278 0 : *aGotFrame = false;
279 : }
280 0 : return NS_OK;
281 : }
282 :
283 : // If we've decoded a frame then we need to output it
284 0 : int64_t pts = mPtsContext.GuessCorrectPts(mFrame->pkt_pts, mFrame->pkt_dts);
285 : // Retrieve duration from dts.
286 : // We use the first entry found matching this dts (this is done to
287 : // handle damaged file with multiple frames with the same dts)
288 :
289 : int64_t duration;
290 0 : if (!mDurationMap.Find(mFrame->pkt_dts, duration)) {
291 0 : NS_WARNING("Unable to retrieve duration from map");
292 0 : duration = aSample->mDuration.ToMicroseconds();
293 : // dts are probably incorrectly reported ; so clear the map as we're
294 : // unlikely to find them in the future anyway. This also guards
295 : // against the map becoming extremely big.
296 0 : mDurationMap.Clear();
297 : }
298 0 : FFMPEG_LOG(
299 : "Got one frame output with pts=%" PRId64 " dts=%" PRId64
300 : " duration=%" PRId64 " opaque=%" PRId64,
301 : pts, mFrame->pkt_dts, duration, mCodecContext->reordered_opaque);
302 :
303 0 : VideoData::YCbCrBuffer b;
304 0 : b.mPlanes[0].mData = mFrame->data[0];
305 0 : b.mPlanes[1].mData = mFrame->data[1];
306 0 : b.mPlanes[2].mData = mFrame->data[2];
307 :
308 0 : b.mPlanes[0].mStride = mFrame->linesize[0];
309 0 : b.mPlanes[1].mStride = mFrame->linesize[1];
310 0 : b.mPlanes[2].mStride = mFrame->linesize[2];
311 :
312 0 : b.mPlanes[0].mOffset = b.mPlanes[0].mSkip = 0;
313 0 : b.mPlanes[1].mOffset = b.mPlanes[1].mSkip = 0;
314 0 : b.mPlanes[2].mOffset = b.mPlanes[2].mSkip = 0;
315 :
316 0 : b.mPlanes[0].mWidth = mFrame->width;
317 0 : b.mPlanes[0].mHeight = mFrame->height;
318 0 : if (mCodecContext->pix_fmt == AV_PIX_FMT_YUV444P) {
319 0 : b.mPlanes[1].mWidth = b.mPlanes[2].mWidth = mFrame->width;
320 0 : b.mPlanes[1].mHeight = b.mPlanes[2].mHeight = mFrame->height;
321 : } else {
322 0 : b.mPlanes[1].mWidth = b.mPlanes[2].mWidth = (mFrame->width + 1) >> 1;
323 0 : b.mPlanes[1].mHeight = b.mPlanes[2].mHeight = (mFrame->height + 1) >> 1;
324 : }
325 0 : if (mLib->av_frame_get_colorspace) {
326 0 : switch (mLib->av_frame_get_colorspace(mFrame)) {
327 : case AVCOL_SPC_BT709:
328 0 : b.mYUVColorSpace = YUVColorSpace::BT709;
329 0 : break;
330 : case AVCOL_SPC_SMPTE170M:
331 : case AVCOL_SPC_BT470BG:
332 0 : b.mYUVColorSpace = YUVColorSpace::BT601;
333 0 : break;
334 : case AVCOL_SPC_UNSPECIFIED:
335 : #if LIBAVCODEC_VERSION_MAJOR >= 55
336 0 : if (mCodecContext->codec_id == AV_CODEC_ID_VP9) {
337 0 : b.mYUVColorSpace = YUVColorSpace::BT709;
338 : }
339 : #endif
340 0 : break;
341 : default:
342 0 : break;
343 : }
344 : }
345 : RefPtr<VideoData> v =
346 0 : VideoData::CreateAndCopyData(mInfo,
347 : mImageContainer,
348 : aSample->mOffset,
349 0 : TimeUnit::FromMicroseconds(pts),
350 0 : TimeUnit::FromMicroseconds(duration),
351 : b,
352 0 : !!mFrame->key_frame,
353 0 : TimeUnit::FromMicroseconds(-1),
354 0 : mInfo.ScaledImageRect(mFrame->width,
355 0 : mFrame->height),
356 0 : mImageAllocator);
357 :
358 0 : if (!v) {
359 : return MediaResult(NS_ERROR_OUT_OF_MEMORY,
360 0 : RESULT_DETAIL("image allocation error"));
361 : }
362 0 : aResults.AppendElement(Move(v));
363 0 : if (aGotFrame) {
364 0 : *aGotFrame = true;
365 : }
366 0 : return NS_OK;
367 : }
368 :
369 : RefPtr<MediaDataDecoder::DecodePromise>
370 0 : FFmpegVideoDecoder<LIBAV_VER>::ProcessDrain()
371 : {
372 0 : RefPtr<MediaRawData> empty(new MediaRawData());
373 0 : empty->mTimecode = TimeUnit::FromMicroseconds(mLastInputDts);
374 0 : bool gotFrame = false;
375 0 : DecodedData results;
376 0 : while (NS_SUCCEEDED(DoDecode(empty, &gotFrame, results)) && gotFrame) {
377 : }
378 0 : return DecodePromise::CreateAndResolve(Move(results), __func__);
379 : }
380 :
381 : RefPtr<MediaDataDecoder::FlushPromise>
382 0 : FFmpegVideoDecoder<LIBAV_VER>::ProcessFlush()
383 : {
384 0 : mPtsContext.Reset();
385 0 : mDurationMap.Clear();
386 0 : return FFmpegDataDecoder::ProcessFlush();
387 : }
388 :
389 0 : FFmpegVideoDecoder<LIBAV_VER>::~FFmpegVideoDecoder()
390 : {
391 0 : MOZ_COUNT_DTOR(FFmpegVideoDecoder);
392 0 : if (mCodecParser) {
393 0 : mLib->av_parser_close(mCodecParser);
394 0 : mCodecParser = nullptr;
395 : }
396 0 : }
397 :
398 : AVCodecID
399 0 : FFmpegVideoDecoder<LIBAV_VER>::GetCodecId(const nsACString& aMimeType)
400 : {
401 0 : if (MP4Decoder::IsH264(aMimeType)) {
402 0 : return AV_CODEC_ID_H264;
403 : }
404 :
405 0 : if (aMimeType.EqualsLiteral("video/x-vnd.on2.vp6")) {
406 0 : return AV_CODEC_ID_VP6F;
407 : }
408 :
409 : #if LIBAVCODEC_VERSION_MAJOR >= 54
410 0 : if (VPXDecoder::IsVP8(aMimeType)) {
411 0 : return AV_CODEC_ID_VP8;
412 : }
413 : #endif
414 :
415 : #if LIBAVCODEC_VERSION_MAJOR >= 55
416 0 : if (VPXDecoder::IsVP9(aMimeType)) {
417 0 : return AV_CODEC_ID_VP9;
418 : }
419 : #endif
420 :
421 0 : return AV_CODEC_ID_NONE;
422 : }
423 :
424 45 : } // namespace mozilla
|
