Line data Source code
1 : /* This Source Code Form is subject to the terms of the Mozilla Public
2 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
4 :
5 : #include "WebrtcGmpVideoCodec.h"
6 :
7 : #include <iostream>
8 : #include <vector>
9 :
10 : #include "mozilla/CheckedInt.h"
11 : #include "mozilla/IntegerPrintfMacros.h"
12 : #include "mozilla/Move.h"
13 : #include "mozilla/SizePrintfMacros.h"
14 : #include "mozilla/SyncRunnable.h"
15 : #include "VideoConduit.h"
16 : #include "AudioConduit.h"
17 : #include "runnable_utils.h"
18 :
19 : #include "mozIGeckoMediaPluginService.h"
20 : #include "nsServiceManagerUtils.h"
21 : #include "GMPVideoDecoderProxy.h"
22 : #include "GMPVideoEncoderProxy.h"
23 : #include "MainThreadUtils.h"
24 :
25 : #include "gmp-video-host.h"
26 : #include "gmp-video-frame-i420.h"
27 : #include "gmp-video-frame-encoded.h"
28 : #include "webrtc/common_video/include/video_frame_buffer.h"
29 : #include "webrtc/base/bind.h"
30 :
31 : namespace mozilla {
32 :
33 : #ifdef LOG
34 : #undef LOG
35 : #endif
36 :
37 : extern mozilla::LogModule* GetGMPLog();
38 :
39 : #define LOGD(msg) MOZ_LOG(GetGMPLog(), mozilla::LogLevel::Debug, msg)
40 : #define LOG(level, msg) MOZ_LOG(GetGMPLog(), (level), msg)
41 :
42 0 : WebrtcGmpPCHandleSetter::WebrtcGmpPCHandleSetter(const std::string& aPCHandle)
43 : {
44 0 : if (!NS_IsMainThread()) {
45 0 : MOZ_ASSERT(false, "WebrtcGmpPCHandleSetter can only be used on main");
46 : return;
47 : }
48 0 : MOZ_ASSERT(sCurrentHandle.empty());
49 0 : sCurrentHandle = aPCHandle;
50 0 : }
51 :
52 0 : WebrtcGmpPCHandleSetter::~WebrtcGmpPCHandleSetter()
53 : {
54 0 : if (!NS_IsMainThread()) {
55 0 : MOZ_ASSERT(false, "WebrtcGmpPCHandleSetter can only be used on main");
56 : return;
57 : }
58 :
59 0 : sCurrentHandle.clear();
60 0 : }
61 :
62 : /* static */ std::string
63 0 : WebrtcGmpPCHandleSetter::GetCurrentHandle()
64 : {
65 0 : if (!NS_IsMainThread()) {
66 0 : MOZ_ASSERT(false, "WebrtcGmpPCHandleSetter can only be used on main");
67 : return "";
68 : }
69 :
70 0 : return sCurrentHandle;
71 : }
72 :
73 3 : std::string WebrtcGmpPCHandleSetter::sCurrentHandle = "";
74 :
75 : // Encoder.
76 0 : WebrtcGmpVideoEncoder::WebrtcGmpVideoEncoder()
77 : : mGMP(nullptr)
78 : , mInitting(false)
79 : , mHost(nullptr)
80 : , mMaxPayloadSize(0)
81 : , mCallbackMutex("WebrtcGmpVideoEncoder encoded callback mutex")
82 : , mCallback(nullptr)
83 0 : , mCachedPluginId(0)
84 : {
85 0 : if (mPCHandle.empty()) {
86 0 : mPCHandle = WebrtcGmpPCHandleSetter::GetCurrentHandle();
87 : }
88 0 : MOZ_ASSERT(!mPCHandle.empty());
89 0 : }
90 :
91 0 : WebrtcGmpVideoEncoder::~WebrtcGmpVideoEncoder()
92 : {
93 : // We should not have been destroyed if we never closed our GMP
94 0 : MOZ_ASSERT(!mGMP);
95 0 : }
96 :
97 : static int
98 0 : WebrtcFrameTypeToGmpFrameType(webrtc::FrameType aIn,
99 : GMPVideoFrameType *aOut)
100 : {
101 0 : MOZ_ASSERT(aOut);
102 0 : switch(aIn) {
103 : case webrtc::kVideoFrameKey:
104 0 : *aOut = kGMPKeyFrame;
105 0 : break;
106 : case webrtc::kVideoFrameDelta:
107 0 : *aOut = kGMPDeltaFrame;
108 0 : break;
109 : case webrtc::kEmptyFrame:
110 0 : *aOut = kGMPSkipFrame;
111 0 : break;
112 : default:
113 0 : MOZ_CRASH("Unexpected webrtc::FrameType");
114 : }
115 :
116 0 : return WEBRTC_VIDEO_CODEC_OK;
117 : }
118 :
119 : static int
120 0 : GmpFrameTypeToWebrtcFrameType(GMPVideoFrameType aIn,
121 : webrtc::FrameType *aOut)
122 : {
123 0 : MOZ_ASSERT(aOut);
124 0 : switch(aIn) {
125 : case kGMPKeyFrame:
126 0 : *aOut = webrtc::kVideoFrameKey;
127 0 : break;
128 : case kGMPDeltaFrame:
129 0 : *aOut = webrtc::kVideoFrameDelta;
130 0 : break;
131 : case kGMPSkipFrame:
132 0 : *aOut = webrtc::kEmptyFrame;
133 0 : break;
134 : default:
135 0 : MOZ_CRASH("Unexpected GMPVideoFrameType");
136 : }
137 :
138 0 : return WEBRTC_VIDEO_CODEC_OK;
139 : }
140 :
141 : int32_t
142 0 : WebrtcGmpVideoEncoder::InitEncode(const webrtc::VideoCodec* aCodecSettings,
143 : int32_t aNumberOfCores,
144 : uint32_t aMaxPayloadSize)
145 : {
146 0 : if (!mMPS) {
147 0 : mMPS = do_GetService("@mozilla.org/gecko-media-plugin-service;1");
148 : }
149 0 : MOZ_ASSERT(mMPS);
150 :
151 0 : if (!mGMPThread) {
152 0 : if (NS_WARN_IF(NS_FAILED(mMPS->GetThread(getter_AddRefs(mGMPThread))))) {
153 0 : return WEBRTC_VIDEO_CODEC_ERROR;
154 : }
155 : }
156 :
157 : // Bug XXXXXX: transfer settings from codecSettings to codec.
158 : GMPVideoCodec codecParams;
159 0 : memset(&codecParams, 0, sizeof(codecParams));
160 :
161 0 : codecParams.mGMPApiVersion = 33;
162 0 : codecParams.mStartBitrate = aCodecSettings->startBitrate;
163 0 : codecParams.mMinBitrate = aCodecSettings->minBitrate;
164 0 : codecParams.mMaxBitrate = aCodecSettings->maxBitrate;
165 0 : codecParams.mMaxFramerate = aCodecSettings->maxFramerate;
166 0 : mMaxPayloadSize = aMaxPayloadSize;
167 :
168 0 : memset(&mCodecSpecificInfo, 0, sizeof(webrtc::CodecSpecificInfo));
169 0 : mCodecSpecificInfo.codecType = webrtc::kVideoCodecH264;
170 0 : mCodecSpecificInfo.codecSpecific.H264.packetization_mode =
171 0 : aCodecSettings->H264().packetizationMode == 1 ?
172 : webrtc::H264PacketizationMode::NonInterleaved :
173 : webrtc::H264PacketizationMode::SingleNalUnit;
174 :
175 0 : if (mCodecSpecificInfo.codecSpecific.H264.packetization_mode ==
176 : webrtc::H264PacketizationMode::NonInterleaved) {
177 0 : mMaxPayloadSize = 0; // No limit, use FUAs
178 : }
179 :
180 0 : if (aCodecSettings->mode == webrtc::kScreensharing) {
181 0 : codecParams.mMode = kGMPScreensharing;
182 : } else {
183 0 : codecParams.mMode = kGMPRealtimeVideo;
184 : }
185 :
186 0 : codecParams.mWidth = aCodecSettings->width;
187 0 : codecParams.mHeight = aCodecSettings->height;
188 :
189 0 : RefPtr<GmpInitDoneRunnable> initDone(new GmpInitDoneRunnable(mPCHandle));
190 0 : mGMPThread->Dispatch(WrapRunnableNM(WebrtcGmpVideoEncoder::InitEncode_g,
191 0 : RefPtr<WebrtcGmpVideoEncoder>(this),
192 : codecParams,
193 : aNumberOfCores,
194 : aMaxPayloadSize,
195 : initDone),
196 0 : NS_DISPATCH_NORMAL);
197 :
198 : // Since init of the GMP encoder is a multi-step async dispatch (including
199 : // dispatches to main), and since this function is invoked on main, there's
200 : // no safe way to block until this init is done. If an error occurs, we'll
201 : // handle it later.
202 0 : return WEBRTC_VIDEO_CODEC_OK;
203 : }
204 :
205 : /* static */
206 : void
207 0 : WebrtcGmpVideoEncoder::InitEncode_g(
208 : const RefPtr<WebrtcGmpVideoEncoder>& aThis,
209 : const GMPVideoCodec& aCodecParams,
210 : int32_t aNumberOfCores,
211 : uint32_t aMaxPayloadSize,
212 : const RefPtr<GmpInitDoneRunnable>& aInitDone)
213 : {
214 0 : nsTArray<nsCString> tags;
215 0 : tags.AppendElement(NS_LITERAL_CSTRING("h264"));
216 : UniquePtr<GetGMPVideoEncoderCallback> callback(
217 0 : new InitDoneCallback(aThis, aInitDone, aCodecParams, aMaxPayloadSize));
218 0 : aThis->mInitting = true;
219 0 : nsresult rv = aThis->mMPS->GetGMPVideoEncoder(nullptr,
220 : &tags,
221 0 : NS_LITERAL_CSTRING(""),
222 0 : Move(callback));
223 0 : if (NS_WARN_IF(NS_FAILED(rv))) {
224 0 : LOGD(("GMP Encode: GetGMPVideoEncoder failed"));
225 0 : aThis->Close_g();
226 0 : aInitDone->Dispatch(WEBRTC_VIDEO_CODEC_ERROR,
227 0 : "GMP Encode: GetGMPVideoEncoder failed");
228 : }
229 0 : }
230 :
231 : int32_t
232 0 : WebrtcGmpVideoEncoder::GmpInitDone(GMPVideoEncoderProxy* aGMP,
233 : GMPVideoHost* aHost,
234 : std::string* aErrorOut)
235 : {
236 0 : if (!mInitting || !aGMP || !aHost) {
237 : *aErrorOut = "GMP Encode: Either init was aborted, "
238 0 : "or init failed to supply either a GMP Encoder or GMP host.";
239 0 : if (aGMP) {
240 : // This could destroy us, since aGMP may be the last thing holding a ref
241 : // Return immediately.
242 0 : aGMP->Close();
243 : }
244 0 : return WEBRTC_VIDEO_CODEC_ERROR;
245 : }
246 :
247 0 : mInitting = false;
248 :
249 0 : if (mGMP && mGMP != aGMP) {
250 0 : Close_g();
251 : }
252 :
253 0 : mGMP = aGMP;
254 0 : mHost = aHost;
255 0 : mCachedPluginId = mGMP->GetPluginId();
256 0 : return WEBRTC_VIDEO_CODEC_OK;
257 : }
258 :
259 : int32_t
260 0 : WebrtcGmpVideoEncoder::GmpInitDone(GMPVideoEncoderProxy* aGMP,
261 : GMPVideoHost* aHost,
262 : const GMPVideoCodec& aCodecParams,
263 : uint32_t aMaxPayloadSize,
264 : std::string* aErrorOut)
265 : {
266 0 : int32_t r = GmpInitDone(aGMP, aHost, aErrorOut);
267 0 : if (r != WEBRTC_VIDEO_CODEC_OK) {
268 : // We might have been destroyed if GmpInitDone failed.
269 : // Return immediately.
270 0 : return r;
271 : }
272 0 : mCodecParams = aCodecParams;
273 0 : return InitEncoderForSize(aCodecParams.mWidth,
274 0 : aCodecParams.mHeight,
275 0 : aErrorOut);
276 : }
277 :
278 : void
279 0 : WebrtcGmpVideoEncoder::Close_g()
280 : {
281 0 : GMPVideoEncoderProxy* gmp(mGMP);
282 0 : mGMP = nullptr;
283 0 : mHost = nullptr;
284 0 : mInitting = false;
285 :
286 0 : if (gmp) {
287 : // Do this last, since this could cause us to be destroyed
288 0 : gmp->Close();
289 : }
290 0 : }
291 :
292 : int32_t
293 0 : WebrtcGmpVideoEncoder::InitEncoderForSize(unsigned short aWidth,
294 : unsigned short aHeight,
295 : std::string* aErrorOut)
296 : {
297 0 : mCodecParams.mWidth = aWidth;
298 0 : mCodecParams.mHeight = aHeight;
299 : // Pass dummy codecSpecific data for now...
300 0 : nsTArray<uint8_t> codecSpecific;
301 :
302 0 : GMPErr err = mGMP->InitEncode(mCodecParams, codecSpecific, this, 1, mMaxPayloadSize);
303 0 : if (err != GMPNoErr) {
304 0 : *aErrorOut = "GMP Encode: InitEncode failed";
305 0 : return WEBRTC_VIDEO_CODEC_ERROR;
306 : }
307 :
308 0 : return WEBRTC_VIDEO_CODEC_OK;
309 : }
310 :
311 :
312 : int32_t
313 0 : WebrtcGmpVideoEncoder::Encode(const webrtc::VideoFrame& aInputImage,
314 : const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
315 : const std::vector<webrtc::FrameType>* aFrameTypes)
316 : {
317 0 : MOZ_ASSERT(aInputImage.width() >= 0 && aInputImage.height() >= 0);
318 : // Would be really nice to avoid this sync dispatch, but it would require a
319 : // copy of the frame, since it doesn't appear to actually have a refcount.
320 : // Passing 'this' is safe since this is synchronous.
321 0 : mGMPThread->Dispatch(
322 0 : WrapRunnable(this,
323 : &WebrtcGmpVideoEncoder::Encode_g,
324 : &aInputImage,
325 : aCodecSpecificInfo,
326 : aFrameTypes),
327 0 : NS_DISPATCH_SYNC);
328 :
329 0 : return WEBRTC_VIDEO_CODEC_OK;
330 : }
331 :
332 : void
333 0 : WebrtcGmpVideoEncoder::RegetEncoderForResolutionChange(
334 : uint32_t aWidth,
335 : uint32_t aHeight,
336 : const RefPtr<GmpInitDoneRunnable>& aInitDone)
337 : {
338 0 : Close_g();
339 :
340 : UniquePtr<GetGMPVideoEncoderCallback> callback(
341 : new InitDoneForResolutionChangeCallback(this,
342 : aInitDone,
343 : aWidth,
344 0 : aHeight));
345 :
346 : // OpenH264 codec (at least) can't handle dynamic input resolution changes
347 : // re-init the plugin when the resolution changes
348 : // XXX allow codec to indicate it doesn't need re-init!
349 0 : nsTArray<nsCString> tags;
350 0 : tags.AppendElement(NS_LITERAL_CSTRING("h264"));
351 0 : mInitting = true;
352 0 : if (NS_WARN_IF(NS_FAILED(mMPS->GetGMPVideoEncoder(nullptr,
353 : &tags,
354 : NS_LITERAL_CSTRING(""),
355 : Move(callback))))) {
356 0 : aInitDone->Dispatch(WEBRTC_VIDEO_CODEC_ERROR,
357 0 : "GMP Encode: GetGMPVideoEncoder failed");
358 : }
359 0 : }
360 :
361 : int32_t
362 0 : WebrtcGmpVideoEncoder::Encode_g(const webrtc::VideoFrame* aInputImage,
363 : const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
364 : const std::vector<webrtc::FrameType>* aFrameTypes)
365 : {
366 0 : if (!mGMP) {
367 : // destroyed via Terminate(), failed to init, or just not initted yet
368 0 : LOGD(("GMP Encode: not initted yet"));
369 0 : return WEBRTC_VIDEO_CODEC_ERROR;
370 : }
371 0 : MOZ_ASSERT(mHost);
372 :
373 0 : if (static_cast<uint32_t>(aInputImage->width()) != mCodecParams.mWidth ||
374 0 : static_cast<uint32_t>(aInputImage->height()) != mCodecParams.mHeight) {
375 0 : LOGD(("GMP Encode: resolution change from %ux%u to %dx%d",
376 : mCodecParams.mWidth, mCodecParams.mHeight, aInputImage->width(), aInputImage->height()));
377 :
378 0 : RefPtr<GmpInitDoneRunnable> initDone(new GmpInitDoneRunnable(mPCHandle));
379 0 : RegetEncoderForResolutionChange(aInputImage->width(),
380 0 : aInputImage->height(),
381 0 : initDone);
382 0 : if (!mGMP) {
383 : // We needed to go async to re-get the encoder. Bail.
384 0 : return WEBRTC_VIDEO_CODEC_ERROR;
385 : }
386 : }
387 :
388 0 : GMPVideoFrame* ftmp = nullptr;
389 0 : GMPErr err = mHost->CreateFrame(kGMPI420VideoFrame, &ftmp);
390 0 : if (err != GMPNoErr) {
391 0 : return WEBRTC_VIDEO_CODEC_ERROR;
392 : }
393 0 : GMPUniquePtr<GMPVideoi420Frame> frame(static_cast<GMPVideoi420Frame*>(ftmp));
394 0 : rtc::scoped_refptr<webrtc::VideoFrameBuffer> input_image = aInputImage->video_frame_buffer();
395 : // check for overflow of stride * height
396 0 : CheckedInt32 ysize = CheckedInt32(input_image->StrideY()) * input_image->height();
397 0 : MOZ_RELEASE_ASSERT(ysize.isValid());
398 : // I will assume that if that doesn't overflow, the others case - YUV
399 : // 4:2:0 has U/V widths <= Y, even with alignment issues.
400 0 : err = frame->CreateFrame(ysize.value(),
401 0 : input_image->DataY(),
402 0 : input_image->StrideU() * ((input_image->height()+1)/2),
403 0 : input_image->DataU(),
404 0 : input_image->StrideV() * ((input_image->height()+1)/2),
405 0 : input_image->DataV(),
406 0 : input_image->width(),
407 0 : input_image->height(),
408 0 : input_image->StrideY(),
409 0 : input_image->StrideU(),
410 0 : input_image->StrideV());
411 0 : if (err != GMPNoErr) {
412 0 : return err;
413 : }
414 0 : frame->SetTimestamp((aInputImage->timestamp() * 1000ll)/90); // note: rounds down!
415 : //frame->SetDuration(1000000ll/30); // XXX base duration on measured current FPS - or don't bother
416 :
417 : // Bug XXXXXX: Set codecSpecific info
418 : GMPCodecSpecificInfo info;
419 0 : memset(&info, 0, sizeof(info));
420 0 : info.mCodecType = kGMPVideoCodecH264;
421 0 : nsTArray<uint8_t> codecSpecificInfo;
422 0 : codecSpecificInfo.AppendElements((uint8_t*)&info, sizeof(GMPCodecSpecificInfo));
423 :
424 0 : nsTArray<GMPVideoFrameType> gmp_frame_types;
425 0 : for (auto it = aFrameTypes->begin(); it != aFrameTypes->end(); ++it) {
426 : GMPVideoFrameType ft;
427 :
428 0 : int32_t ret = WebrtcFrameTypeToGmpFrameType(*it, &ft);
429 0 : if (ret != WEBRTC_VIDEO_CODEC_OK) {
430 0 : return ret;
431 : }
432 :
433 0 : gmp_frame_types.AppendElement(ft);
434 : }
435 :
436 0 : LOGD(("GMP Encode: %llu", (aInputImage->timestamp() * 1000ll)/90));
437 0 : err = mGMP->Encode(Move(frame), codecSpecificInfo, gmp_frame_types);
438 0 : if (err != GMPNoErr) {
439 0 : return err;
440 : }
441 :
442 0 : return WEBRTC_VIDEO_CODEC_OK;
443 : }
444 :
445 : int32_t
446 0 : WebrtcGmpVideoEncoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* aCallback)
447 : {
448 0 : MutexAutoLock lock(mCallbackMutex);
449 0 : mCallback = aCallback;
450 :
451 0 : return WEBRTC_VIDEO_CODEC_OK;
452 : }
453 :
454 : /* static */ void
455 0 : WebrtcGmpVideoEncoder::ReleaseGmp_g(RefPtr<WebrtcGmpVideoEncoder>& aEncoder)
456 : {
457 0 : aEncoder->Close_g();
458 0 : }
459 :
460 : int32_t
461 0 : WebrtcGmpVideoEncoder::ReleaseGmp()
462 : {
463 0 : LOGD(("GMP Released:"));
464 0 : if (mGMPThread) {
465 0 : mGMPThread->Dispatch(
466 0 : WrapRunnableNM(&WebrtcGmpVideoEncoder::ReleaseGmp_g,
467 0 : RefPtr<WebrtcGmpVideoEncoder>(this)),
468 0 : NS_DISPATCH_NORMAL);
469 : }
470 0 : return WEBRTC_VIDEO_CODEC_OK;
471 : }
472 :
473 : int32_t
474 0 : WebrtcGmpVideoEncoder::SetChannelParameters(uint32_t aPacketLoss, int aRTT)
475 : {
476 0 : return WEBRTC_VIDEO_CODEC_OK;
477 : }
478 :
479 : int32_t
480 0 : WebrtcGmpVideoEncoder::SetRates(uint32_t aNewBitRate, uint32_t aFrameRate)
481 : {
482 0 : MOZ_ASSERT(mGMPThread);
483 0 : if (aFrameRate == 0) {
484 0 : aFrameRate = 30; // Assume 30fps if we don't know the rate
485 : }
486 0 : mGMPThread->Dispatch(WrapRunnableNM(&WebrtcGmpVideoEncoder::SetRates_g,
487 0 : RefPtr<WebrtcGmpVideoEncoder>(this),
488 : aNewBitRate,
489 : aFrameRate),
490 0 : NS_DISPATCH_NORMAL);
491 :
492 0 : return WEBRTC_VIDEO_CODEC_OK;
493 : }
494 :
495 : /* static */ int32_t
496 0 : WebrtcGmpVideoEncoder::SetRates_g(RefPtr<WebrtcGmpVideoEncoder> aThis,
497 : uint32_t aNewBitRate,
498 : uint32_t aFrameRate)
499 : {
500 0 : if (!aThis->mGMP) {
501 : // destroyed via Terminate()
502 0 : return WEBRTC_VIDEO_CODEC_ERROR;
503 : }
504 :
505 0 : GMPErr err = aThis->mGMP->SetRates(aNewBitRate, aFrameRate);
506 0 : if (err != GMPNoErr) {
507 0 : return WEBRTC_VIDEO_CODEC_ERROR;
508 : }
509 :
510 0 : return WEBRTC_VIDEO_CODEC_OK;
511 : }
512 :
513 : // GMPVideoEncoderCallback virtual functions.
514 : void
515 0 : WebrtcGmpVideoEncoder::Terminated()
516 : {
517 0 : LOGD(("GMP Encoder Terminated: %p", (void *)this));
518 :
519 0 : mGMP->Close();
520 0 : mGMP = nullptr;
521 0 : mHost = nullptr;
522 0 : mInitting = false;
523 : // Could now notify that it's dead
524 0 : }
525 :
526 : void
527 0 : WebrtcGmpVideoEncoder::Encoded(GMPVideoEncodedFrame* aEncodedFrame,
528 : const nsTArray<uint8_t>& aCodecSpecificInfo)
529 : {
530 0 : MutexAutoLock lock(mCallbackMutex);
531 0 : if (mCallback) {
532 : webrtc::FrameType ft;
533 0 : GmpFrameTypeToWebrtcFrameType(aEncodedFrame->FrameType(), &ft);
534 0 : uint32_t timestamp = (aEncodedFrame->TimeStamp() * 90ll + 999)/1000;
535 :
536 0 : LOGD(("GMP Encoded: %" PRIu64 ", type %d, len %d",
537 : aEncodedFrame->TimeStamp(),
538 : aEncodedFrame->BufferType(),
539 : aEncodedFrame->Size()));
540 :
541 : // Right now makes one Encoded() callback per unit
542 : // XXX convert to FragmentationHeader format (array of offsets and sizes plus a buffer) in
543 : // combination with H264 packetization changes in webrtc/trunk code
544 0 : uint8_t *buffer = aEncodedFrame->Buffer();
545 0 : uint8_t *end = aEncodedFrame->Buffer() + aEncodedFrame->Size();
546 : size_t size_bytes;
547 0 : switch (aEncodedFrame->BufferType()) {
548 : case GMP_BufferSingle:
549 0 : size_bytes = 0;
550 0 : break;
551 : case GMP_BufferLength8:
552 0 : size_bytes = 1;
553 0 : break;
554 : case GMP_BufferLength16:
555 0 : size_bytes = 2;
556 0 : break;
557 : case GMP_BufferLength24:
558 0 : size_bytes = 3;
559 0 : break;
560 : case GMP_BufferLength32:
561 0 : size_bytes = 4;
562 0 : break;
563 : default:
564 : // Really that it's not in the enum
565 0 : LOG(LogLevel::Error,
566 : ("GMP plugin returned incorrect type (%d)", aEncodedFrame->BufferType()));
567 : // XXX Bug 1041232 - need a better API for interfacing to the
568 : // plugin so we can kill it here
569 0 : return;
570 : }
571 :
572 : struct nal_entry {
573 : uint32_t offset;
574 : uint32_t size;
575 : };
576 0 : AutoTArray<nal_entry, 1> nals;
577 0 : uint32_t size = 0;
578 : // make sure we don't read past the end of the buffer getting the size
579 0 : while (buffer+size_bytes < end) {
580 0 : switch (aEncodedFrame->BufferType()) {
581 : case GMP_BufferSingle:
582 0 : size = aEncodedFrame->Size();
583 0 : break;
584 : case GMP_BufferLength8:
585 0 : size = *buffer++;
586 0 : break;
587 : case GMP_BufferLength16:
588 : // presumes we can do unaligned loads
589 0 : size = *(reinterpret_cast<uint16_t*>(buffer));
590 0 : buffer += 2;
591 0 : break;
592 : case GMP_BufferLength24:
593 : // 24-bits is a pain, since byte-order issues make things painful
594 : // I'm going to define 24-bit as little-endian always; big-endian must convert
595 0 : size = ((uint32_t) *buffer) |
596 0 : (((uint32_t) *(buffer+1)) << 8) |
597 0 : (((uint32_t) *(buffer+2)) << 16);
598 0 : buffer += 3;
599 0 : break;
600 : case GMP_BufferLength32:
601 : // presumes we can do unaligned loads
602 0 : size = *(reinterpret_cast<uint32_t*>(buffer));
603 0 : buffer += 4;
604 0 : break;
605 : default:
606 0 : MOZ_CRASH("GMP_BufferType already handled in switch above");
607 : }
608 0 : MOZ_ASSERT(size != 0 &&
609 : buffer+size <= end); // in non-debug code, don't crash in this case
610 0 : if (size == 0 || buffer+size > end) {
611 : // XXX see above - should we kill the plugin for returning extra bytes? Probably
612 0 : LOG(LogLevel::Error,
613 : ("GMP plugin returned badly formatted encoded data: buffer=%p, size=%d, end=%p", buffer, size, end));
614 0 : return;
615 : }
616 : // XXX optimize by making buffer an offset
617 0 : nal_entry nal = {((uint32_t) (buffer-aEncodedFrame->Buffer())), (uint32_t) size};
618 0 : nals.AppendElement(nal);
619 0 : buffer += size;
620 : // on last one, buffer == end normally
621 : }
622 0 : if (buffer != end) {
623 : // At most 3 bytes can be left over, depending on buffertype
624 0 : LOGD(("GMP plugin returned %td extra bytes", end - buffer));
625 : }
626 :
627 0 : size_t num_nals = nals.Length();
628 0 : if (num_nals > 0) {
629 0 : webrtc::RTPFragmentationHeader fragmentation;
630 0 : fragmentation.VerifyAndAllocateFragmentationHeader(num_nals);
631 0 : for (size_t i = 0; i < num_nals; i++) {
632 0 : fragmentation.fragmentationOffset[i] = nals[i].offset;
633 0 : fragmentation.fragmentationLength[i] = nals[i].size;
634 : }
635 :
636 0 : webrtc::EncodedImage unit(aEncodedFrame->Buffer(), size, size);
637 0 : unit._frameType = ft;
638 0 : unit._timeStamp = timestamp;
639 : // Ensure we ignore this when calculating RTCP timestamps
640 0 : unit.capture_time_ms_ = -1;
641 0 : unit._completeFrame = true;
642 :
643 : // TODO: Currently the OpenH264 codec does not preserve any codec
644 : // specific info passed into it and just returns default values.
645 : // If this changes in the future, it would be nice to get rid of
646 : // mCodecSpecificInfo.
647 0 : mCallback->OnEncodedImage(unit, &mCodecSpecificInfo, &fragmentation);
648 : }
649 : }
650 : }
651 :
652 : // Decoder.
653 0 : WebrtcGmpVideoDecoder::WebrtcGmpVideoDecoder() :
654 : mGMP(nullptr),
655 : mInitting(false),
656 : mHost(nullptr),
657 : mCallbackMutex("WebrtcGmpVideoDecoder decoded callback mutex"),
658 : mCallback(nullptr),
659 : mCachedPluginId(0),
660 0 : mDecoderStatus(GMPNoErr)
661 : {
662 0 : if (mPCHandle.empty()) {
663 0 : mPCHandle = WebrtcGmpPCHandleSetter::GetCurrentHandle();
664 : }
665 0 : MOZ_ASSERT(!mPCHandle.empty());
666 0 : }
667 :
668 0 : WebrtcGmpVideoDecoder::~WebrtcGmpVideoDecoder()
669 : {
670 : // We should not have been destroyed if we never closed our GMP
671 0 : MOZ_ASSERT(!mGMP);
672 0 : }
673 :
674 : int32_t
675 0 : WebrtcGmpVideoDecoder::InitDecode(const webrtc::VideoCodec* aCodecSettings,
676 : int32_t aNumberOfCores)
677 : {
678 0 : if (!mMPS) {
679 0 : mMPS = do_GetService("@mozilla.org/gecko-media-plugin-service;1");
680 : }
681 0 : MOZ_ASSERT(mMPS);
682 :
683 0 : if (!mGMPThread) {
684 0 : if (NS_WARN_IF(NS_FAILED(mMPS->GetThread(getter_AddRefs(mGMPThread))))) {
685 0 : return WEBRTC_VIDEO_CODEC_ERROR;
686 : }
687 : }
688 :
689 0 : RefPtr<GmpInitDoneRunnable> initDone(new GmpInitDoneRunnable(mPCHandle));
690 0 : mGMPThread->Dispatch(WrapRunnableNM(&WebrtcGmpVideoDecoder::InitDecode_g,
691 0 : RefPtr<WebrtcGmpVideoDecoder>(this),
692 : aCodecSettings,
693 : aNumberOfCores,
694 : initDone),
695 0 : NS_DISPATCH_NORMAL);
696 :
697 0 : return WEBRTC_VIDEO_CODEC_OK;
698 : }
699 :
700 : /* static */ void
701 0 : WebrtcGmpVideoDecoder::InitDecode_g(
702 : const RefPtr<WebrtcGmpVideoDecoder>& aThis,
703 : const webrtc::VideoCodec* aCodecSettings,
704 : int32_t aNumberOfCores,
705 : const RefPtr<GmpInitDoneRunnable>& aInitDone)
706 : {
707 0 : nsTArray<nsCString> tags;
708 0 : tags.AppendElement(NS_LITERAL_CSTRING("h264"));
709 : UniquePtr<GetGMPVideoDecoderCallback> callback(
710 0 : new InitDoneCallback(aThis, aInitDone));
711 0 : aThis->mInitting = true;
712 0 : nsresult rv = aThis->mMPS->GetGMPVideoDecoder(nullptr,
713 : &tags,
714 0 : NS_LITERAL_CSTRING(""),
715 0 : Move(callback));
716 0 : if (NS_WARN_IF(NS_FAILED(rv))) {
717 0 : LOGD(("GMP Decode: GetGMPVideoDecoder failed"));
718 0 : aThis->Close_g();
719 0 : aInitDone->Dispatch(WEBRTC_VIDEO_CODEC_ERROR,
720 0 : "GMP Decode: GetGMPVideoDecoder failed.");
721 : }
722 0 : }
723 :
724 : int32_t
725 0 : WebrtcGmpVideoDecoder::GmpInitDone(GMPVideoDecoderProxy* aGMP,
726 : GMPVideoHost* aHost,
727 : std::string* aErrorOut)
728 : {
729 0 : if (!mInitting || !aGMP || !aHost) {
730 : *aErrorOut = "GMP Decode: Either init was aborted, "
731 0 : "or init failed to supply either a GMP decoder or GMP host.";
732 0 : if (aGMP) {
733 : // This could destroy us, since aGMP may be the last thing holding a ref
734 : // Return immediately.
735 0 : aGMP->Close();
736 : }
737 0 : return WEBRTC_VIDEO_CODEC_ERROR;
738 : }
739 :
740 0 : mInitting = false;
741 :
742 0 : if (mGMP && mGMP != aGMP) {
743 0 : Close_g();
744 : }
745 :
746 0 : mGMP = aGMP;
747 0 : mHost = aHost;
748 0 : mCachedPluginId = mGMP->GetPluginId();
749 : // Bug XXXXXX: transfer settings from codecSettings to codec.
750 : GMPVideoCodec codec;
751 0 : memset(&codec, 0, sizeof(codec));
752 0 : codec.mGMPApiVersion = 33;
753 :
754 : // XXX this is currently a hack
755 : //GMPVideoCodecUnion codecSpecific;
756 : //memset(&codecSpecific, 0, sizeof(codecSpecific));
757 0 : nsTArray<uint8_t> codecSpecific;
758 0 : nsresult rv = mGMP->InitDecode(codec, codecSpecific, this, 1);
759 0 : if (NS_FAILED(rv)) {
760 0 : *aErrorOut = "GMP Decode: InitDecode failed";
761 0 : mQueuedFrames.Clear();
762 0 : return WEBRTC_VIDEO_CODEC_ERROR;
763 : }
764 :
765 : // now release any frames that got queued waiting for InitDone
766 0 : if (!mQueuedFrames.IsEmpty()) {
767 : // So we're safe to call Decode_g(), which asserts it's empty
768 0 : nsTArray<UniquePtr<GMPDecodeData>> temp;
769 0 : temp.SwapElements(mQueuedFrames);
770 0 : for (auto& queued : temp) {
771 0 : int rv = Decode_g(queued->mImage,
772 0 : queued->mMissingFrames,
773 : nullptr,
774 : nullptr,
775 0 : queued->mRenderTimeMs);
776 0 : if (rv) {
777 0 : return rv;
778 : }
779 : }
780 : }
781 :
782 0 : return WEBRTC_VIDEO_CODEC_OK;
783 : }
784 :
785 : void
786 0 : WebrtcGmpVideoDecoder::Close_g()
787 : {
788 0 : GMPVideoDecoderProxy* gmp(mGMP);
789 0 : mGMP = nullptr;
790 0 : mHost = nullptr;
791 0 : mInitting = false;
792 :
793 0 : if (gmp) {
794 : // Do this last, since this could cause us to be destroyed
795 0 : gmp->Close();
796 : }
797 0 : }
798 :
799 : int32_t
800 0 : WebrtcGmpVideoDecoder::Decode(const webrtc::EncodedImage& aInputImage,
801 : bool aMissingFrames,
802 : const webrtc::RTPFragmentationHeader* aFragmentation,
803 : const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
804 : int64_t aRenderTimeMs)
805 : {
806 : int32_t ret;
807 0 : MOZ_ASSERT(mGMPThread);
808 0 : MOZ_ASSERT(!NS_IsMainThread());
809 : // Would be really nice to avoid this sync dispatch, but it would require a
810 : // copy of the frame, since it doesn't appear to actually have a refcount.
811 : // Passing 'this' is safe since this is synchronous.
812 0 : mozilla::SyncRunnable::DispatchToThread(mGMPThread,
813 0 : WrapRunnableRet(&ret, this,
814 : &WebrtcGmpVideoDecoder::Decode_g,
815 : aInputImage,
816 : aMissingFrames,
817 : aFragmentation,
818 : aCodecSpecificInfo,
819 0 : aRenderTimeMs));
820 :
821 0 : return ret;
822 : }
823 :
824 : int32_t
825 0 : WebrtcGmpVideoDecoder::Decode_g(const webrtc::EncodedImage& aInputImage,
826 : bool aMissingFrames,
827 : const webrtc::RTPFragmentationHeader* aFragmentation,
828 : const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
829 : int64_t aRenderTimeMs)
830 : {
831 0 : if (!mGMP) {
832 0 : if (mInitting) {
833 : // InitDone hasn't been called yet (race)
834 : GMPDecodeData *data = new GMPDecodeData(aInputImage,
835 : aMissingFrames,
836 0 : aRenderTimeMs);
837 0 : mQueuedFrames.AppendElement(data);
838 0 : return WEBRTC_VIDEO_CODEC_OK;
839 : }
840 : // destroyed via Terminate(), failed to init, or just not initted yet
841 0 : LOGD(("GMP Decode: not initted yet"));
842 0 : return WEBRTC_VIDEO_CODEC_ERROR;
843 : }
844 0 : MOZ_ASSERT(mQueuedFrames.IsEmpty());
845 0 : MOZ_ASSERT(mHost);
846 :
847 0 : if (!aInputImage._length) {
848 0 : return WEBRTC_VIDEO_CODEC_ERROR;
849 : }
850 :
851 0 : GMPVideoFrame* ftmp = nullptr;
852 0 : GMPErr err = mHost->CreateFrame(kGMPEncodedVideoFrame, &ftmp);
853 0 : if (err != GMPNoErr) {
854 0 : return WEBRTC_VIDEO_CODEC_ERROR;
855 : }
856 :
857 0 : GMPUniquePtr<GMPVideoEncodedFrame> frame(static_cast<GMPVideoEncodedFrame*>(ftmp));
858 0 : err = frame->CreateEmptyFrame(aInputImage._length);
859 0 : if (err != GMPNoErr) {
860 0 : return WEBRTC_VIDEO_CODEC_ERROR;
861 : }
862 :
863 : // XXX At this point, we only will get mode1 data (a single length and a buffer)
864 : // Session_info.cc/etc code needs to change to support mode 0.
865 0 : *(reinterpret_cast<uint32_t*>(frame->Buffer())) = frame->Size();
866 :
867 : // XXX It'd be wonderful not to have to memcpy the encoded data!
868 0 : memcpy(frame->Buffer()+4, aInputImage._buffer+4, frame->Size()-4);
869 :
870 0 : frame->SetEncodedWidth(aInputImage._encodedWidth);
871 0 : frame->SetEncodedHeight(aInputImage._encodedHeight);
872 0 : frame->SetTimeStamp((aInputImage._timeStamp * 1000ll)/90); // rounds down
873 0 : frame->SetCompleteFrame(aInputImage._completeFrame);
874 0 : frame->SetBufferType(GMP_BufferLength32);
875 :
876 : GMPVideoFrameType ft;
877 0 : int32_t ret = WebrtcFrameTypeToGmpFrameType(aInputImage._frameType, &ft);
878 0 : if (ret != WEBRTC_VIDEO_CODEC_OK) {
879 0 : return ret;
880 : }
881 :
882 : // Bug XXXXXX: Set codecSpecific info
883 : GMPCodecSpecificInfo info;
884 0 : memset(&info, 0, sizeof(info));
885 0 : info.mCodecType = kGMPVideoCodecH264;
886 0 : info.mCodecSpecific.mH264.mSimulcastIdx = 0;
887 0 : nsTArray<uint8_t> codecSpecificInfo;
888 0 : codecSpecificInfo.AppendElements((uint8_t*)&info, sizeof(GMPCodecSpecificInfo));
889 :
890 0 : LOGD(("GMP Decode: %" PRIu64 ", len %" PRIuSIZE "%s", frame->TimeStamp(), aInputImage._length,
891 : ft == kGMPKeyFrame ? ", KeyFrame" : ""));
892 0 : nsresult rv = mGMP->Decode(Move(frame),
893 : aMissingFrames,
894 : codecSpecificInfo,
895 0 : aRenderTimeMs);
896 0 : if (NS_FAILED(rv)) {
897 0 : return WEBRTC_VIDEO_CODEC_ERROR;
898 : }
899 0 : if(mDecoderStatus != GMPNoErr){
900 0 : mDecoderStatus = GMPNoErr;
901 0 : return WEBRTC_VIDEO_CODEC_ERROR;
902 : }
903 0 : return WEBRTC_VIDEO_CODEC_OK;
904 : }
905 :
906 : int32_t
907 0 : WebrtcGmpVideoDecoder::RegisterDecodeCompleteCallback( webrtc::DecodedImageCallback* aCallback)
908 : {
909 0 : MutexAutoLock lock(mCallbackMutex);
910 0 : mCallback = aCallback;
911 :
912 0 : return WEBRTC_VIDEO_CODEC_OK;
913 : }
914 :
915 :
916 : /* static */ void
917 0 : WebrtcGmpVideoDecoder::ReleaseGmp_g(RefPtr<WebrtcGmpVideoDecoder>& aDecoder)
918 : {
919 0 : aDecoder->Close_g();
920 0 : }
921 :
922 : int32_t
923 0 : WebrtcGmpVideoDecoder::ReleaseGmp()
924 : {
925 0 : LOGD(("GMP Released:"));
926 0 : if (mGMPThread) {
927 0 : mGMPThread->Dispatch(
928 0 : WrapRunnableNM(&WebrtcGmpVideoDecoder::ReleaseGmp_g,
929 0 : RefPtr<WebrtcGmpVideoDecoder>(this)),
930 0 : NS_DISPATCH_NORMAL);
931 : }
932 0 : return WEBRTC_VIDEO_CODEC_OK;
933 : }
934 :
935 : void
936 0 : WebrtcGmpVideoDecoder::Terminated()
937 : {
938 0 : LOGD(("GMP Decoder Terminated: %p", (void *)this));
939 :
940 0 : mGMP->Close();
941 0 : mGMP = nullptr;
942 0 : mHost = nullptr;
943 0 : mInitting = false;
944 : // Could now notify that it's dead
945 0 : }
946 :
947 0 : static void DeleteBuffer(uint8_t* data)
948 : {
949 0 : delete[] data;
950 0 : }
951 :
952 : void
953 0 : WebrtcGmpVideoDecoder::Decoded(GMPVideoi420Frame* aDecodedFrame)
954 : {
955 : // we have two choices here: wrap the frame with a callback that frees
956 : // the data later (risking running out of shmems), or copy the data out
957 : // always. Also, we can only Destroy() the frame on the gmp thread, so
958 : // copying is simplest if expensive.
959 : // I420 size including rounding...
960 0 : CheckedInt32 length = (CheckedInt32(aDecodedFrame->Stride(kGMPYPlane)) * aDecodedFrame->Height()) +
961 0 : (aDecodedFrame->Stride(kGMPVPlane) +
962 0 : aDecodedFrame->Stride(kGMPUPlane)) * ((aDecodedFrame->Height()+1)/2);
963 0 : int32_t size = length.value();
964 0 : MOZ_RELEASE_ASSERT(length.isValid() && size > 0);
965 0 : auto buffer = MakeUniqueFallible<uint8_t[]>(size);
966 0 : if (buffer) {
967 : // This is 3 separate buffers currently anyways, no use in trying to
968 : // see if we can use a single memcpy.
969 0 : uint8_t* buffer_y = buffer.get();
970 0 : memcpy(buffer_y, aDecodedFrame->Buffer(kGMPYPlane),
971 0 : aDecodedFrame->Stride(kGMPYPlane) * aDecodedFrame->Height());
972 : // Should this be aligned, making it non-contiguous? Assume no, this is already
973 : // factored into the strides.
974 0 : uint8_t* buffer_u = buffer_y +
975 0 : aDecodedFrame->Stride(kGMPYPlane) * aDecodedFrame->Height();
976 0 : memcpy(buffer_u, aDecodedFrame->Buffer(kGMPUPlane),
977 0 : aDecodedFrame->Stride(kGMPUPlane) * ((aDecodedFrame->Height()+1)/2));
978 0 : uint8_t* buffer_v = buffer_u +
979 0 : aDecodedFrame->Stride(kGMPUPlane) * ((aDecodedFrame->Height()+1)/2);
980 0 : memcpy(buffer_v, aDecodedFrame->Buffer(kGMPVPlane),
981 0 : aDecodedFrame->Stride(kGMPVPlane) * ((aDecodedFrame->Height()+1)/2));
982 :
983 0 : MutexAutoLock lock(mCallbackMutex);
984 0 : if (mCallback) {
985 : rtc::scoped_refptr<webrtc::WrappedI420Buffer> video_frame_buffer(
986 : new rtc::RefCountedObject<webrtc::WrappedI420Buffer>(
987 0 : aDecodedFrame->Width(), aDecodedFrame->Height(),
988 0 : buffer_y, aDecodedFrame->Stride(kGMPYPlane),
989 0 : buffer_u, aDecodedFrame->Stride(kGMPUPlane),
990 0 : buffer_v, aDecodedFrame->Stride(kGMPVPlane),
991 0 : rtc::Bind(&DeleteBuffer, buffer.release())));
992 :
993 : webrtc::VideoFrame image(video_frame_buffer, 0, 0,
994 0 : webrtc::kVideoRotation_0);
995 0 : image.set_timestamp((aDecodedFrame->Timestamp() * 90ll + 999)/1000); // round up
996 0 : image.set_render_time_ms(0);
997 :
998 0 : LOGD(("GMP Decoded: %" PRIu64, aDecodedFrame->Timestamp()));
999 0 : mCallback->Decoded(image);
1000 : }
1001 : }
1002 0 : aDecodedFrame->Destroy();
1003 0 : }
1004 :
1005 : }
|
