Line data Source code
1 : /*
2 : * Copyright (c) 2012, The WebRTC project authors. All rights reserved.
3 : *
4 : * Redistribution and use in source and binary forms, with or without
5 : * modification, are permitted provided that the following conditions are
6 : * met:
7 : *
8 : * * Redistributions of source code must retain the above copyright
9 : * notice, this list of conditions and the following disclaimer.
10 : *
11 : * * Redistributions in binary form must reproduce the above copyright
12 : * notice, this list of conditions and the following disclaimer in
13 : * the documentation and/or other materials provided with the
14 : * distribution.
15 : *
16 : * * Neither the name of Google nor the names of its contributors may
17 : * be used to endorse or promote products derived from this software
18 : * without specific prior written permission.
19 : *
20 : * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 : * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 : * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 : * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 : * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 : * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 : * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 : * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 : * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 : * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 : * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 : */
32 :
33 : #ifndef WEBRTCGMPVIDEOCODEC_H_
34 : #define WEBRTCGMPVIDEOCODEC_H_
35 :
36 : #include <iostream>
37 : #include <queue>
38 : #include <string>
39 :
40 : #include "nsThreadUtils.h"
41 : #include "mozilla/Monitor.h"
42 : #include "mozilla/Mutex.h"
43 :
44 : #include "mozIGeckoMediaPluginService.h"
45 : #include "MediaConduitInterface.h"
46 : #include "AudioConduit.h"
47 : #include "VideoConduit.h"
48 : #include "webrtc/modules/video_coding/include/video_codec_interface.h"
49 :
50 : #include "gmp-video-host.h"
51 : #include "GMPVideoDecoderProxy.h"
52 : #include "GMPVideoEncoderProxy.h"
53 :
54 : #include "PeerConnectionImpl.h"
55 :
56 : namespace mozilla {
57 :
58 : // Class that allows code on the other side of webrtc.org to tell
59 : // WebrtcGmpVideoEncoder/Decoder what PC they should send errors to.
60 : // This is necessary because webrtc.org gives us no way to plumb the handle
61 : // through, nor does it give us any way to inform it of an error that will
62 : // make it back to the PC that cares (except for errors encountered
63 : // synchronously in functions like InitEncode/Decode, which will not happen
64 : // because GMP init is async).
65 : // Right now, this is used in MediaPipelineFactory.
66 : class WebrtcGmpPCHandleSetter
67 : {
68 : public:
69 : explicit WebrtcGmpPCHandleSetter(const std::string& aPCHandle);
70 :
71 : ~WebrtcGmpPCHandleSetter();
72 :
73 : static std::string GetCurrentHandle();
74 :
75 : private:
76 : static std::string sCurrentHandle;
77 : };
78 :
79 0 : class GmpInitDoneRunnable : public Runnable
80 : {
81 : public:
82 0 : explicit GmpInitDoneRunnable(const std::string& aPCHandle) :
83 : Runnable("GmpInitDoneRunnable"),
84 : mResult(WEBRTC_VIDEO_CODEC_OK),
85 0 : mPCHandle(aPCHandle)
86 : {
87 0 : }
88 :
89 0 : NS_IMETHOD Run() override
90 : {
91 0 : if (mResult == WEBRTC_VIDEO_CODEC_OK) {
92 : // Might be useful to notify the PeerConnection about successful init
93 : // someday.
94 0 : return NS_OK;
95 : }
96 :
97 0 : PeerConnectionWrapper wrapper(mPCHandle);
98 0 : if (wrapper.impl()) {
99 0 : wrapper.impl()->OnMediaError(mError);
100 : }
101 0 : return NS_OK;
102 : }
103 :
104 0 : void Dispatch(int32_t aResult, const std::string& aError = "")
105 : {
106 0 : mResult = aResult;
107 0 : mError = aError;
108 0 : nsCOMPtr<nsIThread> mainThread(do_GetMainThread());
109 0 : if (mainThread) {
110 : // For some reason, the compiler on CI is treating |this| as a const
111 : // pointer, despite the fact that we're in a non-const function. And,
112 : // interestingly enough, correcting this doesn't require a const_cast.
113 0 : mainThread->Dispatch(do_AddRef(static_cast<nsIRunnable*>(this)),
114 0 : NS_DISPATCH_NORMAL);
115 : }
116 0 : }
117 :
118 : int32_t Result()
119 : {
120 : return mResult;
121 : }
122 :
123 : private:
124 : int32_t mResult;
125 : std::string mPCHandle;
126 : std::string mError;
127 : };
128 :
129 : // Hold a frame for later decode
130 : class GMPDecodeData
131 : {
132 : public:
133 0 : GMPDecodeData(const webrtc::EncodedImage& aInputImage,
134 : bool aMissingFrames,
135 : int64_t aRenderTimeMs)
136 0 : : mImage(aInputImage)
137 : , mMissingFrames(aMissingFrames)
138 0 : , mRenderTimeMs(aRenderTimeMs)
139 : {
140 : // We want to use this for queuing, and the calling code recycles the
141 : // buffer on return from Decode()
142 0 : mImage._length = aInputImage._length;
143 0 : mImage._size = aInputImage._length +
144 0 : webrtc::EncodedImage::GetBufferPaddingBytes(webrtc::kVideoCodecH264);
145 0 : mImage._buffer = new uint8_t[mImage._size];
146 0 : memcpy(mImage._buffer, aInputImage._buffer, aInputImage._length);
147 0 : }
148 :
149 0 : ~GMPDecodeData() {
150 0 : delete [] mImage._buffer;
151 0 : }
152 :
153 : webrtc::EncodedImage mImage;
154 : bool mMissingFrames;
155 : int64_t mRenderTimeMs;
156 : };
157 :
158 : class WebrtcGmpVideoEncoder : public GMPVideoEncoderCallbackProxy
159 : {
160 : public:
161 : WebrtcGmpVideoEncoder();
162 0 : NS_INLINE_DECL_THREADSAFE_REFCOUNTING(WebrtcGmpVideoEncoder);
163 :
164 : // Implement VideoEncoder interface, sort of.
165 : // (We cannot use |Release|, since that's needed for nsRefPtr)
166 0 : virtual uint64_t PluginID() const
167 : {
168 0 : return mCachedPluginId;
169 : }
170 :
171 : virtual int32_t InitEncode(const webrtc::VideoCodec* aCodecSettings,
172 : int32_t aNumberOfCores,
173 : uint32_t aMaxPayloadSize);
174 :
175 : virtual int32_t Encode(const webrtc::VideoFrame& aInputImage,
176 : const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
177 : const std::vector<webrtc::FrameType>* aFrameTypes);
178 :
179 : virtual int32_t RegisterEncodeCompleteCallback(
180 : webrtc::EncodedImageCallback* aCallback);
181 :
182 : virtual int32_t ReleaseGmp();
183 :
184 : virtual int32_t SetChannelParameters(uint32_t aPacketLoss,
185 : int aRTT);
186 :
187 : virtual int32_t SetRates(uint32_t aNewBitRate,
188 : uint32_t aFrameRate);
189 :
190 : // GMPVideoEncoderCallback virtual functions.
191 : virtual void Terminated() override;
192 :
193 : virtual void Encoded(GMPVideoEncodedFrame* aEncodedFrame,
194 : const nsTArray<uint8_t>& aCodecSpecificInfo) override;
195 :
196 0 : virtual void Error(GMPErr aError) override {
197 0 : }
198 :
199 : private:
200 : virtual ~WebrtcGmpVideoEncoder();
201 :
202 : static void InitEncode_g(const RefPtr<WebrtcGmpVideoEncoder>& aThis,
203 : const GMPVideoCodec& aCodecParams,
204 : int32_t aNumberOfCores,
205 : uint32_t aMaxPayloadSize,
206 : const RefPtr<GmpInitDoneRunnable>& aInitDone);
207 : int32_t GmpInitDone(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost,
208 : const GMPVideoCodec& aCodecParams,
209 : uint32_t aMaxPayloadSize,
210 : std::string* aErrorOut);
211 : int32_t GmpInitDone(GMPVideoEncoderProxy* aGMP,
212 : GMPVideoHost* aHost,
213 : std::string* aErrorOut);
214 : int32_t InitEncoderForSize(unsigned short aWidth,
215 : unsigned short aHeight,
216 : std::string* aErrorOut);
217 : static void ReleaseGmp_g(RefPtr<WebrtcGmpVideoEncoder>& aEncoder);
218 : void Close_g();
219 :
220 0 : class InitDoneCallback : public GetGMPVideoEncoderCallback
221 : {
222 : public:
223 0 : InitDoneCallback(const RefPtr<WebrtcGmpVideoEncoder>& aEncoder,
224 : const RefPtr<GmpInitDoneRunnable>& aInitDone,
225 : const GMPVideoCodec& aCodecParams,
226 : uint32_t aMaxPayloadSize)
227 0 : : mEncoder(aEncoder),
228 : mInitDone(aInitDone),
229 : mCodecParams(aCodecParams),
230 0 : mMaxPayloadSize(aMaxPayloadSize)
231 : {
232 0 : }
233 :
234 0 : virtual void Done(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost) override
235 : {
236 0 : std::string errorOut;
237 0 : int32_t result = mEncoder->GmpInitDone(aGMP,
238 : aHost,
239 : mCodecParams,
240 : mMaxPayloadSize,
241 0 : &errorOut);
242 :
243 0 : mInitDone->Dispatch(result, errorOut);
244 0 : }
245 :
246 : private:
247 : RefPtr<WebrtcGmpVideoEncoder> mEncoder;
248 : RefPtr<GmpInitDoneRunnable> mInitDone;
249 : GMPVideoCodec mCodecParams;
250 : uint32_t mMaxPayloadSize;
251 : };
252 :
253 : int32_t Encode_g(const webrtc::VideoFrame* aInputImage,
254 : const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
255 : const std::vector<webrtc::FrameType>* aFrameTypes);
256 : void RegetEncoderForResolutionChange(
257 : uint32_t aWidth,
258 : uint32_t aHeight,
259 : const RefPtr<GmpInitDoneRunnable>& aInitDone);
260 :
261 0 : class InitDoneForResolutionChangeCallback : public GetGMPVideoEncoderCallback
262 : {
263 : public:
264 0 : InitDoneForResolutionChangeCallback(
265 : const RefPtr<WebrtcGmpVideoEncoder>& aEncoder,
266 : const RefPtr<GmpInitDoneRunnable>& aInitDone,
267 : uint32_t aWidth,
268 : uint32_t aHeight)
269 0 : : mEncoder(aEncoder),
270 : mInitDone(aInitDone),
271 : mWidth(aWidth),
272 0 : mHeight(aHeight)
273 : {
274 0 : }
275 :
276 0 : virtual void Done(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost) override
277 : {
278 0 : std::string errorOut;
279 0 : int32_t result = mEncoder->GmpInitDone(aGMP, aHost, &errorOut);
280 0 : if (result != WEBRTC_VIDEO_CODEC_OK) {
281 0 : mInitDone->Dispatch(result, errorOut);
282 0 : return;
283 : }
284 :
285 0 : result = mEncoder->InitEncoderForSize(mWidth, mHeight, &errorOut);
286 0 : mInitDone->Dispatch(result, errorOut);
287 : }
288 :
289 : private:
290 : RefPtr<WebrtcGmpVideoEncoder> mEncoder;
291 : RefPtr<GmpInitDoneRunnable> mInitDone;
292 : uint32_t mWidth;
293 : uint32_t mHeight;
294 : };
295 :
296 : static int32_t SetRates_g(RefPtr<WebrtcGmpVideoEncoder> aThis,
297 : uint32_t aNewBitRate,
298 : uint32_t aFrameRate);
299 :
300 : nsCOMPtr<mozIGeckoMediaPluginService> mMPS;
301 : nsCOMPtr<nsIThread> mGMPThread;
302 : GMPVideoEncoderProxy* mGMP;
303 : // Used to handle a race where Release() is called while init is in progress
304 : bool mInitting;
305 : GMPVideoHost* mHost;
306 : GMPVideoCodec mCodecParams;
307 : uint32_t mMaxPayloadSize;
308 : webrtc::CodecSpecificInfo mCodecSpecificInfo;
309 : // Protects mCallback
310 : Mutex mCallbackMutex;
311 : webrtc::EncodedImageCallback* mCallback;
312 : uint64_t mCachedPluginId;
313 : std::string mPCHandle;
314 : };
315 :
316 :
317 : // Basically a strong ref to a WebrtcGmpVideoEncoder, that also translates
318 : // from Release() to WebrtcGmpVideoEncoder::ReleaseGmp(), since we need
319 : // WebrtcGmpVideoEncoder::Release() for managing the refcount.
320 : // The webrtc.org code gets one of these, so it doesn't unilaterally delete
321 : // the "real" encoder.
322 : class WebrtcVideoEncoderProxy : public WebrtcVideoEncoder
323 : {
324 : public:
325 0 : WebrtcVideoEncoderProxy() :
326 0 : mEncoderImpl(new WebrtcGmpVideoEncoder)
327 0 : {}
328 :
329 0 : virtual ~WebrtcVideoEncoderProxy()
330 0 : {
331 0 : RegisterEncodeCompleteCallback(nullptr);
332 0 : }
333 :
334 0 : uint64_t PluginID() const override
335 : {
336 0 : return mEncoderImpl->PluginID();
337 : }
338 :
339 0 : int32_t InitEncode(const webrtc::VideoCodec* aCodecSettings,
340 : int32_t aNumberOfCores,
341 : size_t aMaxPayloadSize) override
342 : {
343 0 : return mEncoderImpl->InitEncode(aCodecSettings,
344 : aNumberOfCores,
345 0 : aMaxPayloadSize);
346 : }
347 :
348 0 : int32_t Encode(
349 : const webrtc::VideoFrame& aInputImage,
350 : const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
351 : const std::vector<webrtc::FrameType>* aFrameTypes) override
352 : {
353 0 : return mEncoderImpl->Encode(aInputImage,
354 : aCodecSpecificInfo,
355 0 : aFrameTypes);
356 : }
357 :
358 0 : int32_t RegisterEncodeCompleteCallback(
359 : webrtc::EncodedImageCallback* aCallback) override
360 : {
361 0 : return mEncoderImpl->RegisterEncodeCompleteCallback(aCallback);
362 : }
363 :
364 0 : int32_t Release() override
365 : {
366 0 : return mEncoderImpl->ReleaseGmp();
367 : }
368 :
369 0 : int32_t SetChannelParameters(uint32_t aPacketLoss,
370 : int64_t aRTT) override
371 : {
372 0 : return mEncoderImpl->SetChannelParameters(aPacketLoss, aRTT);
373 : }
374 :
375 0 : int32_t SetRates(uint32_t aNewBitRate,
376 : uint32_t aFrameRate) override
377 : {
378 0 : return mEncoderImpl->SetRates(aNewBitRate, aFrameRate);
379 : }
380 :
381 : private:
382 : RefPtr<WebrtcGmpVideoEncoder> mEncoderImpl;
383 : };
384 :
385 : class WebrtcGmpVideoDecoder : public GMPVideoDecoderCallbackProxy
386 : {
387 : public:
388 : WebrtcGmpVideoDecoder();
389 0 : NS_INLINE_DECL_THREADSAFE_REFCOUNTING(WebrtcGmpVideoDecoder);
390 :
391 : // Implement VideoEncoder interface, sort of.
392 : // (We cannot use |Release|, since that's needed for nsRefPtr)
393 0 : virtual uint64_t PluginID() const
394 : {
395 0 : return mCachedPluginId;
396 : }
397 :
398 : virtual int32_t InitDecode(const webrtc::VideoCodec* aCodecSettings,
399 : int32_t aNumberOfCores);
400 : virtual int32_t Decode(const webrtc::EncodedImage& aInputImage,
401 : bool aMissingFrames,
402 : const webrtc::RTPFragmentationHeader* aFragmentation,
403 : const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
404 : int64_t aRenderTimeMs);
405 : virtual int32_t RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback* aCallback);
406 :
407 : virtual int32_t ReleaseGmp();
408 :
409 : // GMPVideoDecoderCallbackProxy
410 : virtual void Terminated() override;
411 :
412 : virtual void Decoded(GMPVideoi420Frame* aDecodedFrame) override;
413 :
414 0 : virtual void ReceivedDecodedReferenceFrame(const uint64_t aPictureId) override {
415 0 : MOZ_CRASH();
416 : }
417 :
418 0 : virtual void ReceivedDecodedFrame(const uint64_t aPictureId) override {
419 0 : MOZ_CRASH();
420 : }
421 :
422 0 : virtual void InputDataExhausted() override {
423 0 : }
424 :
425 0 : virtual void DrainComplete() override {
426 0 : }
427 :
428 0 : virtual void ResetComplete() override {
429 0 : }
430 :
431 0 : virtual void Error(GMPErr aError) override {
432 0 : mDecoderStatus = aError;
433 0 : }
434 :
435 : private:
436 : virtual ~WebrtcGmpVideoDecoder();
437 :
438 : static void InitDecode_g(
439 : const RefPtr<WebrtcGmpVideoDecoder>& aThis,
440 : const webrtc::VideoCodec* aCodecSettings,
441 : int32_t aNumberOfCores,
442 : const RefPtr<GmpInitDoneRunnable>& aInitDone);
443 : int32_t GmpInitDone(GMPVideoDecoderProxy* aGMP,
444 : GMPVideoHost* aHost,
445 : std::string* aErrorOut);
446 : static void ReleaseGmp_g(RefPtr<WebrtcGmpVideoDecoder>& aDecoder);
447 : void Close_g();
448 :
449 0 : class InitDoneCallback : public GetGMPVideoDecoderCallback
450 : {
451 : public:
452 0 : explicit InitDoneCallback(WebrtcGmpVideoDecoder* aDecoder,
453 : const RefPtr<GmpInitDoneRunnable>& aInitDone)
454 0 : : mDecoder(aDecoder),
455 0 : mInitDone(aInitDone)
456 : {
457 0 : }
458 :
459 0 : virtual void Done(GMPVideoDecoderProxy* aGMP, GMPVideoHost* aHost)
460 : {
461 0 : std::string errorOut;
462 0 : int32_t result = mDecoder->GmpInitDone(aGMP, aHost, &errorOut);
463 :
464 0 : mInitDone->Dispatch(result, errorOut);
465 0 : }
466 :
467 : private:
468 : WebrtcGmpVideoDecoder* mDecoder;
469 : RefPtr<GmpInitDoneRunnable> mInitDone;
470 : };
471 :
472 : virtual int32_t Decode_g(const webrtc::EncodedImage& aInputImage,
473 : bool aMissingFrames,
474 : const webrtc::RTPFragmentationHeader* aFragmentation,
475 : const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
476 : int64_t aRenderTimeMs);
477 :
478 : nsCOMPtr<mozIGeckoMediaPluginService> mMPS;
479 : nsCOMPtr<nsIThread> mGMPThread;
480 : GMPVideoDecoderProxy* mGMP; // Addref is held for us
481 : // Used to handle a race where Release() is called while init is in progress
482 : bool mInitting;
483 : // Frames queued for decode while mInitting is true
484 : nsTArray<UniquePtr<GMPDecodeData>> mQueuedFrames;
485 : GMPVideoHost* mHost;
486 : // Protects mCallback
487 : Mutex mCallbackMutex;
488 : webrtc::DecodedImageCallback* mCallback;
489 : Atomic<uint64_t> mCachedPluginId;
490 : GMPErr mDecoderStatus;
491 : std::string mPCHandle;
492 : };
493 :
494 : // Basically a strong ref to a WebrtcGmpVideoDecoder, that also translates
495 : // from Release() to WebrtcGmpVideoDecoder::ReleaseGmp(), since we need
496 : // WebrtcGmpVideoDecoder::Release() for managing the refcount.
497 : // The webrtc.org code gets one of these, so it doesn't unilaterally delete
498 : // the "real" encoder.
499 : class WebrtcVideoDecoderProxy : public WebrtcVideoDecoder
500 : {
501 : public:
502 0 : WebrtcVideoDecoderProxy() :
503 0 : mDecoderImpl(new WebrtcGmpVideoDecoder)
504 0 : {}
505 :
506 0 : virtual ~WebrtcVideoDecoderProxy()
507 0 : {
508 0 : RegisterDecodeCompleteCallback(nullptr);
509 0 : }
510 :
511 0 : uint64_t PluginID() const override
512 : {
513 0 : return mDecoderImpl->PluginID();
514 : }
515 :
516 0 : int32_t InitDecode(const webrtc::VideoCodec* aCodecSettings,
517 : int32_t aNumberOfCores) override
518 : {
519 0 : return mDecoderImpl->InitDecode(aCodecSettings, aNumberOfCores);
520 : }
521 :
522 0 : int32_t Decode(
523 : const webrtc::EncodedImage& aInputImage,
524 : bool aMissingFrames,
525 : const webrtc::RTPFragmentationHeader* aFragmentation,
526 : const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
527 : int64_t aRenderTimeMs) override
528 : {
529 0 : return mDecoderImpl->Decode(aInputImage,
530 : aMissingFrames,
531 : aFragmentation,
532 : aCodecSpecificInfo,
533 0 : aRenderTimeMs);
534 : }
535 :
536 0 : int32_t RegisterDecodeCompleteCallback(
537 : webrtc::DecodedImageCallback* aCallback) override
538 : {
539 0 : return mDecoderImpl->RegisterDecodeCompleteCallback(aCallback);
540 : }
541 :
542 0 : int32_t Release() override
543 : {
544 0 : return mDecoderImpl->ReleaseGmp();
545 : }
546 :
547 : private:
548 : RefPtr<WebrtcGmpVideoDecoder> mDecoderImpl;
549 : };
550 :
551 : }
552 :
553 : #endif
|
