Line data Source code
1 : /* This Source Code Form is subject to the terms of the Mozilla Public
2 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
4 :
5 : #ifndef MEDIA_CONDUIT_ABSTRACTION_
6 : #define MEDIA_CONDUIT_ABSTRACTION_
7 :
8 : #include "nsISupportsImpl.h"
9 : #include "nsXPCOM.h"
10 : #include "nsDOMNavigationTiming.h"
11 : #include "mozilla/RefPtr.h"
12 : #include "mozilla/RefCounted.h"
13 : #include "mozilla/UniquePtr.h"
14 : #include "mozilla/utils.h"
15 : #include "CodecConfig.h"
16 : #include "VideoTypes.h"
17 : #include "MediaConduitErrors.h"
18 :
19 : #include "ImageContainer.h"
20 :
21 : #include "webrtc/call.h"
22 : #include "webrtc/config.h"
23 : #include "webrtc/common_types.h"
24 : #include "webrtc/common_types.h"
25 : #include "webrtc/api/video/video_frame_buffer.h"
26 : #include "webrtc/logging/rtc_event_log/rtc_event_log.h"
27 :
28 : #include <vector>
29 :
30 : namespace webrtc {
31 : class VideoFrame;
32 : }
33 :
34 : namespace mozilla {
35 :
36 : // Wrap the webrtc.org Call class adding mozilla add/ref support.
37 : class WebRtcCallWrapper : public RefCounted<WebRtcCallWrapper>
38 : {
39 : public:
40 : typedef webrtc::Call::Config Config;
41 :
42 0 : static RefPtr<WebRtcCallWrapper> Create()
43 : {
44 0 : return new WebRtcCallWrapper();
45 : }
46 :
47 0 : webrtc::Call* Call() const
48 : {
49 0 : return mCall.get();
50 : }
51 :
52 0 : virtual ~WebRtcCallWrapper()
53 0 : {
54 0 : if (mCall->voice_engine()) {
55 0 : webrtc::VoiceEngine* voice_engine = mCall->voice_engine();
56 0 : mCall.reset(nullptr); // Force it to release the voice engine reference
57 : // Delete() must be after all refs are released
58 0 : webrtc::VoiceEngine::Delete(voice_engine);
59 : } else {
60 : // Must ensure it's destroyed *before* the EventLog!
61 0 : mCall.reset(nullptr);
62 : }
63 0 : }
64 :
65 0 : MOZ_DECLARE_REFCOUNTED_TYPENAME(WebRtcCallWrapper)
66 :
67 : private:
68 0 : WebRtcCallWrapper()
69 0 : {
70 0 : webrtc::Call::Config config(&mEventLog);
71 0 : mCall.reset(webrtc::Call::Create(config));
72 0 : }
73 : DISALLOW_COPY_AND_ASSIGN(WebRtcCallWrapper);
74 : UniquePtr<webrtc::Call> mCall;
75 : webrtc::RtcEventLogNullImpl mEventLog;
76 : };
77 :
78 :
79 : /**
80 : * Abstract Interface for transporting RTP packets - audio/vidoeo
81 : * The consumers of this interface are responsible for passing in
82 : * the RTPfied media packets
83 : */
84 0 : class TransportInterface
85 : {
86 : protected:
87 0 : virtual ~TransportInterface() {}
88 :
89 : public:
90 : /**
91 : * RTP Transport Function to be implemented by concrete transport implementation
92 : * @param data : RTP Packet (audio/video) to be transported
93 : * @param len : Length of the media packet
94 : * @result : NS_OK on success, NS_ERROR_FAILURE otherwise
95 : */
96 : virtual nsresult SendRtpPacket(const uint8_t* data, size_t len) = 0;
97 :
98 : /**
99 : * RTCP Transport Function to be implemented by concrete transport implementation
100 : * @param data : RTCP Packet to be transported
101 : * @param len : Length of the RTCP packet
102 : * @result : NS_OK on success, NS_ERROR_FAILURE otherwise
103 : */
104 : virtual nsresult SendRtcpPacket(const uint8_t* data, size_t len) = 0;
105 0 : NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TransportInterface)
106 : };
107 :
108 : /**
109 : * This class wraps image object for VideoRenderer::RenderVideoFrame()
110 : * callback implementation to use for rendering.
111 : */
112 0 : class ImageHandle
113 : {
114 : public:
115 0 : explicit ImageHandle(layers::Image* image) : mImage(image) {}
116 :
117 0 : const RefPtr<layers::Image>& GetImage() const { return mImage; }
118 :
119 : private:
120 : RefPtr<layers::Image> mImage;
121 : };
122 :
123 : /**
124 : * 1. Abstract renderer for video data
125 : * 2. This class acts as abstract interface between the video-engine and
126 : * video-engine agnostic renderer implementation.
127 : * 3. Concrete implementation of this interface is responsible for
128 : * processing and/or rendering the obtained raw video frame to appropriate
129 : * output , say, <video>
130 : */
131 0 : class VideoRenderer
132 : {
133 : protected:
134 0 : virtual ~VideoRenderer() {}
135 :
136 : public:
137 : /**
138 : * Callback Function reportng any change in the video-frame dimensions
139 : * @param width: current width of the video @ decoder
140 : * @param height: current height of the video @ decoder
141 : * @param number_of_streams: number of participating video streams
142 : */
143 : virtual void FrameSizeChange(unsigned int width,
144 : unsigned int height,
145 : unsigned int number_of_streams) = 0;
146 :
147 : /**
148 : * Callback Function reporting decoded I420 frame for processing.
149 : * @param buffer: pointer to decoded video frame
150 : * @param buffer_size: size of the decoded frame
151 : * @param time_stamp: Decoder timestamp, typically 90KHz as per RTP
152 : * @render_time: Wall-clock time at the decoder for synchronization
153 : * purposes in milliseconds
154 : * @handle: opaque handle for image object of decoded video frame.
155 : * NOTE: If decoded video frame is passed through buffer , it is the
156 : * responsibility of the concrete implementations of this class to own copy
157 : * of the frame if needed for time longer than scope of this callback.
158 : * Such implementations should be quick in processing the frames and return
159 : * immediately.
160 : * On the other hand, if decoded video frame is passed through handle, the
161 : * implementations should keep a reference to the (ref-counted) image object
162 : * inside until it's no longer needed.
163 : */
164 : virtual void RenderVideoFrame(const webrtc::VideoFrameBuffer& buffer,
165 : uint32_t time_stamp,
166 : int64_t render_time,
167 : const ImageHandle& handle) = 0;
168 : virtual void RenderVideoFrame(const uint8_t* buffer_y,
169 : uint32_t y_stride,
170 : const uint8_t* buffer_u,
171 : uint32_t u_stride,
172 : const uint8_t* buffer_v,
173 : uint32_t v_stride,
174 : uint32_t time_stamp,
175 : int64_t render_time,
176 : const ImageHandle& handle) = 0;
177 :
178 0 : NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoRenderer)
179 : };
180 :
181 :
182 : /**
183 : * Generic Interface for representing Audio/Video Session
184 : * MediaSession conduit is identified by 2 main components
185 : * 1. Attached Transport Interface for inbound and outbound RTP transport
186 : * 2. Attached Renderer Interface for rendering media data off the network
187 : * This class hides specifics of Media-Engine implementation from the consumers
188 : * of this interface.
189 : * Also provides codec configuration API for the media sent and recevied
190 : */
191 0 : class MediaSessionConduit
192 : {
193 : protected:
194 0 : virtual ~MediaSessionConduit() {}
195 :
196 : public:
197 : enum Type { AUDIO, VIDEO } ;
198 :
199 : virtual Type type() const = 0;
200 :
201 : /**
202 : * Function triggered on Incoming RTP packet from the remote
203 : * endpoint by the transport implementation.
204 : * @param data : RTP Packet (audio/video) to be processed
205 : * @param len : Length of the media packet
206 : * Obtained packets are passed to the Media-Engine for further
207 : * processing , say, decoding
208 : */
209 : virtual MediaConduitErrorCode ReceivedRTPPacket(const void *data, int len, uint32_t ssrc) = 0;
210 :
211 : /**
212 : * Function triggered on Incoming RTCP packet from the remote
213 : * endpoint by the transport implementation.
214 : * @param data : RTCP Packet (audio/video) to be processed
215 : * @param len : Length of the media packet
216 : * Obtained packets are passed to the Media-Engine for further
217 : * processing , say, decoding
218 : */
219 : virtual MediaConduitErrorCode ReceivedRTCPPacket(const void *data, int len) = 0;
220 :
221 : virtual MediaConduitErrorCode StopTransmitting() = 0;
222 : virtual MediaConduitErrorCode StartTransmitting() = 0;
223 : virtual MediaConduitErrorCode StopReceiving() = 0;
224 : virtual MediaConduitErrorCode StartReceiving() = 0;
225 :
226 :
227 : /**
228 : * Function to attach transmitter transport end-point of the Media conduit.
229 : * @param aTransport: Reference to the concrete teansport implementation
230 : * When nullptr, unsets the transmitter transport endpoint.
231 : * Note: Multiple invocations of this call , replaces existing transport with
232 : * with the new one.
233 : * Note: This transport is used for RTP, and RTCP if no receiver transport is
234 : * set. In the future, we should ensure that RTCP sender reports use this
235 : * regardless of whether the receiver transport is set.
236 : */
237 : virtual MediaConduitErrorCode SetTransmitterTransport(RefPtr<TransportInterface> aTransport) = 0;
238 :
239 : /**
240 : * Function to attach receiver transport end-point of the Media conduit.
241 : * @param aTransport: Reference to the concrete teansport implementation
242 : * When nullptr, unsets the receiver transport endpoint.
243 : * Note: Multiple invocations of this call , replaces existing transport with
244 : * with the new one.
245 : * Note: This transport is used for RTCP.
246 : * Note: In the future, we should avoid using this for RTCP sender reports.
247 : */
248 : virtual MediaConduitErrorCode SetReceiverTransport(RefPtr<TransportInterface> aTransport) = 0;
249 :
250 : /* Sets the local SSRCs
251 : * @return true iff the local ssrcs == aSSRCs upon return
252 : * Note: this is an ordered list and {a,b,c} != {b,a,c}
253 : */
254 : virtual bool SetLocalSSRCs(const std::vector<unsigned int>& aSSRCs) = 0;
255 : virtual std::vector<unsigned int> GetLocalSSRCs() const = 0;
256 :
257 : virtual bool GetRemoteSSRC(unsigned int* ssrc) = 0;
258 : virtual bool SetRemoteSSRC(unsigned int ssrc) = 0;
259 : virtual bool SetLocalCNAME(const char* cname) = 0;
260 :
261 : /**
262 : * Functions returning stats needed by w3c stats model.
263 : */
264 :
265 : virtual bool
266 : GetSendPacketTypeStats(webrtc::RtcpPacketTypeCounter* aPacketCounts) = 0;
267 :
268 : virtual bool
269 : GetRecvPacketTypeStats(webrtc::RtcpPacketTypeCounter* aPacketCounts) = 0;
270 :
271 : virtual bool GetVideoEncoderStats(double* framerateMean,
272 : double* framerateStdDev,
273 : double* bitrateMean,
274 : double* bitrateStdDev,
275 : uint32_t* droppedFrames,
276 : uint32_t* framesEncoded) = 0;
277 : virtual bool GetVideoDecoderStats(double* framerateMean,
278 : double* framerateStdDev,
279 : double* bitrateMean,
280 : double* bitrateStdDev,
281 : uint32_t* discardedPackets,
282 : uint32_t* framesDecoded) = 0;
283 : virtual bool GetAVStats(int32_t* jitterBufferDelayMs,
284 : int32_t* playoutBufferDelayMs,
285 : int32_t* avSyncOffsetMs) = 0;
286 : virtual bool GetRTPStats(unsigned int* jitterMs,
287 : unsigned int* cumulativeLost) = 0;
288 : virtual bool GetRTCPReceiverReport(DOMHighResTimeStamp* timestamp,
289 : uint32_t* jitterMs,
290 : uint32_t* packetsReceived,
291 : uint64_t* bytesReceived,
292 : uint32_t* cumulativeLost,
293 : int32_t* rttMs) = 0;
294 : virtual bool GetRTCPSenderReport(DOMHighResTimeStamp* timestamp,
295 : unsigned int* packetsSent,
296 : uint64_t* bytesSent) = 0;
297 :
298 : virtual uint64_t CodecPluginID() = 0;
299 :
300 : virtual void SetPCHandle(const std::string& aPCHandle) = 0;
301 :
302 0 : NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSessionConduit)
303 :
304 : };
305 :
306 : // Abstract base classes for external encoder/decoder.
307 0 : class CodecPluginID
308 : {
309 : public:
310 0 : virtual ~CodecPluginID() {}
311 :
312 : virtual uint64_t PluginID() const = 0;
313 : };
314 :
315 0 : class VideoEncoder : public CodecPluginID
316 : {
317 : public:
318 0 : virtual ~VideoEncoder() {}
319 : };
320 :
321 0 : class VideoDecoder : public CodecPluginID
322 : {
323 : public:
324 0 : virtual ~VideoDecoder() {}
325 : };
326 :
327 : /**
328 : * MediaSessionConduit for video
329 : * Refer to the comments on MediaSessionConduit above for overall
330 : * information
331 : */
332 : class VideoSessionConduit : public MediaSessionConduit
333 : {
334 : public:
335 : /**
336 : * Factory function to create and initialize a Video Conduit Session
337 : * @param webrtc::Call instance shared by paired audio and video
338 : * media conduits
339 : * @result Concrete VideoSessionConduitObject or nullptr in the case
340 : * of failure
341 : */
342 : static RefPtr<VideoSessionConduit> Create(RefPtr<WebRtcCallWrapper> aCall);
343 :
344 : enum FrameRequestType
345 : {
346 : FrameRequestNone,
347 : FrameRequestFir,
348 : FrameRequestPli,
349 : FrameRequestUnknown
350 : };
351 :
352 0 : VideoSessionConduit() : mFrameRequestMethod(FrameRequestNone),
353 : mUsingNackBasic(false),
354 : mUsingTmmbr(false),
355 0 : mUsingFEC(false) {}
356 :
357 0 : virtual ~VideoSessionConduit() {}
358 :
359 0 : virtual Type type() const { return VIDEO; }
360 :
361 : /**
362 : * Adds negotiated RTP extensions
363 : * XXX Move to MediaSessionConduit
364 : */
365 : virtual void SetLocalRTPExtensions(bool aIsSend,
366 : const std::vector<webrtc::RtpExtension>& extensions) = 0;
367 :
368 : /**
369 : * Returns the negotiated RTP extensions
370 : */
371 : virtual std::vector<webrtc::RtpExtension> GetLocalRTPExtensions(bool aIsSend) const = 0;
372 :
373 :
374 : /**
375 : * Function to attach Renderer end-point of the Media-Video conduit.
376 : * @param aRenderer : Reference to the concrete Video renderer implementation
377 : * Note: Multiple invocations of this API shall remove an existing renderer
378 : * and attaches the new to the Conduit.
379 : */
380 : virtual MediaConduitErrorCode AttachRenderer(RefPtr<mozilla::VideoRenderer> aRenderer) = 0;
381 : virtual void DetachRenderer() = 0;
382 :
383 : virtual bool SetRemoteSSRC(unsigned int ssrc) = 0;
384 :
385 : /**
386 : * Function to deliver a capture video frame for encoding and transport
387 : * @param video_frame: pointer to captured video-frame.
388 : * @param video_frame_length: size of the frame
389 : * @param width, height: dimensions of the frame
390 : * @param video_type: Type of the video frame - I420, RAW
391 : * @param captured_time: timestamp when the frame was captured.
392 : * if 0 timestamp is automatcally generated
393 : * NOTE: ConfigureSendMediaCodec() MUST be called before this function can be invoked
394 : * This ensures the inserted video-frames can be transmitted by the conduit
395 : */
396 : virtual MediaConduitErrorCode SendVideoFrame(unsigned char* video_frame,
397 : unsigned int video_frame_length,
398 : unsigned short width,
399 : unsigned short height,
400 : VideoType video_type,
401 : uint64_t capture_time) = 0;
402 : virtual MediaConduitErrorCode SendVideoFrame(webrtc::VideoFrame& frame) = 0;
403 :
404 : virtual MediaConduitErrorCode ConfigureCodecMode(webrtc::VideoCodecMode) = 0;
405 : /**
406 : * Function to configure send codec for the video session
407 : * @param sendSessionConfig: CodecConfiguration
408 : * @result: On Success, the video engine is configured with passed in codec for send
409 : * On failure, video engine transmit functionality is disabled.
410 : * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting
411 : * transmission sub-system on the engine
412 : *
413 : */
414 : virtual MediaConduitErrorCode ConfigureSendMediaCodec(const VideoCodecConfig* sendSessionConfig) = 0;
415 :
416 : /**
417 : * Function to configurelist of receive codecs for the video session
418 : * @param sendSessionConfig: CodecConfiguration
419 : * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting
420 : * reception sub-system on the engine
421 : *
422 : */
423 : virtual MediaConduitErrorCode ConfigureRecvMediaCodecs(
424 : const std::vector<VideoCodecConfig* >& recvCodecConfigList) = 0;
425 :
426 : /**
427 : * These methods allow unit tests to double-check that the
428 : * max-fs and max-fr related settings are as expected.
429 : */
430 : virtual unsigned short SendingWidth() = 0;
431 :
432 : virtual unsigned short SendingHeight() = 0;
433 :
434 : virtual unsigned int SendingMaxFs() = 0;
435 :
436 : virtual unsigned int SendingMaxFr() = 0;
437 :
438 : /**
439 : * These methods allow unit tests to double-check that the
440 : * rtcp-fb settings are as expected.
441 : */
442 : FrameRequestType FrameRequestMethod() const {
443 : return mFrameRequestMethod;
444 : }
445 :
446 : bool UsingNackBasic() const {
447 : return mUsingNackBasic;
448 : }
449 :
450 : bool UsingTmmbr() const {
451 : return mUsingTmmbr;
452 : }
453 :
454 : bool UsingFEC() const {
455 : return mUsingFEC;
456 : }
457 :
458 : protected:
459 : /* RTCP feedback settings, for unit testing purposes */
460 : FrameRequestType mFrameRequestMethod;
461 : bool mUsingNackBasic;
462 : bool mUsingTmmbr;
463 : bool mUsingFEC;
464 : };
465 :
466 : /**
467 : * MediaSessionConduit for audio
468 : * Refer to the comments on MediaSessionConduit above for overall
469 : * information
470 : */
471 0 : class AudioSessionConduit : public MediaSessionConduit
472 : {
473 : public:
474 :
475 : /**
476 : * Factory function to create and initialize an Audio Conduit Session
477 : * @param webrtc::Call instance shared by paired audio and video
478 : * media conduits
479 : * @result Concrete AudioSessionConduitObject or nullptr in the case
480 : * of failure
481 : */
482 : static RefPtr<AudioSessionConduit> Create();
483 :
484 0 : virtual ~AudioSessionConduit() {}
485 :
486 0 : virtual Type type() const { return AUDIO; }
487 :
488 :
489 : /**
490 : * Function to deliver externally captured audio sample for encoding and transport
491 : * @param audioData [in]: Pointer to array containing a frame of audio
492 : * @param lengthSamples [in]: Length of audio frame in samples in multiple of 10 milliseconds
493 : * Ex: Frame length is 160, 320, 440 for 16, 32, 44 kHz sampling rates
494 : respectively.
495 : audioData[] is lengthSamples in size
496 : say, for 16kz sampling rate, audioData[] should contain 160
497 : samples of 16-bits each for a 10m audio frame.
498 : * @param samplingFreqHz [in]: Frequency/rate of the sampling in Hz ( 16000, 32000 ...)
499 : * @param capture_delay [in]: Approx Delay from recording until it is delivered to VoiceEngine
500 : in milliseconds.
501 : * NOTE: ConfigureSendMediaCodec() SHOULD be called before this function can be invoked
502 : * This ensures the inserted audio-samples can be transmitted by the conduit
503 : *
504 : */
505 : virtual MediaConduitErrorCode SendAudioFrame(const int16_t audioData[],
506 : int32_t lengthSamples,
507 : int32_t samplingFreqHz,
508 : int32_t capture_delay) = 0;
509 :
510 : /**
511 : * Function to grab a decoded audio-sample from the media engine for rendering
512 : * / playoutof length 10 milliseconds.
513 : *
514 : * @param speechData [in]: Pointer to a array to which a 10ms frame of audio will be copied
515 : * @param samplingFreqHz [in]: Frequency of the sampling for playback in Hertz (16000, 32000,..)
516 : * @param capture_delay [in]: Estimated Time between reading of the samples to rendering/playback
517 : * @param lengthSamples [out]: Will contain length of the audio frame in samples at return.
518 : Ex: A value of 160 implies 160 samples each of 16-bits was copied
519 : into speechData
520 : * NOTE: This function should be invoked every 10 milliseconds for the best
521 : * peformance
522 : * NOTE: ConfigureRecvMediaCodec() SHOULD be called before this function can be invoked
523 : * This ensures the decoded samples are ready for reading.
524 : *
525 : */
526 : virtual MediaConduitErrorCode GetAudioFrame(int16_t speechData[],
527 : int32_t samplingFreqHz,
528 : int32_t capture_delay,
529 : int& lengthSamples) = 0;
530 :
531 : /**
532 : * Function to configure send codec for the audio session
533 : * @param sendSessionConfig: CodecConfiguration
534 : * NOTE: See VideoConduit for more information
535 : */
536 :
537 : virtual MediaConduitErrorCode ConfigureSendMediaCodec(const AudioCodecConfig* sendCodecConfig) = 0;
538 :
539 : /**
540 : * Function to configure list of receive codecs for the audio session
541 : * @param sendSessionConfig: CodecConfiguration
542 : * NOTE: See VideoConduit for more information
543 : */
544 : virtual MediaConduitErrorCode ConfigureRecvMediaCodecs(
545 : const std::vector<AudioCodecConfig* >& recvCodecConfigList) = 0;
546 : /**
547 : * Function to enable the audio level extension
548 : * @param enabled: enable extension
549 : * @param id: id to be used for this rtp header extension
550 : * NOTE: See AudioConduit for more information
551 : */
552 : virtual MediaConduitErrorCode EnableAudioLevelExtension(bool enabled, uint8_t id) = 0;
553 :
554 : virtual bool SetDtmfPayloadType(unsigned char type, int freq) = 0;
555 :
556 : virtual bool InsertDTMFTone(int channel, int eventCode, bool outOfBand,
557 : int lengthMs, int attenuationDb) = 0;
558 :
559 : };
560 : }
561 : #endif
|
