Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #ifndef WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODING_H_
12 : #define WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODING_H_
13 :
14 : #if defined(WEBRTC_WIN)
15 : // This is a workaround on Windows due to the fact that some Windows
16 : // headers define CreateEvent as a macro to either CreateEventW or CreateEventA.
17 : // This can cause problems since we use that name as well and could
18 : // declare them as one thing here whereas in another place a windows header
19 : // may have been included and then implementing CreateEvent() causes compilation
20 : // errors. So for consistency, we include the main windows header here.
21 : #include <windows.h>
22 : #endif
23 :
24 : #include "webrtc/api/video/video_frame.h"
25 : #include "webrtc/modules/include/module.h"
26 : #include "webrtc/modules/include/module_common_types.h"
27 : #include "webrtc/modules/video_coding/include/video_coding_defines.h"
28 : #include "webrtc/system_wrappers/include/event_wrapper.h"
29 :
30 : namespace webrtc {
31 :
32 : class Clock;
33 : class EncodedImageCallback;
34 : // TODO(pbos): Remove VCMQMSettingsCallback completely. This might be done by
35 : // removing the VCM and use VideoSender/VideoReceiver as a public interface
36 : // directly.
37 : class VCMQMSettingsCallback;
38 : class VideoBitrateAllocator;
39 : class VideoEncoder;
40 : class VideoDecoder;
41 : struct CodecSpecificInfo;
42 :
43 : class EventFactory {
44 : public:
45 : virtual ~EventFactory() {}
46 :
47 : virtual EventWrapper* CreateEvent() = 0;
48 : };
49 :
50 : class EventFactoryImpl : public EventFactory {
51 : public:
52 : virtual ~EventFactoryImpl() {}
53 :
54 : virtual EventWrapper* CreateEvent() { return EventWrapper::Create(); }
55 : };
56 :
57 : // Used to indicate which decode with errors mode should be used.
58 : enum VCMDecodeErrorMode {
59 : kNoErrors, // Never decode with errors. Video will freeze
60 : // if nack is disabled.
61 : kSelectiveErrors, // Frames that are determined decodable in
62 : // VCMSessionInfo may be decoded with missing
63 : // packets. As not all incomplete frames will be
64 : // decodable, video will freeze if nack is disabled.
65 : kWithErrors // Release frames as needed. Errors may be
66 : // introduced as some encoded frames may not be
67 : // complete.
68 : };
69 :
70 0 : class VideoCodingModule : public Module {
71 : public:
72 : enum SenderNackMode { kNackNone, kNackAll, kNackSelective };
73 :
74 : enum ReceiverRobustness { kNone, kHardNack, kSoftNack, kReferenceSelection };
75 :
76 : static VideoCodingModule* Create(Clock* clock, EventFactory* event_factory);
77 :
78 : static VideoCodingModule* Create(
79 : Clock* clock,
80 : VCMQMSettingsCallback* qm_settings_callback,
81 : NackSender* nack_sender,
82 : KeyFrameRequestSender* keyframe_request_sender,
83 : EncodedImageCallback* pre_decode_image_callback);
84 :
85 : static VideoCodingModule* Create(
86 : Clock* clock,
87 : EventFactory* event_factory,
88 : NackSender* nack_sender,
89 : KeyFrameRequestSender* keyframe_request_sender);
90 :
91 : // Get supported codec settings using codec type
92 : //
93 : // Input:
94 : // - codecType : The codec type to get settings for
95 : // - codec : Memory where the codec settings will be stored
96 : //
97 : // Return value : VCM_OK, on success
98 : // VCM_PARAMETER_ERROR if codec not supported
99 : static void Codec(VideoCodecType codecType, VideoCodec* codec);
100 :
101 : /*
102 : * Sender
103 : */
104 :
105 : // Registers a codec to be used for encoding. Calling this
106 : // API multiple times overwrites any previously registered codecs.
107 : //
108 : // NOTE: Must be called on the thread that constructed the VCM instance.
109 : //
110 : // Input:
111 : // - sendCodec : Settings for the codec to be registered.
112 : // - numberOfCores : The number of cores the codec is allowed
113 : // to use.
114 : // - maxPayloadSize : The maximum size each payload is allowed
115 : // to have. Usually MTU - overhead.
116 : //
117 : // Return value : VCM_OK, on success.
118 : // < 0, on error.
119 : virtual int32_t RegisterSendCodec(const VideoCodec* sendCodec,
120 : uint32_t numberOfCores,
121 : uint32_t maxPayloadSize) = 0;
122 :
123 : // Register an external encoder object. This can not be used together with
124 : // external decoder callbacks.
125 : //
126 : // Input:
127 : // - externalEncoder : Encoder object to be used for encoding frames
128 : // inserted
129 : // with the AddVideoFrame API.
130 : // - payloadType : The payload type bound which this encoder is bound
131 : // to.
132 : //
133 : // Return value : VCM_OK, on success.
134 : // < 0, on error.
135 : // TODO(pbos): Remove return type when unused elsewhere.
136 : virtual int32_t RegisterExternalEncoder(VideoEncoder* externalEncoder,
137 : uint8_t payloadType,
138 : bool internalSource = false) = 0;
139 :
140 : // API to get currently configured encoder target bitrate in bits/s.
141 : //
142 : // Return value : 0, on success.
143 : // < 0, on error.
144 : virtual int Bitrate(unsigned int* bitrate) const = 0;
145 :
146 : // API to get currently configured encoder target frame rate.
147 : //
148 : // Return value : 0, on success.
149 : // < 0, on error.
150 : virtual int FrameRate(unsigned int* framerate) const = 0;
151 :
152 : // Sets the parameters describing the send channel. These parameters are
153 : // inputs to the
154 : // Media Optimization inside the VCM and also specifies the target bit rate
155 : // for the
156 : // encoder. Bit rate used by NACK should already be compensated for by the
157 : // user.
158 : //
159 : // Input:
160 : // - target_bitrate : The target bitrate for VCM in bits/s.
161 : // - lossRate : Fractions of lost packets the past second.
162 : // (loss rate in percent = 100 * packetLoss /
163 : // 255)
164 : // - rtt : Current round-trip time in ms.
165 : //
166 : // Return value : VCM_OK, on success.
167 : // < 0, on error.
168 : virtual int32_t SetChannelParameters(uint32_t target_bitrate,
169 : uint8_t lossRate,
170 : int64_t rtt) = 0;
171 :
172 : // Sets the parameters describing the receive channel. These parameters are
173 : // inputs to the
174 : // Media Optimization inside the VCM.
175 : //
176 : // Input:
177 : // - rtt : Current round-trip time in ms.
178 : // with the most amount available bandwidth in
179 : // a conference
180 : // scenario
181 : //
182 : // Return value : VCM_OK, on success.
183 : // < 0, on error.
184 : virtual int32_t SetReceiveChannelParameters(int64_t rtt) = 0;
185 :
186 : // Deprecated: This method currently does not have any effect.
187 : // Register a video protection callback which will be called to deliver
188 : // the requested FEC rate and NACK status (on/off).
189 : // TODO(perkj): Remove once no projects use it.
190 : virtual int32_t RegisterProtectionCallback(
191 : VCMProtectionCallback* protection) = 0;
192 :
193 : // Enable or disable a video protection method.
194 : //
195 : // Input:
196 : // - videoProtection : The method to enable or disable.
197 : // - enable : True if the method should be enabled, false if
198 : // it should be disabled.
199 : //
200 : // Return value : VCM_OK, on success.
201 : // < 0, on error.
202 : virtual int32_t SetVideoProtection(VCMVideoProtection videoProtection,
203 : bool enable) = 0;
204 :
205 : // Add one raw video frame to the encoder. This function does all the
206 : // necessary
207 : // processing, then decides what frame type to encode, or if the frame should
208 : // be
209 : // dropped. If the frame should be encoded it passes the frame to the encoder
210 : // before it returns.
211 : //
212 : // Input:
213 : // - videoFrame : Video frame to encode.
214 : // - codecSpecificInfo : Extra codec information, e.g., pre-parsed
215 : // in-band signaling.
216 : //
217 : // Return value : VCM_OK, on success.
218 : // < 0, on error.
219 : virtual int32_t AddVideoFrame(
220 : const VideoFrame& videoFrame,
221 : const CodecSpecificInfo* codecSpecificInfo = NULL) = 0;
222 :
223 : // Next frame encoded should be an intra frame (keyframe).
224 : //
225 : // Return value : VCM_OK, on success.
226 : // < 0, on error.
227 : virtual int32_t IntraFrameRequest(size_t stream_index) = 0;
228 :
229 : // Frame Dropper enable. Can be used to disable the frame dropping when the
230 : // encoder
231 : // over-uses its bit rate. This API is designed to be used when the encoded
232 : // frames
233 : // are supposed to be stored to an AVI file, or when the I420 codec is used
234 : // and the
235 : // target bit rate shouldn't affect the frame rate.
236 : //
237 : // Input:
238 : // - enable : True to enable the setting, false to disable it.
239 : //
240 : // Return value : VCM_OK, on success.
241 : // < 0, on error.
242 : virtual int32_t EnableFrameDropper(bool enable) = 0;
243 :
244 : /*
245 : * Receiver
246 : */
247 :
248 : // Register possible receive codecs, can be called multiple times for
249 : // different codecs.
250 : // The module will automatically switch between registered codecs depending on
251 : // the
252 : // payload type of incoming frames. The actual decoder will be created when
253 : // needed.
254 : //
255 : // Input:
256 : // - receiveCodec : Settings for the codec to be registered.
257 : // - numberOfCores : Number of CPU cores that the decoder is allowed
258 : // to use.
259 : // - requireKeyFrame : Set this to true if you don't want any delta
260 : // frames
261 : // to be decoded until the first key frame has been
262 : // decoded.
263 : //
264 : // Return value : VCM_OK, on success.
265 : // < 0, on error.
266 : virtual int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec,
267 : int32_t numberOfCores,
268 : bool requireKeyFrame = false) = 0;
269 :
270 : // Register an externally defined decoder/renderer object. Can be a decoder
271 : // only or a
272 : // decoder coupled with a renderer. Note that RegisterReceiveCodec must be
273 : // called to
274 : // be used for decoding incoming streams.
275 : //
276 : // Input:
277 : // - externalDecoder : The external decoder/renderer object.
278 : // - payloadType : The payload type which this decoder should
279 : // be
280 : // registered to.
281 : //
282 : virtual void RegisterExternalDecoder(VideoDecoder* externalDecoder,
283 : uint8_t payloadType) = 0;
284 :
285 : // Register a receive callback. Will be called whenever there is a new frame
286 : // ready
287 : // for rendering.
288 : //
289 : // Input:
290 : // - receiveCallback : The callback object to be used by the
291 : // module when a
292 : // frame is ready for rendering.
293 : // De-register with a NULL pointer.
294 : //
295 : // Return value : VCM_OK, on success.
296 : // < 0, on error.
297 : virtual int32_t RegisterReceiveCallback(
298 : VCMReceiveCallback* receiveCallback) = 0;
299 :
300 : // Register a receive statistics callback which will be called to deliver
301 : // information
302 : // about the video stream received by the receiving side of the VCM, for
303 : // instance the
304 : // average frame rate and bit rate.
305 : //
306 : // Input:
307 : // - receiveStats : The callback object to register.
308 : //
309 : // Return value : VCM_OK, on success.
310 : // < 0, on error.
311 : virtual int32_t RegisterReceiveStatisticsCallback(
312 : VCMReceiveStatisticsCallback* receiveStats) = 0;
313 :
314 : // Register a decoder timing callback which will be called to deliver
315 : // information about the timing of the decoder in the receiving side of the
316 : // VCM, for instance the current and maximum frame decode latency.
317 : //
318 : // Input:
319 : // - decoderTiming : The callback object to register.
320 : //
321 : // Return value : VCM_OK, on success.
322 : // < 0, on error.
323 : virtual int32_t RegisterDecoderTimingCallback(
324 : VCMDecoderTimingCallback* decoderTiming) = 0;
325 :
326 : // Register a frame type request callback. This callback will be called when
327 : // the
328 : // module needs to request specific frame types from the send side.
329 : //
330 : // Input:
331 : // - frameTypeCallback : The callback object to be used by the
332 : // module when
333 : // requesting a specific type of frame from
334 : // the send side.
335 : // De-register with a NULL pointer.
336 : //
337 : // Return value : VCM_OK, on success.
338 : // < 0, on error.
339 : virtual int32_t RegisterFrameTypeCallback(
340 : VCMFrameTypeCallback* frameTypeCallback) = 0;
341 :
342 : // Registers a callback which is called whenever the receive side of the VCM
343 : // encounters holes in the packet sequence and needs packets to be
344 : // retransmitted.
345 : //
346 : // Input:
347 : // - callback : The callback to be registered in the VCM.
348 : //
349 : // Return value : VCM_OK, on success.
350 : // <0, on error.
351 : virtual int32_t RegisterPacketRequestCallback(
352 : VCMPacketRequestCallback* callback) = 0;
353 :
354 : // Register a receive state change callback. This callback will be called when the
355 : // module state has changed
356 : //
357 : // Input:
358 : // - callback : The callback object to be used by the module when
359 : // the receiver decode state changes.
360 : // De-register with a NULL pointer.
361 : //
362 : // Return value : VCM_OK, on success.
363 : // < 0, on error.
364 : virtual int32_t RegisterReceiveStateCallback(
365 : VCMReceiveStateCallback* callback) = 0;
366 :
367 : // Waits for the next frame in the jitter buffer to become complete
368 : // (waits no longer than maxWaitTimeMs), then passes it to the decoder for
369 : // decoding.
370 : // Should be called as often as possible to get the most out of the decoder.
371 : //
372 : // Return value : VCM_OK, on success.
373 : // < 0, on error.
374 : virtual int32_t Decode(uint16_t maxWaitTimeMs = 200) = 0;
375 :
376 : // API to get the codec which is currently used for decoding by the module.
377 : //
378 : // Input:
379 : // - currentReceiveCodec : Settings for the codec to be registered.
380 : //
381 : // Return value : VCM_OK, on success.
382 : // < 0, on error.
383 : virtual int32_t ReceiveCodec(VideoCodec* currentReceiveCodec) const = 0;
384 :
385 : // API to get the codec type currently used for decoding by the module.
386 : //
387 : // Return value : codecy type, on success.
388 : // kVideoCodecUnknown, on error or if no receive codec is
389 : // registered
390 : virtual VideoCodecType ReceiveCodec() const = 0;
391 :
392 : // Insert a parsed packet into the receiver side of the module. Will be placed
393 : // in the
394 : // jitter buffer waiting for the frame to become complete. Returns as soon as
395 : // the packet
396 : // has been placed in the jitter buffer.
397 : //
398 : // Input:
399 : // - incomingPayload : Payload of the packet.
400 : // - payloadLength : Length of the payload.
401 : // - rtpInfo : The parsed header.
402 : //
403 : // Return value : VCM_OK, on success.
404 : // < 0, on error.
405 : virtual int32_t IncomingPacket(const uint8_t* incomingPayload,
406 : size_t payloadLength,
407 : const WebRtcRTPHeader& rtpInfo) = 0;
408 :
409 : // Minimum playout delay (Used for lip-sync). This is the minimum delay
410 : // required
411 : // to sync with audio. Not included in VideoCodingModule::Delay()
412 : // Defaults to 0 ms.
413 : //
414 : // Input:
415 : // - minPlayoutDelayMs : Additional delay in ms.
416 : //
417 : // Return value : VCM_OK, on success.
418 : // < 0, on error.
419 : virtual int32_t SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) = 0;
420 :
421 : // Set the time required by the renderer to render a frame.
422 : //
423 : // Input:
424 : // - timeMS : The time in ms required by the renderer to render a
425 : // frame.
426 : //
427 : // Return value : VCM_OK, on success.
428 : // < 0, on error.
429 : virtual int32_t SetRenderDelay(uint32_t timeMS) = 0;
430 :
431 : // The total delay desired by the VCM. Can be less than the minimum
432 : // delay set with SetMinimumPlayoutDelay.
433 : //
434 : // Return value : Total delay in ms, on success.
435 : // < 0, on error.
436 : virtual int32_t Delay() const = 0;
437 :
438 : // Returns the number of packets discarded by the jitter buffer due to being
439 : // too late. This can include duplicated packets which arrived after the
440 : // frame was sent to the decoder. Therefore packets which were prematurely
441 : // NACKed will be counted.
442 : virtual uint32_t DiscardedPackets() const = 0;
443 :
444 : // Robustness APIs
445 :
446 : // Set the receiver robustness mode. The mode decides how the receiver
447 : // responds to losses in the stream. The type of counter-measure (soft or
448 : // hard NACK, dual decoder, RPS, etc.) is selected through the
449 : // robustnessMode parameter. The errorMode parameter decides if it is
450 : // allowed to display frames corrupted by losses. Note that not all
451 : // combinations of the two parameters are feasible. An error will be
452 : // returned for invalid combinations.
453 : // Input:
454 : // - robustnessMode : selected robustness mode.
455 : // - errorMode : selected error mode.
456 : //
457 : // Return value : VCM_OK, on success;
458 : // < 0, on error.
459 : virtual int SetReceiverRobustnessMode(ReceiverRobustness robustnessMode,
460 : VCMDecodeErrorMode errorMode) = 0;
461 :
462 : // Set the decode error mode. The mode decides which errors (if any) are
463 : // allowed in decodable frames. Note that setting decode_error_mode to
464 : // anything other than kWithErrors without enabling nack will cause
465 : // long-term freezes (resulting from frequent key frame requests) if
466 : // packet loss occurs.
467 : virtual void SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) = 0;
468 :
469 : // Sets the maximum number of sequence numbers that we are allowed to NACK
470 : // and the oldest sequence number that we will consider to NACK. If a
471 : // sequence number older than |max_packet_age_to_nack| is missing
472 : // a key frame will be requested. A key frame will also be requested if the
473 : // time of incomplete or non-continuous frames in the jitter buffer is above
474 : // |max_incomplete_time_ms|.
475 : virtual void SetNackSettings(size_t max_nack_list_size,
476 : int max_packet_age_to_nack,
477 : int max_incomplete_time_ms) = 0;
478 :
479 : // Setting a desired delay to the VCM receiver. Video rendering will be
480 : // delayed by at least desired_delay_ms.
481 : virtual int SetMinReceiverDelay(int desired_delay_ms) = 0;
482 :
483 : // Set current load state of the CPU
484 : virtual void SetCPULoadState(CPULoadState state) = 0;
485 :
486 : virtual void RegisterPostEncodeImageCallback(
487 : EncodedImageCallback* post_encode_callback) = 0;
488 : // Releases pending decode calls, permitting faster thread shutdown.
489 : virtual void TriggerDecoderShutdown() = 0;
490 : // resets underlying objects
491 : virtual void Reset() = 0;
492 : };
493 :
494 : } // namespace webrtc
495 :
496 : #endif // WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODING_H_
|
