Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_NETEQ_H_
12 : #define WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_NETEQ_H_
13 :
14 : #include <string.h> // Provide access to size_t.
15 :
16 : #include <string>
17 :
18 : #include "webrtc/base/constructormagic.h"
19 : #include "webrtc/base/optional.h"
20 : #include "webrtc/base/scoped_ref_ptr.h"
21 : #include "webrtc/common_types.h"
22 : #include "webrtc/modules/audio_coding/neteq/audio_decoder_impl.h"
23 : #include "webrtc/typedefs.h"
24 :
25 : namespace webrtc {
26 :
27 : // Forward declarations.
28 : class AudioFrame;
29 : struct WebRtcRTPHeader;
30 : class AudioDecoderFactory;
31 :
32 : struct NetEqNetworkStatistics {
33 : uint16_t current_buffer_size_ms; // Current jitter buffer size in ms.
34 : uint16_t preferred_buffer_size_ms; // Target buffer size in ms.
35 : uint16_t jitter_peaks_found; // 1 if adding extra delay due to peaky
36 : // jitter; 0 otherwise.
37 : uint16_t packet_loss_rate; // Loss rate (network + late) in Q14.
38 : uint16_t packet_discard_rate; // Late loss rate in Q14.
39 : uint16_t expand_rate; // Fraction (of original stream) of synthesized
40 : // audio inserted through expansion (in Q14).
41 : uint16_t speech_expand_rate; // Fraction (of original stream) of synthesized
42 : // speech inserted through expansion (in Q14).
43 : uint16_t preemptive_rate; // Fraction of data inserted through pre-emptive
44 : // expansion (in Q14).
45 : uint16_t accelerate_rate; // Fraction of data removed through acceleration
46 : // (in Q14).
47 : uint16_t secondary_decoded_rate; // Fraction of data coming from secondary
48 : // decoding (in Q14).
49 : int32_t clockdrift_ppm; // Average clock-drift in parts-per-million
50 : // (positive or negative).
51 : size_t added_zero_samples; // Number of zero samples added in "off" mode.
52 : // Statistics for packet waiting times, i.e., the time between a packet
53 : // arrives until it is decoded.
54 : int mean_waiting_time_ms;
55 : int median_waiting_time_ms;
56 : int min_waiting_time_ms;
57 : int max_waiting_time_ms;
58 : };
59 :
60 : enum NetEqPlayoutMode {
61 : kPlayoutOn,
62 : kPlayoutOff,
63 : kPlayoutFax,
64 : kPlayoutStreaming
65 : };
66 :
67 : // This is the interface class for NetEq.
68 : class NetEq {
69 : public:
70 : enum BackgroundNoiseMode {
71 : kBgnOn, // Default behavior with eternal noise.
72 : kBgnFade, // Noise fades to zero after some time.
73 : kBgnOff // Background noise is always zero.
74 : };
75 :
76 : struct Config {
77 0 : Config()
78 0 : : sample_rate_hz(16000),
79 : enable_post_decode_vad(false),
80 : max_packets_in_buffer(50),
81 : // |max_delay_ms| has the same effect as calling SetMaximumDelay().
82 : max_delay_ms(2000),
83 : background_noise_mode(kBgnOff),
84 : playout_mode(kPlayoutOn),
85 0 : enable_fast_accelerate(false) {}
86 :
87 : std::string ToString() const;
88 :
89 : int sample_rate_hz; // Initial value. Will change with input data.
90 : bool enable_post_decode_vad;
91 : size_t max_packets_in_buffer;
92 : int max_delay_ms;
93 : BackgroundNoiseMode background_noise_mode;
94 : NetEqPlayoutMode playout_mode;
95 : bool enable_fast_accelerate;
96 : bool enable_muted_state = false;
97 : };
98 :
99 : enum ReturnCodes {
100 : kOK = 0,
101 : kFail = -1,
102 : kNotImplemented = -2
103 : };
104 :
105 : enum ErrorCodes {
106 : kNoError = 0,
107 : kOtherError,
108 : kInvalidRtpPayloadType,
109 : kUnknownRtpPayloadType,
110 : kCodecNotSupported,
111 : kDecoderExists,
112 : kDecoderNotFound,
113 : kInvalidSampleRate,
114 : kInvalidPointer,
115 : kAccelerateError,
116 : kPreemptiveExpandError,
117 : kComfortNoiseErrorCode,
118 : kDecoderErrorCode,
119 : kOtherDecoderError,
120 : kInvalidOperation,
121 : kDtmfParameterError,
122 : kDtmfParsingError,
123 : kDtmfInsertError,
124 : kStereoNotSupported,
125 : kSampleUnderrun,
126 : kDecodedTooMuch,
127 : kRedundancySplitError,
128 : kPacketBufferCorruption
129 : };
130 :
131 : // Creates a new NetEq object, with parameters set in |config|. The |config|
132 : // object will only have to be valid for the duration of the call to this
133 : // method.
134 : static NetEq* Create(
135 : const NetEq::Config& config,
136 : const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory);
137 :
138 0 : virtual ~NetEq() {}
139 :
140 : // Inserts a new packet into NetEq. The |receive_timestamp| is an indication
141 : // of the time when the packet was received, and should be measured with
142 : // the same tick rate as the RTP timestamp of the current payload.
143 : // Returns 0 on success, -1 on failure.
144 : virtual int InsertPacket(const WebRtcRTPHeader& rtp_header,
145 : rtc::ArrayView<const uint8_t> payload,
146 : uint32_t receive_timestamp) = 0;
147 :
148 : // Instructs NetEq to deliver 10 ms of audio data. The data is written to
149 : // |audio_frame|. All data in |audio_frame| is wiped; |data_|, |speech_type_|,
150 : // |num_channels_|, |sample_rate_hz_|, |samples_per_channel_|, and
151 : // |vad_activity_| are updated upon success. If an error is returned, some
152 : // fields may not have been updated, or may contain inconsistent values.
153 : // If muted state is enabled (through Config::enable_muted_state), |muted|
154 : // may be set to true after a prolonged expand period. When this happens, the
155 : // |data_| in |audio_frame| is not written, but should be interpreted as being
156 : // all zeros.
157 : // Returns kOK on success, or kFail in case of an error.
158 : virtual int GetAudio(AudioFrame* audio_frame, bool* muted) = 0;
159 :
160 : // Associates |rtp_payload_type| with |codec| and |codec_name|, and stores the
161 : // information in the codec database. Returns 0 on success, -1 on failure.
162 : // The name is only used to provide information back to the caller about the
163 : // decoders. Hence, the name is arbitrary, and may be empty.
164 : virtual int RegisterPayloadType(NetEqDecoder codec,
165 : const std::string& codec_name,
166 : uint8_t rtp_payload_type) = 0;
167 :
168 : // Provides an externally created decoder object |decoder| to insert in the
169 : // decoder database. The decoder implements a decoder of type |codec| and
170 : // associates it with |rtp_payload_type| and |codec_name|. Returns kOK on
171 : // success, kFail on failure. The name is only used to provide information
172 : // back to the caller about the decoders. Hence, the name is arbitrary, and
173 : // may be empty.
174 : virtual int RegisterExternalDecoder(AudioDecoder* decoder,
175 : NetEqDecoder codec,
176 : const std::string& codec_name,
177 : uint8_t rtp_payload_type) = 0;
178 :
179 : // Associates |rtp_payload_type| with the given codec, which NetEq will
180 : // instantiate when it needs it. Returns true iff successful.
181 : virtual bool RegisterPayloadType(int rtp_payload_type,
182 : const SdpAudioFormat& audio_format) = 0;
183 :
184 : // Removes |rtp_payload_type| from the codec database. Returns 0 on success,
185 : // -1 on failure.
186 : virtual int RemovePayloadType(uint8_t rtp_payload_type) = 0;
187 :
188 : // Removes all payload types from the codec database.
189 : virtual void RemoveAllPayloadTypes() = 0;
190 :
191 : // Sets a minimum delay in millisecond for packet buffer. The minimum is
192 : // maintained unless a higher latency is dictated by channel condition.
193 : // Returns true if the minimum is successfully applied, otherwise false is
194 : // returned.
195 : virtual bool SetMinimumDelay(int delay_ms) = 0;
196 :
197 : // Sets a maximum delay in milliseconds for packet buffer. The latency will
198 : // not exceed the given value, even required delay (given the channel
199 : // conditions) is higher. Calling this method has the same effect as setting
200 : // the |max_delay_ms| value in the NetEq::Config struct.
201 : virtual bool SetMaximumDelay(int delay_ms) = 0;
202 :
203 : // The smallest latency required. This is computed bases on inter-arrival
204 : // time and internal NetEq logic. Note that in computing this latency none of
205 : // the user defined limits (applied by calling setMinimumDelay() and/or
206 : // SetMaximumDelay()) are applied.
207 : virtual int LeastRequiredDelayMs() const = 0;
208 :
209 : // Not implemented.
210 : virtual int SetTargetDelay() = 0;
211 :
212 : // Not implemented.
213 : virtual int TargetDelay() = 0;
214 :
215 : // Returns the current total delay (packet buffer and sync buffer) in ms.
216 : virtual int CurrentDelayMs() const = 0;
217 :
218 : // Returns the current total delay (packet buffer and sync buffer) in ms,
219 : // with smoothing applied to even out short-time fluctuations due to jitter.
220 : // The packet buffer part of the delay is not updated during DTX/CNG periods.
221 : virtual int FilteredCurrentDelayMs() const = 0;
222 :
223 : // Sets the playout mode to |mode|.
224 : // Deprecated. Set the mode in the Config struct passed to the constructor.
225 : // TODO(henrik.lundin) Delete.
226 : virtual void SetPlayoutMode(NetEqPlayoutMode mode) = 0;
227 :
228 : // Returns the current playout mode.
229 : // Deprecated.
230 : // TODO(henrik.lundin) Delete.
231 : virtual NetEqPlayoutMode PlayoutMode() const = 0;
232 :
233 : // Writes the current network statistics to |stats|. The statistics are reset
234 : // after the call.
235 : virtual int NetworkStatistics(NetEqNetworkStatistics* stats) = 0;
236 :
237 : // Writes the current RTCP statistics to |stats|. The statistics are reset
238 : // and a new report period is started with the call.
239 : virtual void GetRtcpStatistics(RtcpStatistics* stats) = 0;
240 :
241 : // Same as RtcpStatistics(), but does not reset anything.
242 : virtual void GetRtcpStatisticsNoReset(RtcpStatistics* stats) = 0;
243 :
244 : // Enables post-decode VAD. When enabled, GetAudio() will return
245 : // kOutputVADPassive when the signal contains no speech.
246 : virtual void EnableVad() = 0;
247 :
248 : // Disables post-decode VAD.
249 : virtual void DisableVad() = 0;
250 :
251 : // Returns the RTP timestamp for the last sample delivered by GetAudio().
252 : // The return value will be empty if no valid timestamp is available.
253 : virtual rtc::Optional<uint32_t> GetPlayoutTimestamp() const = 0;
254 :
255 : // Returns the sample rate in Hz of the audio produced in the last GetAudio
256 : // call. If GetAudio has not been called yet, the configured sample rate
257 : // (Config::sample_rate_hz) is returned.
258 : virtual int last_output_sample_rate_hz() const = 0;
259 :
260 : // Returns info about the decoder for the given payload type, or an empty
261 : // value if we have no decoder for that payload type.
262 : virtual rtc::Optional<CodecInst> GetDecoder(int payload_type) const = 0;
263 :
264 : // Returns the decoder format for the given payload type. Returns empty if no
265 : // such payload type was registered.
266 : virtual rtc::Optional<SdpAudioFormat> GetDecoderFormat(
267 : int payload_type) const = 0;
268 :
269 : // Not implemented.
270 : virtual int SetTargetNumberOfChannels() = 0;
271 :
272 : // Not implemented.
273 : virtual int SetTargetSampleRate() = 0;
274 :
275 : // Returns the error code for the last occurred error. If no error has
276 : // occurred, 0 is returned.
277 : virtual int LastError() const = 0;
278 :
279 : // Returns the error code last returned by a decoder (audio or comfort noise).
280 : // When LastError() returns kDecoderErrorCode or kComfortNoiseErrorCode, check
281 : // this method to get the decoder's error code.
282 : virtual int LastDecoderError() = 0;
283 :
284 : // Flushes both the packet buffer and the sync buffer.
285 : virtual void FlushBuffers() = 0;
286 :
287 : // Current usage of packet-buffer and it's limits.
288 : virtual void PacketBufferStatistics(int* current_num_packets,
289 : int* max_num_packets) const = 0;
290 :
291 : // Enables NACK and sets the maximum size of the NACK list, which should be
292 : // positive and no larger than Nack::kNackListSizeLimit. If NACK is already
293 : // enabled then the maximum NACK list size is modified accordingly.
294 : virtual void EnableNack(size_t max_nack_list_size) = 0;
295 :
296 : virtual void DisableNack() = 0;
297 :
298 : // Returns a list of RTP sequence numbers corresponding to packets to be
299 : // retransmitted, given an estimate of the round-trip time in milliseconds.
300 : virtual std::vector<uint16_t> GetNackList(
301 : int64_t round_trip_time_ms) const = 0;
302 :
303 : protected:
304 0 : NetEq() {}
305 :
306 : private:
307 : RTC_DISALLOW_COPY_AND_ASSIGN(NetEq);
308 : };
309 :
310 : } // namespace webrtc
311 : #endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_INCLUDE_NETEQ_H_
|
