Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_
12 : #define WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_
13 :
14 : #include <memory>
15 : #include <string>
16 :
17 : #include "webrtc/base/constructormagic.h"
18 : #include "webrtc/base/criticalsection.h"
19 : #include "webrtc/base/optional.h"
20 : #include "webrtc/base/thread_annotations.h"
21 : #include "webrtc/modules/audio_coding/neteq/audio_multi_vector.h"
22 : #include "webrtc/modules/audio_coding/neteq/defines.h"
23 : #include "webrtc/modules/audio_coding/neteq/include/neteq.h"
24 : #include "webrtc/modules/audio_coding/neteq/packet.h" // Declare PacketList.
25 : #include "webrtc/modules/audio_coding/neteq/random_vector.h"
26 : #include "webrtc/modules/audio_coding/neteq/rtcp.h"
27 : #include "webrtc/modules/audio_coding/neteq/statistics_calculator.h"
28 : #include "webrtc/modules/audio_coding/neteq/tick_timer.h"
29 : #include "webrtc/typedefs.h"
30 :
31 : namespace webrtc {
32 :
33 : // Forward declarations.
34 : class Accelerate;
35 : class BackgroundNoise;
36 : class BufferLevelFilter;
37 : class ComfortNoise;
38 : class DecisionLogic;
39 : class DecoderDatabase;
40 : class DelayManager;
41 : class DelayPeakDetector;
42 : class DtmfBuffer;
43 : class DtmfToneGenerator;
44 : class Expand;
45 : class Merge;
46 : class NackTracker;
47 : class Normal;
48 : class PacketBuffer;
49 : class RedPayloadSplitter;
50 : class PostDecodeVad;
51 : class PreemptiveExpand;
52 : class RandomVector;
53 : class SyncBuffer;
54 : class TimestampScaler;
55 : struct AccelerateFactory;
56 : struct DtmfEvent;
57 : struct ExpandFactory;
58 : struct PreemptiveExpandFactory;
59 :
60 0 : class NetEqImpl : public webrtc::NetEq {
61 : public:
62 : enum class OutputType {
63 : kNormalSpeech,
64 : kPLC,
65 : kCNG,
66 : kPLCCNG,
67 : kVadPassive
68 : };
69 :
70 0 : struct Dependencies {
71 : // The constructor populates the Dependencies struct with the default
72 : // implementations of the objects. They can all be replaced by the user
73 : // before sending the struct to the NetEqImpl constructor. However, there
74 : // are dependencies between some of the classes inside the struct, so
75 : // swapping out one may make it necessary to re-create another one.
76 : explicit Dependencies(
77 : const NetEq::Config& config,
78 : const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory);
79 : ~Dependencies();
80 :
81 : std::unique_ptr<TickTimer> tick_timer;
82 : std::unique_ptr<BufferLevelFilter> buffer_level_filter;
83 : std::unique_ptr<DecoderDatabase> decoder_database;
84 : std::unique_ptr<DelayPeakDetector> delay_peak_detector;
85 : std::unique_ptr<DelayManager> delay_manager;
86 : std::unique_ptr<DtmfBuffer> dtmf_buffer;
87 : std::unique_ptr<DtmfToneGenerator> dtmf_tone_generator;
88 : std::unique_ptr<PacketBuffer> packet_buffer;
89 : std::unique_ptr<RedPayloadSplitter> red_payload_splitter;
90 : std::unique_ptr<TimestampScaler> timestamp_scaler;
91 : std::unique_ptr<AccelerateFactory> accelerate_factory;
92 : std::unique_ptr<ExpandFactory> expand_factory;
93 : std::unique_ptr<PreemptiveExpandFactory> preemptive_expand_factory;
94 : };
95 :
96 : // Creates a new NetEqImpl object.
97 : NetEqImpl(const NetEq::Config& config,
98 : Dependencies&& deps,
99 : bool create_components = true);
100 :
101 : ~NetEqImpl() override;
102 :
103 : // Inserts a new packet into NetEq. The |receive_timestamp| is an indication
104 : // of the time when the packet was received, and should be measured with
105 : // the same tick rate as the RTP timestamp of the current payload.
106 : // Returns 0 on success, -1 on failure.
107 : int InsertPacket(const WebRtcRTPHeader& rtp_header,
108 : rtc::ArrayView<const uint8_t> payload,
109 : uint32_t receive_timestamp) override;
110 :
111 : int GetAudio(AudioFrame* audio_frame, bool* muted) override;
112 :
113 : int RegisterPayloadType(NetEqDecoder codec,
114 : const std::string& codec_name,
115 : uint8_t rtp_payload_type) override;
116 :
117 : int RegisterExternalDecoder(AudioDecoder* decoder,
118 : NetEqDecoder codec,
119 : const std::string& codec_name,
120 : uint8_t rtp_payload_type) override;
121 :
122 : bool RegisterPayloadType(int rtp_payload_type,
123 : const SdpAudioFormat& audio_format) override;
124 :
125 : // Removes |rtp_payload_type| from the codec database. Returns 0 on success,
126 : // -1 on failure.
127 : int RemovePayloadType(uint8_t rtp_payload_type) override;
128 :
129 : void RemoveAllPayloadTypes() override;
130 :
131 : bool SetMinimumDelay(int delay_ms) override;
132 :
133 : bool SetMaximumDelay(int delay_ms) override;
134 :
135 : int LeastRequiredDelayMs() const override;
136 :
137 : int SetTargetDelay() override;
138 :
139 : int TargetDelay() override;
140 :
141 : int CurrentDelayMs() const override;
142 :
143 : int FilteredCurrentDelayMs() const override;
144 :
145 : // Sets the playout mode to |mode|.
146 : // Deprecated.
147 : // TODO(henrik.lundin) Delete.
148 : void SetPlayoutMode(NetEqPlayoutMode mode) override;
149 :
150 : // Returns the current playout mode.
151 : // Deprecated.
152 : // TODO(henrik.lundin) Delete.
153 : NetEqPlayoutMode PlayoutMode() const override;
154 :
155 : // Writes the current network statistics to |stats|. The statistics are reset
156 : // after the call.
157 : int NetworkStatistics(NetEqNetworkStatistics* stats) override;
158 :
159 : // Writes the current RTCP statistics to |stats|. The statistics are reset
160 : // and a new report period is started with the call.
161 : void GetRtcpStatistics(RtcpStatistics* stats) override;
162 :
163 : // Same as RtcpStatistics(), but does not reset anything.
164 : void GetRtcpStatisticsNoReset(RtcpStatistics* stats) override;
165 :
166 : // Enables post-decode VAD. When enabled, GetAudio() will return
167 : // kOutputVADPassive when the signal contains no speech.
168 : void EnableVad() override;
169 :
170 : // Disables post-decode VAD.
171 : void DisableVad() override;
172 :
173 : rtc::Optional<uint32_t> GetPlayoutTimestamp() const override;
174 :
175 : int last_output_sample_rate_hz() const override;
176 :
177 : rtc::Optional<CodecInst> GetDecoder(int payload_type) const override;
178 :
179 : rtc::Optional<SdpAudioFormat> GetDecoderFormat(
180 : int payload_type) const override;
181 :
182 : int SetTargetNumberOfChannels() override;
183 :
184 : int SetTargetSampleRate() override;
185 :
186 : // Returns the error code for the last occurred error. If no error has
187 : // occurred, 0 is returned.
188 : int LastError() const override;
189 :
190 : // Returns the error code last returned by a decoder (audio or comfort noise).
191 : // When LastError() returns kDecoderErrorCode or kComfortNoiseErrorCode, check
192 : // this method to get the decoder's error code.
193 : int LastDecoderError() override;
194 :
195 : // Flushes both the packet buffer and the sync buffer.
196 : void FlushBuffers() override;
197 :
198 : void PacketBufferStatistics(int* current_num_packets,
199 : int* max_num_packets) const override;
200 :
201 : void EnableNack(size_t max_nack_list_size) override;
202 :
203 : void DisableNack() override;
204 :
205 : std::vector<uint16_t> GetNackList(int64_t round_trip_time_ms) const override;
206 :
207 : // This accessor method is only intended for testing purposes.
208 : const SyncBuffer* sync_buffer_for_test() const;
209 : Operations last_operation_for_test() const;
210 :
211 : protected:
212 : static const int kOutputSizeMs = 10;
213 : static const size_t kMaxFrameSize = 5760; // 120 ms @ 48 kHz.
214 : // TODO(hlundin): Provide a better value for kSyncBufferSize.
215 : // Current value is kMaxFrameSize + 60 ms * 48 kHz, which is enough for
216 : // calculating correlations of current frame against history.
217 : static const size_t kSyncBufferSize = kMaxFrameSize + 60 * 48;
218 :
219 : // Inserts a new packet into NetEq. This is used by the InsertPacket method
220 : // above. Returns 0 on success, otherwise an error code.
221 : // TODO(hlundin): Merge this with InsertPacket above?
222 : int InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
223 : rtc::ArrayView<const uint8_t> payload,
224 : uint32_t receive_timestamp)
225 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
226 :
227 : // Delivers 10 ms of audio data. The data is written to |audio_frame|.
228 : // Returns 0 on success, otherwise an error code.
229 : int GetAudioInternal(AudioFrame* audio_frame, bool* muted)
230 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
231 :
232 : // Provides a decision to the GetAudioInternal method. The decision what to
233 : // do is written to |operation|. Packets to decode are written to
234 : // |packet_list|, and a DTMF event to play is written to |dtmf_event|. When
235 : // DTMF should be played, |play_dtmf| is set to true by the method.
236 : // Returns 0 on success, otherwise an error code.
237 : int GetDecision(Operations* operation,
238 : PacketList* packet_list,
239 : DtmfEvent* dtmf_event,
240 : bool* play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
241 :
242 : // Decodes the speech packets in |packet_list|, and writes the results to
243 : // |decoded_buffer|, which is allocated to hold |decoded_buffer_length|
244 : // elements. The length of the decoded data is written to |decoded_length|.
245 : // The speech type -- speech or (codec-internal) comfort noise -- is written
246 : // to |speech_type|. If |packet_list| contains any SID frames for RFC 3389
247 : // comfort noise, those are not decoded.
248 : int Decode(PacketList* packet_list,
249 : Operations* operation,
250 : int* decoded_length,
251 : AudioDecoder::SpeechType* speech_type)
252 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
253 :
254 : // Sub-method to Decode(). Performs codec internal CNG.
255 : int DecodeCng(AudioDecoder* decoder, int* decoded_length,
256 : AudioDecoder::SpeechType* speech_type)
257 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
258 :
259 : // Sub-method to Decode(). Performs the actual decoding.
260 : int DecodeLoop(PacketList* packet_list,
261 : const Operations& operation,
262 : AudioDecoder* decoder,
263 : int* decoded_length,
264 : AudioDecoder::SpeechType* speech_type)
265 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
266 :
267 : // Sub-method which calls the Normal class to perform the normal operation.
268 : void DoNormal(const int16_t* decoded_buffer,
269 : size_t decoded_length,
270 : AudioDecoder::SpeechType speech_type,
271 : bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
272 :
273 : // Sub-method which calls the Merge class to perform the merge operation.
274 : void DoMerge(int16_t* decoded_buffer,
275 : size_t decoded_length,
276 : AudioDecoder::SpeechType speech_type,
277 : bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
278 :
279 : // Sub-method which calls the Expand class to perform the expand operation.
280 : int DoExpand(bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
281 :
282 : // Sub-method which calls the Accelerate class to perform the accelerate
283 : // operation.
284 : int DoAccelerate(int16_t* decoded_buffer,
285 : size_t decoded_length,
286 : AudioDecoder::SpeechType speech_type,
287 : bool play_dtmf,
288 : bool fast_accelerate) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
289 :
290 : // Sub-method which calls the PreemptiveExpand class to perform the
291 : // preemtive expand operation.
292 : int DoPreemptiveExpand(int16_t* decoded_buffer,
293 : size_t decoded_length,
294 : AudioDecoder::SpeechType speech_type,
295 : bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
296 :
297 : // Sub-method which calls the ComfortNoise class to generate RFC 3389 comfort
298 : // noise. |packet_list| can either contain one SID frame to update the
299 : // noise parameters, or no payload at all, in which case the previously
300 : // received parameters are used.
301 : int DoRfc3389Cng(PacketList* packet_list, bool play_dtmf)
302 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
303 :
304 : // Calls the audio decoder to generate codec-internal comfort noise when
305 : // no packet was received.
306 : void DoCodecInternalCng(const int16_t* decoded_buffer, size_t decoded_length)
307 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
308 :
309 : // Calls the DtmfToneGenerator class to generate DTMF tones.
310 : int DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf)
311 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
312 :
313 : // Produces packet-loss concealment using alternative methods. If the codec
314 : // has an internal PLC, it is called to generate samples. Otherwise, the
315 : // method performs zero-stuffing.
316 : void DoAlternativePlc(bool increase_timestamp)
317 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
318 :
319 : // Overdub DTMF on top of |output|.
320 : int DtmfOverdub(const DtmfEvent& dtmf_event,
321 : size_t num_channels,
322 : int16_t* output) const EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
323 :
324 : // Extracts packets from |packet_buffer_| to produce at least
325 : // |required_samples| samples. The packets are inserted into |packet_list|.
326 : // Returns the number of samples that the packets in the list will produce, or
327 : // -1 in case of an error.
328 : int ExtractPackets(size_t required_samples, PacketList* packet_list)
329 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
330 :
331 : // Resets various variables and objects to new values based on the sample rate
332 : // |fs_hz| and |channels| number audio channels.
333 : void SetSampleRateAndChannels(int fs_hz, size_t channels)
334 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
335 :
336 : // Returns the output type for the audio produced by the latest call to
337 : // GetAudio().
338 : OutputType LastOutputType() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
339 :
340 : // Updates Expand and Merge.
341 : virtual void UpdatePlcComponents(int fs_hz, size_t channels)
342 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
343 :
344 : // Creates DecisionLogic object with the mode given by |playout_mode_|.
345 : virtual void CreateDecisionLogic() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
346 :
347 : rtc::CriticalSection crit_sect_;
348 : const std::unique_ptr<TickTimer> tick_timer_ GUARDED_BY(crit_sect_);
349 : const std::unique_ptr<BufferLevelFilter> buffer_level_filter_
350 : GUARDED_BY(crit_sect_);
351 : const std::unique_ptr<DecoderDatabase> decoder_database_
352 : GUARDED_BY(crit_sect_);
353 : const std::unique_ptr<DelayManager> delay_manager_ GUARDED_BY(crit_sect_);
354 : const std::unique_ptr<DelayPeakDetector> delay_peak_detector_
355 : GUARDED_BY(crit_sect_);
356 : const std::unique_ptr<DtmfBuffer> dtmf_buffer_ GUARDED_BY(crit_sect_);
357 : const std::unique_ptr<DtmfToneGenerator> dtmf_tone_generator_
358 : GUARDED_BY(crit_sect_);
359 : const std::unique_ptr<PacketBuffer> packet_buffer_ GUARDED_BY(crit_sect_);
360 : const std::unique_ptr<RedPayloadSplitter> red_payload_splitter_
361 : GUARDED_BY(crit_sect_);
362 : const std::unique_ptr<TimestampScaler> timestamp_scaler_
363 : GUARDED_BY(crit_sect_);
364 : const std::unique_ptr<PostDecodeVad> vad_ GUARDED_BY(crit_sect_);
365 : const std::unique_ptr<ExpandFactory> expand_factory_ GUARDED_BY(crit_sect_);
366 : const std::unique_ptr<AccelerateFactory> accelerate_factory_
367 : GUARDED_BY(crit_sect_);
368 : const std::unique_ptr<PreemptiveExpandFactory> preemptive_expand_factory_
369 : GUARDED_BY(crit_sect_);
370 :
371 : std::unique_ptr<BackgroundNoise> background_noise_ GUARDED_BY(crit_sect_);
372 : std::unique_ptr<DecisionLogic> decision_logic_ GUARDED_BY(crit_sect_);
373 : std::unique_ptr<AudioMultiVector> algorithm_buffer_ GUARDED_BY(crit_sect_);
374 : std::unique_ptr<SyncBuffer> sync_buffer_ GUARDED_BY(crit_sect_);
375 : std::unique_ptr<Expand> expand_ GUARDED_BY(crit_sect_);
376 : std::unique_ptr<Normal> normal_ GUARDED_BY(crit_sect_);
377 : std::unique_ptr<Merge> merge_ GUARDED_BY(crit_sect_);
378 : std::unique_ptr<Accelerate> accelerate_ GUARDED_BY(crit_sect_);
379 : std::unique_ptr<PreemptiveExpand> preemptive_expand_ GUARDED_BY(crit_sect_);
380 : RandomVector random_vector_ GUARDED_BY(crit_sect_);
381 : std::unique_ptr<ComfortNoise> comfort_noise_ GUARDED_BY(crit_sect_);
382 : Rtcp rtcp_ GUARDED_BY(crit_sect_);
383 : StatisticsCalculator stats_ GUARDED_BY(crit_sect_);
384 : int fs_hz_ GUARDED_BY(crit_sect_);
385 : int fs_mult_ GUARDED_BY(crit_sect_);
386 : int last_output_sample_rate_hz_ GUARDED_BY(crit_sect_);
387 : size_t output_size_samples_ GUARDED_BY(crit_sect_);
388 : size_t decoder_frame_length_ GUARDED_BY(crit_sect_);
389 : Modes last_mode_ GUARDED_BY(crit_sect_);
390 : Operations last_operation_ GUARDED_BY(crit_sect_);
391 : std::unique_ptr<int16_t[]> mute_factor_array_ GUARDED_BY(crit_sect_);
392 : size_t decoded_buffer_length_ GUARDED_BY(crit_sect_);
393 : std::unique_ptr<int16_t[]> decoded_buffer_ GUARDED_BY(crit_sect_);
394 : uint32_t playout_timestamp_ GUARDED_BY(crit_sect_);
395 : bool new_codec_ GUARDED_BY(crit_sect_);
396 : uint32_t timestamp_ GUARDED_BY(crit_sect_);
397 : bool reset_decoder_ GUARDED_BY(crit_sect_);
398 : rtc::Optional<uint8_t> current_rtp_payload_type_ GUARDED_BY(crit_sect_);
399 : rtc::Optional<uint8_t> current_cng_rtp_payload_type_ GUARDED_BY(crit_sect_);
400 : uint32_t ssrc_ GUARDED_BY(crit_sect_);
401 : bool first_packet_ GUARDED_BY(crit_sect_);
402 : int error_code_ GUARDED_BY(crit_sect_); // Store last error code.
403 : int decoder_error_code_ GUARDED_BY(crit_sect_);
404 : const BackgroundNoiseMode background_noise_mode_ GUARDED_BY(crit_sect_);
405 : NetEqPlayoutMode playout_mode_ GUARDED_BY(crit_sect_);
406 : bool enable_fast_accelerate_ GUARDED_BY(crit_sect_);
407 : std::unique_ptr<NackTracker> nack_ GUARDED_BY(crit_sect_);
408 : bool nack_enabled_ GUARDED_BY(crit_sect_);
409 : const bool enable_muted_state_ GUARDED_BY(crit_sect_);
410 : AudioFrame::VADActivity last_vad_activity_ GUARDED_BY(crit_sect_) =
411 : AudioFrame::kVadPassive;
412 : std::unique_ptr<TickTimer::Stopwatch> generated_noise_stopwatch_
413 : GUARDED_BY(crit_sect_);
414 :
415 : private:
416 : RTC_DISALLOW_COPY_AND_ASSIGN(NetEqImpl);
417 : };
418 :
419 : } // namespace webrtc
420 : #endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_
|
