Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
12 : #define WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
13 :
14 : #include <list>
15 : #include <memory>
16 : #include <string>
17 : #include <vector>
18 :
19 : #include "webrtc/base/criticalsection.h"
20 : #include "webrtc/base/function_view.h"
21 : #include "webrtc/base/gtest_prod_util.h"
22 : #include "webrtc/base/ignore_wundef.h"
23 : #include "webrtc/base/swap_queue.h"
24 : #include "webrtc/base/thread_annotations.h"
25 : #include "webrtc/modules/audio_processing/audio_buffer.h"
26 : #include "webrtc/modules/audio_processing/include/audio_processing.h"
27 : #include "webrtc/modules/audio_processing/render_queue_item_verifier.h"
28 : #include "webrtc/modules/audio_processing/rms_level.h"
29 : #include "webrtc/system_wrappers/include/file_wrapper.h"
30 :
31 : #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
32 : // Files generated at build-time by the protobuf compiler.
33 : RTC_PUSH_IGNORING_WUNDEF()
34 : #ifdef WEBRTC_ANDROID_PLATFORM_BUILD
35 : #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h"
36 : #else
37 : #include "webrtc/modules/audio_processing/debug.pb.h"
38 : #endif
39 : RTC_POP_IGNORING_WUNDEF()
40 : #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
41 :
42 : namespace webrtc {
43 :
44 : class AgcManagerDirect;
45 : class AudioConverter;
46 :
47 : class NonlinearBeamformer;
48 :
49 : class AudioProcessingImpl : public AudioProcessing {
50 : public:
51 : // Methods forcing APM to run in a single-threaded manner.
52 : // Acquires both the render and capture locks.
53 : explicit AudioProcessingImpl(const webrtc::Config& config);
54 : // AudioProcessingImpl takes ownership of beamformer.
55 : AudioProcessingImpl(const webrtc::Config& config,
56 : NonlinearBeamformer* beamformer);
57 : ~AudioProcessingImpl() override;
58 : int Initialize() override;
59 : int Initialize(int capture_input_sample_rate_hz,
60 : int capture_output_sample_rate_hz,
61 : int render_sample_rate_hz,
62 : ChannelLayout capture_input_layout,
63 : ChannelLayout capture_output_layout,
64 : ChannelLayout render_input_layout) override;
65 : int Initialize(const ProcessingConfig& processing_config) override;
66 : void ApplyConfig(const AudioProcessing::Config& config) override;
67 : void SetExtraOptions(const webrtc::Config& config) override;
68 : void UpdateHistogramsOnCallEnd() override;
69 : int StartDebugRecording(const char filename[kMaxFilenameSize],
70 : int64_t max_log_size_bytes) override;
71 : int StartDebugRecording(FILE* handle, int64_t max_log_size_bytes) override;
72 : int StartDebugRecording(FILE* handle) override;
73 : int StartDebugRecordingForPlatformFile(rtc::PlatformFile handle) override;
74 : int StopDebugRecording() override;
75 :
76 : // Capture-side exclusive methods possibly running APM in a
77 : // multi-threaded manner. Acquire the capture lock.
78 : int ProcessStream(AudioFrame* frame) override;
79 : int ProcessStream(const float* const* src,
80 : size_t samples_per_channel,
81 : int input_sample_rate_hz,
82 : ChannelLayout input_layout,
83 : int output_sample_rate_hz,
84 : ChannelLayout output_layout,
85 : float* const* dest) override;
86 : int ProcessStream(const float* const* src,
87 : const StreamConfig& input_config,
88 : const StreamConfig& output_config,
89 : float* const* dest) override;
90 : void set_output_will_be_muted(bool muted) override;
91 : int set_stream_delay_ms(int delay) override;
92 : void set_delay_offset_ms(int offset) override;
93 : int delay_offset_ms() const override;
94 : void set_stream_key_pressed(bool key_pressed) override;
95 :
96 : // Render-side exclusive methods possibly running APM in a
97 : // multi-threaded manner. Acquire the render lock.
98 : int ProcessReverseStream(AudioFrame* frame) override;
99 : int AnalyzeReverseStream(const float* const* data,
100 : size_t samples_per_channel,
101 : int sample_rate_hz,
102 : ChannelLayout layout) override;
103 : int ProcessReverseStream(const float* const* src,
104 : const StreamConfig& input_config,
105 : const StreamConfig& output_config,
106 : float* const* dest) override;
107 :
108 : // Methods only accessed from APM submodules or
109 : // from AudioProcessing tests in a single-threaded manner.
110 : // Hence there is no need for locks in these.
111 : int proc_sample_rate_hz() const override;
112 : int proc_split_sample_rate_hz() const override;
113 : size_t num_input_channels() const override;
114 : size_t num_proc_channels() const override;
115 : size_t num_output_channels() const override;
116 : size_t num_reverse_channels() const override;
117 : int stream_delay_ms() const override;
118 : bool was_stream_delay_set() const override
119 : EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
120 :
121 : AudioProcessingStatistics GetStatistics() const override;
122 :
123 : // Methods returning pointers to APM submodules.
124 : // No locks are aquired in those, as those locks
125 : // would offer no protection (the submodules are
126 : // created only once in a single-treaded manner
127 : // during APM creation).
128 : EchoCancellation* echo_cancellation() const override;
129 : EchoControlMobile* echo_control_mobile() const override;
130 : GainControl* gain_control() const override;
131 : // TODO(peah): Deprecate this API call.
132 : HighPassFilter* high_pass_filter() const override;
133 : LevelEstimator* level_estimator() const override;
134 : NoiseSuppression* noise_suppression() const override;
135 : VoiceDetection* voice_detection() const override;
136 :
137 : // TODO(peah): Remove these two methods once the new API allows that.
138 : void MutateConfig(rtc::FunctionView<void(AudioProcessing::Config*)> mutator);
139 : AudioProcessing::Config GetConfig() const;
140 :
141 : protected:
142 : // Overridden in a mock.
143 : virtual int InitializeLocked()
144 : EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
145 :
146 : private:
147 : // TODO(peah): These friend classes should be removed as soon as the new
148 : // parameter setting scheme allows.
149 : FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, DefaultBehavior);
150 : FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, ValidConfigBehavior);
151 : FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, InValidConfigBehavior);
152 : struct ApmPublicSubmodules;
153 : struct ApmPrivateSubmodules;
154 :
155 : // Submodule interface implementations.
156 : std::unique_ptr<HighPassFilter> high_pass_filter_impl_;
157 :
158 : class ApmSubmoduleStates {
159 : public:
160 : ApmSubmoduleStates();
161 : // Updates the submodule state and returns true if it has changed.
162 : bool Update(bool low_cut_filter_enabled,
163 : bool echo_canceller_enabled,
164 : bool mobile_echo_controller_enabled,
165 : bool residual_echo_detector_enabled,
166 : bool noise_suppressor_enabled,
167 : bool intelligibility_enhancer_enabled,
168 : bool beamformer_enabled,
169 : bool adaptive_gain_controller_enabled,
170 : bool level_controller_enabled,
171 : bool echo_canceller3_enabled,
172 : bool voice_activity_detector_enabled,
173 : bool level_estimator_enabled,
174 : bool transient_suppressor_enabled);
175 : bool CaptureMultiBandSubModulesActive() const;
176 : bool CaptureMultiBandProcessingActive() const;
177 : bool RenderMultiBandSubModulesActive() const;
178 : bool RenderMultiBandProcessingActive() const;
179 :
180 : private:
181 : bool low_cut_filter_enabled_ = false;
182 : bool echo_canceller_enabled_ = false;
183 : bool mobile_echo_controller_enabled_ = false;
184 : bool residual_echo_detector_enabled_ = false;
185 : bool noise_suppressor_enabled_ = false;
186 : bool intelligibility_enhancer_enabled_ = false;
187 : bool beamformer_enabled_ = false;
188 : bool adaptive_gain_controller_enabled_ = false;
189 : bool level_controller_enabled_ = false;
190 : bool echo_canceller3_enabled_ = false;
191 : bool level_estimator_enabled_ = false;
192 : bool voice_activity_detector_enabled_ = false;
193 : bool transient_suppressor_enabled_ = false;
194 : bool first_update_ = true;
195 : };
196 :
197 : #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
198 : // State for the debug dump.
199 : struct ApmDebugDumpThreadState {
200 : ApmDebugDumpThreadState();
201 : ~ApmDebugDumpThreadState();
202 : std::unique_ptr<audioproc::Event> event_msg; // Protobuf message.
203 : std::string event_str; // Memory for protobuf serialization.
204 :
205 : // Serialized string of last saved APM configuration.
206 : std::string last_serialized_config;
207 : };
208 :
209 : struct ApmDebugDumpState {
210 : ApmDebugDumpState();
211 : ~ApmDebugDumpState();
212 : // Number of bytes that can still be written to the log before the maximum
213 : // size is reached. A value of <= 0 indicates that no limit is used.
214 : int64_t num_bytes_left_for_log_ = -1;
215 : std::unique_ptr<FileWrapper> debug_file;
216 : ApmDebugDumpThreadState render;
217 : ApmDebugDumpThreadState capture;
218 : };
219 : #endif
220 :
221 : // Method for modifying the formats struct that are called from both
222 : // the render and capture threads. The check for whether modifications
223 : // are needed is done while holding the render lock only, thereby avoiding
224 : // that the capture thread blocks the render thread.
225 : // The struct is modified in a single-threaded manner by holding both the
226 : // render and capture locks.
227 : int MaybeInitialize(const ProcessingConfig& config, bool force_initialization)
228 : EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
229 :
230 : int MaybeInitializeRender(const ProcessingConfig& processing_config)
231 : EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
232 :
233 : int MaybeInitializeCapture(const ProcessingConfig& processing_config,
234 : bool force_initialization)
235 : EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
236 :
237 : // Method for updating the state keeping track of the active submodules.
238 : // Returns a bool indicating whether the state has changed.
239 : bool UpdateActiveSubmoduleStates() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
240 :
241 : // Methods requiring APM running in a single-threaded manner.
242 : // Are called with both the render and capture locks already
243 : // acquired.
244 : void InitializeTransient()
245 : EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
246 : void InitializeBeamformer()
247 : EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
248 : void InitializeIntelligibility()
249 : EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
250 : int InitializeLocked(const ProcessingConfig& config)
251 : EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
252 : void InitializeLevelController() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
253 : void InitializeResidualEchoDetector()
254 : EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
255 : void InitializeLowCutFilter() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
256 : void InitializeEchoCanceller3() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
257 :
258 : void EmptyQueuedRenderAudio();
259 : void AllocateRenderQueue()
260 : EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
261 : void QueueRenderAudio(AudioBuffer* audio)
262 : EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
263 :
264 : // Capture-side exclusive methods possibly running APM in a multi-threaded
265 : // manner that are called with the render lock already acquired.
266 : int ProcessCaptureStreamLocked() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
267 : void MaybeUpdateHistograms() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
268 :
269 : // Render-side exclusive methods possibly running APM in a multi-threaded
270 : // manner that are called with the render lock already acquired.
271 : // TODO(ekm): Remove once all clients updated to new interface.
272 : int AnalyzeReverseStreamLocked(const float* const* src,
273 : const StreamConfig& input_config,
274 : const StreamConfig& output_config)
275 : EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
276 : int ProcessRenderStreamLocked() EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
277 :
278 : // Debug dump methods that are internal and called without locks.
279 : // TODO(peah): Make thread safe.
280 : #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
281 : // TODO(andrew): make this more graceful. Ideally we would split this stuff
282 : // out into a separate class with an "enabled" and "disabled" implementation.
283 : static int WriteMessageToDebugFile(FileWrapper* debug_file,
284 : int64_t* filesize_limit_bytes,
285 : rtc::CriticalSection* crit_debug,
286 : ApmDebugDumpThreadState* debug_state);
287 : int WriteInitMessage() EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
288 :
289 : // Writes Config message. If not |forced|, only writes the current config if
290 : // it is different from the last saved one; if |forced|, writes the config
291 : // regardless of the last saved.
292 : int WriteConfigMessage(bool forced) EXCLUSIVE_LOCKS_REQUIRED(crit_capture_)
293 : EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
294 :
295 : // Critical section.
296 : rtc::CriticalSection crit_debug_;
297 :
298 : // Debug dump state.
299 : ApmDebugDumpState debug_dump_;
300 : #endif
301 :
302 : // Critical sections.
303 : rtc::CriticalSection crit_render_ ACQUIRED_BEFORE(crit_capture_);
304 : rtc::CriticalSection crit_capture_;
305 :
306 : // Struct containing the Config specifying the behavior of APM.
307 : AudioProcessing::Config config_;
308 :
309 : // Class containing information about what submodules are active.
310 : ApmSubmoduleStates submodule_states_;
311 :
312 : // Structs containing the pointers to the submodules.
313 : std::unique_ptr<ApmPublicSubmodules> public_submodules_;
314 : std::unique_ptr<ApmPrivateSubmodules> private_submodules_;
315 :
316 : // State that is written to while holding both the render and capture locks
317 : // but can be read without any lock being held.
318 : // As this is only accessed internally of APM, and all internal methods in APM
319 : // either are holding the render or capture locks, this construct is safe as
320 : // it is not possible to read the variables while writing them.
321 : struct ApmFormatState {
322 0 : ApmFormatState()
323 0 : : // Format of processing streams at input/output call sites.
324 : api_format({{{kSampleRate16kHz, 1, false},
325 : {kSampleRate16kHz, 1, false},
326 : {kSampleRate16kHz, 1, false},
327 : {kSampleRate16kHz, 1, false}}}),
328 0 : render_processing_format(kSampleRate16kHz, 1) {}
329 : ProcessingConfig api_format;
330 : StreamConfig render_processing_format;
331 : } formats_;
332 :
333 : // APM constants.
334 : const struct ApmConstants {
335 0 : ApmConstants(int agc_startup_min_volume,
336 : int agc_clipped_level_min,
337 : bool use_experimental_agc)
338 0 : : // Format of processing streams at input/output call sites.
339 : agc_startup_min_volume(agc_startup_min_volume),
340 : agc_clipped_level_min(agc_clipped_level_min),
341 0 : use_experimental_agc(use_experimental_agc) {}
342 : int agc_startup_min_volume;
343 : int agc_clipped_level_min;
344 : bool use_experimental_agc;
345 : } constants_;
346 :
347 0 : struct ApmCaptureState {
348 : ApmCaptureState(bool transient_suppressor_enabled,
349 : const std::vector<Point>& array_geometry,
350 : SphericalPointf target_direction);
351 : ~ApmCaptureState();
352 : int aec_system_delay_jumps;
353 : int delay_offset_ms;
354 : bool was_stream_delay_set;
355 : int last_stream_delay_ms;
356 : int last_aec_system_delay_ms;
357 : int stream_delay_jumps;
358 : bool output_will_be_muted;
359 : bool key_pressed;
360 : bool transient_suppressor_enabled;
361 : std::vector<Point> array_geometry;
362 : SphericalPointf target_direction;
363 : std::unique_ptr<AudioBuffer> capture_audio;
364 : // Only the rate and samples fields of capture_processing_format_ are used
365 : // because the capture processing number of channels is mutable and is
366 : // tracked by the capture_audio_.
367 : StreamConfig capture_processing_format;
368 : int split_rate;
369 : } capture_ GUARDED_BY(crit_capture_);
370 :
371 : struct ApmCaptureNonLockedState {
372 0 : ApmCaptureNonLockedState(bool beamformer_enabled,
373 : bool intelligibility_enabled)
374 0 : : capture_processing_format(kSampleRate16kHz),
375 : split_rate(kSampleRate16kHz),
376 : stream_delay_ms(0),
377 : beamformer_enabled(beamformer_enabled),
378 0 : intelligibility_enabled(intelligibility_enabled) {}
379 : // Only the rate and samples fields of capture_processing_format_ are used
380 : // because the forward processing number of channels is mutable and is
381 : // tracked by the capture_audio_.
382 : StreamConfig capture_processing_format;
383 : int split_rate;
384 : int stream_delay_ms;
385 : bool beamformer_enabled;
386 : bool intelligibility_enabled;
387 : bool level_controller_enabled = false;
388 : bool echo_canceller3_enabled = false;
389 : } capture_nonlocked_;
390 :
391 0 : struct ApmRenderState {
392 : ApmRenderState();
393 : ~ApmRenderState();
394 : std::unique_ptr<AudioConverter> render_converter;
395 : std::unique_ptr<AudioBuffer> render_audio;
396 : } render_ GUARDED_BY(crit_render_);
397 :
398 : size_t aec_render_queue_element_max_size_ GUARDED_BY(crit_render_)
399 : GUARDED_BY(crit_capture_) = 0;
400 : std::vector<float> aec_render_queue_buffer_ GUARDED_BY(crit_render_);
401 : std::vector<float> aec_capture_queue_buffer_ GUARDED_BY(crit_capture_);
402 :
403 : size_t aecm_render_queue_element_max_size_ GUARDED_BY(crit_render_)
404 : GUARDED_BY(crit_capture_) = 0;
405 : std::vector<int16_t> aecm_render_queue_buffer_ GUARDED_BY(crit_render_);
406 : std::vector<int16_t> aecm_capture_queue_buffer_ GUARDED_BY(crit_capture_);
407 :
408 : size_t agc_render_queue_element_max_size_ GUARDED_BY(crit_render_)
409 : GUARDED_BY(crit_capture_) = 0;
410 : std::vector<int16_t> agc_render_queue_buffer_ GUARDED_BY(crit_render_);
411 : std::vector<int16_t> agc_capture_queue_buffer_ GUARDED_BY(crit_capture_);
412 :
413 : size_t red_render_queue_element_max_size_ GUARDED_BY(crit_render_)
414 : GUARDED_BY(crit_capture_) = 0;
415 : std::vector<float> red_render_queue_buffer_ GUARDED_BY(crit_render_);
416 : std::vector<float> red_capture_queue_buffer_ GUARDED_BY(crit_capture_);
417 :
418 : RmsLevel capture_input_rms_ GUARDED_BY(crit_capture_);
419 : RmsLevel capture_output_rms_ GUARDED_BY(crit_capture_);
420 : int capture_rms_interval_counter_ GUARDED_BY(crit_capture_) = 0;
421 :
422 : // Lock protection not needed.
423 : std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
424 : aec_render_signal_queue_;
425 : std::unique_ptr<
426 : SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
427 : aecm_render_signal_queue_;
428 : std::unique_ptr<
429 : SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
430 : agc_render_signal_queue_;
431 : std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
432 : red_render_signal_queue_;
433 : };
434 :
435 : } // namespace webrtc
436 :
437 : #endif // WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
|
