Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/modules/audio_mixer/audio_mixer_impl.h"
12 :
13 : #include <algorithm>
14 : #include <functional>
15 : #include <iterator>
16 : #include <utility>
17 :
18 : #include "webrtc/audio/utility/audio_frame_operations.h"
19 : #include "webrtc/base/logging.h"
20 : #include "webrtc/modules/audio_mixer/audio_frame_manipulator.h"
21 : #include "webrtc/modules/audio_mixer/default_output_rate_calculator.h"
22 :
23 : namespace webrtc {
24 : namespace {
25 :
26 : struct SourceFrame {
27 0 : SourceFrame(AudioMixerImpl::SourceStatus* source_status,
28 : AudioFrame* audio_frame,
29 : bool muted)
30 0 : : source_status(source_status), audio_frame(audio_frame), muted(muted) {
31 0 : RTC_DCHECK(source_status);
32 0 : RTC_DCHECK(audio_frame);
33 0 : if (!muted) {
34 0 : energy = AudioMixerCalculateEnergy(*audio_frame);
35 : }
36 0 : }
37 :
38 0 : SourceFrame(AudioMixerImpl::SourceStatus* source_status,
39 : AudioFrame* audio_frame,
40 : bool muted,
41 : uint32_t energy)
42 0 : : source_status(source_status),
43 : audio_frame(audio_frame),
44 : muted(muted),
45 0 : energy(energy) {
46 0 : RTC_DCHECK(source_status);
47 0 : RTC_DCHECK(audio_frame);
48 0 : }
49 :
50 : AudioMixerImpl::SourceStatus* source_status = nullptr;
51 : AudioFrame* audio_frame = nullptr;
52 : bool muted = true;
53 : uint32_t energy = 0;
54 : };
55 :
56 : // ShouldMixBefore(a, b) is used to select mixer sources.
57 0 : bool ShouldMixBefore(const SourceFrame& a, const SourceFrame& b) {
58 0 : if (a.muted != b.muted) {
59 0 : return b.muted;
60 : }
61 :
62 0 : const auto a_activity = a.audio_frame->vad_activity_;
63 0 : const auto b_activity = b.audio_frame->vad_activity_;
64 :
65 0 : if (a_activity != b_activity) {
66 0 : return a_activity == AudioFrame::kVadActive;
67 : }
68 :
69 0 : return a.energy > b.energy;
70 : }
71 :
72 0 : void RampAndUpdateGain(
73 : const std::vector<SourceFrame>& mixed_sources_and_frames) {
74 0 : for (const auto& source_frame : mixed_sources_and_frames) {
75 0 : float target_gain = source_frame.source_status->is_mixed ? 1.0f : 0.0f;
76 0 : Ramp(source_frame.source_status->gain, target_gain,
77 0 : source_frame.audio_frame);
78 0 : source_frame.source_status->gain = target_gain;
79 : }
80 0 : }
81 :
82 : // Mix the AudioFrames stored in audioFrameList into mixed_audio.
83 0 : int32_t MixFromList(AudioFrame* mixed_audio,
84 : const AudioFrameList& audio_frame_list,
85 : bool use_limiter) {
86 0 : if (audio_frame_list.empty()) {
87 0 : return 0;
88 : }
89 :
90 0 : if (audio_frame_list.size() == 1) {
91 0 : mixed_audio->timestamp_ = audio_frame_list.front()->timestamp_;
92 0 : mixed_audio->elapsed_time_ms_ = audio_frame_list.front()->elapsed_time_ms_;
93 : } else {
94 : // TODO(wu): Issue 3390.
95 : // Audio frame timestamp is only supported in one channel case.
96 0 : mixed_audio->timestamp_ = 0;
97 0 : mixed_audio->elapsed_time_ms_ = -1;
98 : }
99 :
100 0 : for (const auto& frame : audio_frame_list) {
101 0 : RTC_DCHECK_EQ(mixed_audio->sample_rate_hz_, frame->sample_rate_hz_);
102 0 : RTC_DCHECK_EQ(
103 : frame->samples_per_channel_,
104 : static_cast<size_t>((mixed_audio->sample_rate_hz_ *
105 : webrtc::AudioMixerImpl::kFrameDurationInMs) /
106 0 : 1000));
107 :
108 : // Mix |f.frame| into |mixed_audio|, with saturation protection.
109 : // These effect is applied to |f.frame| itself prior to mixing.
110 0 : if (use_limiter) {
111 : // This is to avoid saturation in the mixing. It is only
112 : // meaningful if the limiter will be used.
113 0 : AudioFrameOperations::ApplyHalfGain(frame);
114 : }
115 0 : RTC_DCHECK_EQ(frame->num_channels_, mixed_audio->num_channels_);
116 0 : AudioFrameOperations::Add(*frame, mixed_audio);
117 : }
118 0 : return 0;
119 : }
120 :
121 0 : AudioMixerImpl::SourceStatusList::const_iterator FindSourceInList(
122 : AudioMixerImpl::Source const* audio_source,
123 : AudioMixerImpl::SourceStatusList const* audio_source_list) {
124 : return std::find_if(
125 : audio_source_list->begin(), audio_source_list->end(),
126 0 : [audio_source](const std::unique_ptr<AudioMixerImpl::SourceStatus>& p) {
127 0 : return p->audio_source == audio_source;
128 0 : });
129 : }
130 :
131 : // TODO(aleloi): remove non-const version when WEBRTC only supports modern STL.
132 0 : AudioMixerImpl::SourceStatusList::iterator FindSourceInList(
133 : AudioMixerImpl::Source const* audio_source,
134 : AudioMixerImpl::SourceStatusList* audio_source_list) {
135 : return std::find_if(
136 : audio_source_list->begin(), audio_source_list->end(),
137 0 : [audio_source](const std::unique_ptr<AudioMixerImpl::SourceStatus>& p) {
138 0 : return p->audio_source == audio_source;
139 0 : });
140 : }
141 :
142 0 : std::unique_ptr<AudioProcessing> CreateLimiter() {
143 0 : Config config;
144 0 : config.Set<ExperimentalAgc>(new ExperimentalAgc(false));
145 0 : std::unique_ptr<AudioProcessing> limiter(AudioProcessing::Create(config));
146 0 : if (!limiter.get()) {
147 0 : return nullptr;
148 : }
149 :
150 0 : if (limiter->gain_control()->set_mode(GainControl::kFixedDigital) !=
151 0 : limiter->kNoError) {
152 0 : return nullptr;
153 : }
154 :
155 : // We smoothly limit the mixed frame to -7 dbFS. -6 would correspond to the
156 : // divide-by-2 but -7 is used instead to give a bit of headroom since the
157 : // AGC is not a hard limiter.
158 0 : if (limiter->gain_control()->set_target_level_dbfs(7) != limiter->kNoError) {
159 0 : return nullptr;
160 : }
161 :
162 0 : if (limiter->gain_control()->set_compression_gain_db(0) !=
163 0 : limiter->kNoError) {
164 0 : return nullptr;
165 : }
166 :
167 0 : if (limiter->gain_control()->enable_limiter(true) != limiter->kNoError) {
168 0 : return nullptr;
169 : }
170 :
171 0 : if (limiter->gain_control()->Enable(true) != limiter->kNoError) {
172 0 : return nullptr;
173 : }
174 0 : return limiter;
175 : }
176 :
177 : } // namespace
178 :
179 0 : AudioMixerImpl::AudioMixerImpl(
180 : std::unique_ptr<AudioProcessing> limiter,
181 0 : std::unique_ptr<OutputRateCalculator> output_rate_calculator)
182 0 : : output_rate_calculator_(std::move(output_rate_calculator)),
183 : output_frequency_(0),
184 : sample_size_(0),
185 : audio_source_list_(),
186 : use_limiter_(true),
187 : time_stamp_(0),
188 0 : limiter_(std::move(limiter)) {}
189 :
190 0 : AudioMixerImpl::~AudioMixerImpl() {}
191 :
192 0 : rtc::scoped_refptr<AudioMixerImpl> AudioMixerImpl::Create() {
193 : return CreateWithOutputRateCalculator(
194 0 : std::unique_ptr<DefaultOutputRateCalculator>(
195 0 : new DefaultOutputRateCalculator()));
196 : }
197 :
198 : rtc::scoped_refptr<AudioMixerImpl>
199 0 : AudioMixerImpl::CreateWithOutputRateCalculator(
200 : std::unique_ptr<OutputRateCalculator> output_rate_calculator) {
201 : return rtc::scoped_refptr<AudioMixerImpl>(
202 : new rtc::RefCountedObject<AudioMixerImpl>(
203 0 : CreateLimiter(), std::move(output_rate_calculator)));
204 : }
205 :
206 0 : void AudioMixerImpl::Mix(size_t number_of_channels,
207 : AudioFrame* audio_frame_for_mixing) {
208 0 : RTC_DCHECK(number_of_channels == 1 || number_of_channels == 2);
209 0 : RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
210 :
211 0 : CalculateOutputFrequency();
212 :
213 0 : AudioFrameList mix_list;
214 : {
215 0 : rtc::CritScope lock(&crit_);
216 0 : mix_list = GetAudioFromSources();
217 :
218 0 : for (const auto& frame : mix_list) {
219 0 : RemixFrame(number_of_channels, frame);
220 : }
221 :
222 0 : audio_frame_for_mixing->UpdateFrame(
223 : -1, time_stamp_, NULL, 0, OutputFrequency(), AudioFrame::kNormalSpeech,
224 0 : AudioFrame::kVadPassive, number_of_channels);
225 :
226 0 : time_stamp_ += static_cast<uint32_t>(sample_size_);
227 :
228 0 : use_limiter_ = mix_list.size() > 1;
229 :
230 : // We only use the limiter if we're actually mixing multiple streams.
231 0 : MixFromList(audio_frame_for_mixing, mix_list, use_limiter_);
232 : }
233 :
234 0 : if (audio_frame_for_mixing->samples_per_channel_ == 0) {
235 : // Nothing was mixed, set the audio samples to silence.
236 0 : audio_frame_for_mixing->samples_per_channel_ = sample_size_;
237 0 : AudioFrameOperations::Mute(audio_frame_for_mixing);
238 : } else {
239 : // Only call the limiter if we have something to mix.
240 0 : LimitMixedAudio(audio_frame_for_mixing);
241 : }
242 :
243 0 : return;
244 : }
245 :
246 0 : void AudioMixerImpl::CalculateOutputFrequency() {
247 0 : RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
248 0 : rtc::CritScope lock(&crit_);
249 :
250 0 : std::vector<int> preferred_rates;
251 : std::transform(audio_source_list_.begin(), audio_source_list_.end(),
252 : std::back_inserter(preferred_rates),
253 0 : [&](std::unique_ptr<SourceStatus>& a) {
254 0 : return a->audio_source->PreferredSampleRate();
255 0 : });
256 :
257 0 : output_frequency_ =
258 0 : output_rate_calculator_->CalculateOutputRate(preferred_rates);
259 0 : sample_size_ = (output_frequency_ * kFrameDurationInMs) / 1000;
260 0 : }
261 :
262 0 : int AudioMixerImpl::OutputFrequency() const {
263 0 : RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
264 0 : return output_frequency_;
265 : }
266 :
267 0 : bool AudioMixerImpl::AddSource(Source* audio_source) {
268 0 : RTC_DCHECK(audio_source);
269 0 : rtc::CritScope lock(&crit_);
270 0 : RTC_DCHECK(FindSourceInList(audio_source, &audio_source_list_) ==
271 : audio_source_list_.end())
272 0 : << "Source already added to mixer";
273 0 : audio_source_list_.emplace_back(new SourceStatus(audio_source, false, 0));
274 0 : return true;
275 : }
276 :
277 0 : void AudioMixerImpl::RemoveSource(Source* audio_source) {
278 0 : RTC_DCHECK(audio_source);
279 0 : rtc::CritScope lock(&crit_);
280 0 : const auto iter = FindSourceInList(audio_source, &audio_source_list_);
281 0 : RTC_DCHECK(iter != audio_source_list_.end()) << "Source not present in mixer";
282 0 : audio_source_list_.erase(iter);
283 0 : }
284 :
285 0 : AudioFrameList AudioMixerImpl::GetAudioFromSources() {
286 0 : RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
287 0 : AudioFrameList result;
288 0 : std::vector<SourceFrame> audio_source_mixing_data_list;
289 0 : std::vector<SourceFrame> ramp_list;
290 :
291 : // Get audio from the audio sources and put it in the SourceFrame vector.
292 0 : for (auto& source_and_status : audio_source_list_) {
293 : const auto audio_frame_info =
294 0 : source_and_status->audio_source->GetAudioFrameWithInfo(
295 0 : OutputFrequency(), &source_and_status->audio_frame);
296 :
297 0 : if (audio_frame_info == Source::AudioFrameInfo::kError) {
298 0 : LOG_F(LS_WARNING) << "failed to GetAudioFrameWithInfo() from source";
299 0 : continue;
300 : }
301 : audio_source_mixing_data_list.emplace_back(
302 0 : source_and_status.get(), &source_and_status->audio_frame,
303 0 : audio_frame_info == Source::AudioFrameInfo::kMuted);
304 : }
305 :
306 : // Sort frames by sorting function.
307 0 : std::sort(audio_source_mixing_data_list.begin(),
308 0 : audio_source_mixing_data_list.end(), ShouldMixBefore);
309 :
310 0 : int max_audio_frame_counter = kMaximumAmountOfMixedAudioSources;
311 :
312 : // Go through list in order and put unmuted frames in result list.
313 0 : for (const auto& p : audio_source_mixing_data_list) {
314 : // Filter muted.
315 0 : if (p.muted) {
316 0 : p.source_status->is_mixed = false;
317 0 : continue;
318 : }
319 :
320 : // Add frame to result vector for mixing.
321 0 : bool is_mixed = false;
322 0 : if (max_audio_frame_counter > 0) {
323 0 : --max_audio_frame_counter;
324 0 : result.push_back(p.audio_frame);
325 0 : ramp_list.emplace_back(p.source_status, p.audio_frame, false, -1);
326 0 : is_mixed = true;
327 : }
328 0 : p.source_status->is_mixed = is_mixed;
329 : }
330 0 : RampAndUpdateGain(ramp_list);
331 0 : return result;
332 : }
333 :
334 :
335 0 : bool AudioMixerImpl::LimitMixedAudio(AudioFrame* mixed_audio) const {
336 0 : RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
337 0 : if (!use_limiter_) {
338 0 : return true;
339 : }
340 :
341 : // Smoothly limit the mixed frame.
342 0 : const int error = limiter_->ProcessStream(mixed_audio);
343 :
344 : // And now we can safely restore the level. This procedure results in
345 : // some loss of resolution, deemed acceptable.
346 : //
347 : // It's possible to apply the gain in the AGC (with a target level of 0 dbFS
348 : // and compression gain of 6 dB). However, in the transition frame when this
349 : // is enabled (moving from one to two audio sources) it has the potential to
350 : // create discontinuities in the mixed frame.
351 : //
352 : // Instead we double the frame (with addition since left-shifting a
353 : // negative value is undefined).
354 0 : AudioFrameOperations::Add(*mixed_audio, mixed_audio);
355 :
356 0 : if (error != limiter_->kNoError) {
357 0 : LOG_F(LS_ERROR) << "Error from AudioProcessing: " << error;
358 0 : RTC_NOTREACHED();
359 0 : return false;
360 : }
361 0 : return true;
362 : }
363 :
364 0 : bool AudioMixerImpl::GetAudioSourceMixabilityStatusForTest(
365 : AudioMixerImpl::Source* audio_source) const {
366 0 : RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
367 0 : rtc::CritScope lock(&crit_);
368 :
369 0 : const auto iter = FindSourceInList(audio_source, &audio_source_list_);
370 0 : if (iter != audio_source_list_.end()) {
371 0 : return (*iter)->is_mixed;
372 : }
373 :
374 0 : LOG(LS_ERROR) << "Audio source unknown";
375 0 : return false;
376 : }
377 : } // namespace webrtc
|
