Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/audio/utility/audio_frame_operations.h"
12 : #include "webrtc/modules/audio_conference_mixer/include/audio_conference_mixer_defines.h"
13 : #include "webrtc/modules/audio_conference_mixer/source/audio_conference_mixer_impl.h"
14 : #include "webrtc/modules/audio_conference_mixer/source/audio_frame_manipulator.h"
15 : #include "webrtc/modules/audio_processing/include/audio_processing.h"
16 : #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
17 : #include "webrtc/system_wrappers/include/trace.h"
18 :
19 : namespace webrtc {
20 : namespace {
21 :
22 : struct ParticipantFrameStruct {
23 0 : ParticipantFrameStruct(MixerParticipant* p, AudioFrame* a, bool m)
24 0 : : participant(p), audioFrame(a), muted(m) {}
25 : MixerParticipant* participant;
26 : AudioFrame* audioFrame;
27 : bool muted;
28 : };
29 :
30 : typedef std::list<ParticipantFrameStruct*> ParticipantFrameStructList;
31 :
32 : // Mix |frame| into |mixed_frame|, with saturation protection and upmixing.
33 : // These effects are applied to |frame| itself prior to mixing. Assumes that
34 : // |mixed_frame| always has at least as many channels as |frame|. Supports
35 : // stereo at most.
36 : //
37 : // TODO(andrew): consider not modifying |frame| here.
38 0 : void MixFrames(AudioFrame* mixed_frame, AudioFrame* frame, bool use_limiter) {
39 0 : assert(mixed_frame->num_channels_ >= frame->num_channels_);
40 0 : if (use_limiter) {
41 : // This is to avoid saturation in the mixing. It is only
42 : // meaningful if the limiter will be used.
43 0 : AudioFrameOperations::ApplyHalfGain(frame);
44 : }
45 0 : if (mixed_frame->num_channels_ > frame->num_channels_) {
46 : // We only support mono-to-stereo.
47 0 : assert(mixed_frame->num_channels_ == 2 &&
48 : frame->num_channels_ == 1);
49 0 : AudioFrameOperations::MonoToStereo(frame);
50 : }
51 :
52 0 : AudioFrameOperations::Add(*frame, mixed_frame);
53 0 : }
54 :
55 : // Return the max number of channels from a |list| composed of AudioFrames.
56 0 : size_t MaxNumChannels(const AudioFrameList* list) {
57 0 : size_t max_num_channels = 1;
58 0 : for (AudioFrameList::const_iterator iter = list->begin();
59 0 : iter != list->end();
60 : ++iter) {
61 0 : max_num_channels = std::max(max_num_channels, (*iter).frame->num_channels_);
62 : }
63 0 : return max_num_channels;
64 : }
65 :
66 : } // namespace
67 :
68 0 : MixerParticipant::MixerParticipant()
69 0 : : _mixHistory(new MixHistory()) {
70 0 : }
71 :
72 0 : MixerParticipant::~MixerParticipant() {
73 0 : delete _mixHistory;
74 0 : }
75 :
76 0 : bool MixerParticipant::IsMixed() const {
77 0 : return _mixHistory->IsMixed();
78 : }
79 :
80 0 : MixHistory::MixHistory()
81 0 : : _isMixed(0) {
82 0 : }
83 :
84 0 : MixHistory::~MixHistory() {
85 0 : }
86 :
87 0 : bool MixHistory::IsMixed() const {
88 0 : return _isMixed;
89 : }
90 :
91 0 : bool MixHistory::WasMixed() const {
92 : // Was mixed is the same as is mixed depending on perspective. This function
93 : // is for the perspective of AudioConferenceMixerImpl.
94 0 : return IsMixed();
95 : }
96 :
97 0 : int32_t MixHistory::SetIsMixed(const bool mixed) {
98 0 : _isMixed = mixed;
99 0 : return 0;
100 : }
101 :
102 0 : void MixHistory::ResetMixedStatus() {
103 0 : _isMixed = false;
104 0 : }
105 :
106 0 : AudioConferenceMixer* AudioConferenceMixer::Create(int id) {
107 0 : AudioConferenceMixerImpl* mixer = new AudioConferenceMixerImpl(id);
108 0 : if(!mixer->Init()) {
109 0 : delete mixer;
110 0 : return NULL;
111 : }
112 0 : return mixer;
113 : }
114 :
115 0 : AudioConferenceMixerImpl::AudioConferenceMixerImpl(int id)
116 : : _id(id),
117 : _minimumMixingFreq(kLowestPossible),
118 : _mixReceiver(NULL),
119 : _outputFrequency(kDefaultFrequency),
120 : _sampleSize(0),
121 : _audioFramePool(NULL),
122 : _participantList(),
123 : _additionalParticipantList(),
124 : _numMixedParticipants(0),
125 : use_limiter_(true),
126 : _timeStamp(0),
127 : _timeScheduler(kProcessPeriodicityInMs),
128 0 : _processCalls(0) {}
129 :
130 0 : bool AudioConferenceMixerImpl::Init() {
131 0 : _crit.reset(CriticalSectionWrapper::CreateCriticalSection());
132 0 : if (_crit.get() == NULL)
133 0 : return false;
134 :
135 0 : _cbCrit.reset(CriticalSectionWrapper::CreateCriticalSection());
136 0 : if(_cbCrit.get() == NULL)
137 0 : return false;
138 :
139 0 : Config config;
140 0 : config.Set<ExperimentalAgc>(new ExperimentalAgc(false));
141 0 : _limiter.reset(AudioProcessing::Create(config));
142 0 : if(!_limiter.get())
143 0 : return false;
144 :
145 0 : MemoryPool<AudioFrame>::CreateMemoryPool(_audioFramePool,
146 0 : DEFAULT_AUDIO_FRAME_POOLSIZE);
147 0 : if(_audioFramePool == NULL)
148 0 : return false;
149 :
150 0 : if(SetOutputFrequency(kDefaultFrequency) == -1)
151 0 : return false;
152 :
153 0 : if(_limiter->gain_control()->set_mode(GainControl::kFixedDigital) !=
154 0 : _limiter->kNoError)
155 0 : return false;
156 :
157 : // We smoothly limit the mixed frame to -7 dbFS. -6 would correspond to the
158 : // divide-by-2 but -7 is used instead to give a bit of headroom since the
159 : // AGC is not a hard limiter.
160 0 : if(_limiter->gain_control()->set_target_level_dbfs(7) != _limiter->kNoError)
161 0 : return false;
162 :
163 0 : if(_limiter->gain_control()->set_compression_gain_db(0)
164 0 : != _limiter->kNoError)
165 0 : return false;
166 :
167 0 : if(_limiter->gain_control()->enable_limiter(true) != _limiter->kNoError)
168 0 : return false;
169 :
170 0 : if(_limiter->gain_control()->Enable(true) != _limiter->kNoError)
171 0 : return false;
172 :
173 0 : return true;
174 : }
175 :
176 0 : AudioConferenceMixerImpl::~AudioConferenceMixerImpl() {
177 0 : MemoryPool<AudioFrame>::DeleteMemoryPool(_audioFramePool);
178 0 : assert(_audioFramePool == NULL);
179 0 : }
180 :
181 : // Process should be called every kProcessPeriodicityInMs ms
182 0 : int64_t AudioConferenceMixerImpl::TimeUntilNextProcess() {
183 0 : int64_t timeUntilNextProcess = 0;
184 0 : CriticalSectionScoped cs(_crit.get());
185 0 : if(_timeScheduler.TimeToNextUpdate(timeUntilNextProcess) != 0) {
186 : WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
187 : "failed in TimeToNextUpdate() call");
188 : // Sanity check
189 0 : assert(false);
190 : return -1;
191 : }
192 0 : return timeUntilNextProcess;
193 : }
194 :
195 0 : void AudioConferenceMixerImpl::Process() {
196 : size_t remainingParticipantsAllowedToMix =
197 0 : kMaximumAmountOfMixedParticipants;
198 : {
199 0 : CriticalSectionScoped cs(_crit.get());
200 0 : assert(_processCalls == 0);
201 0 : _processCalls++;
202 :
203 : // Let the scheduler know that we are running one iteration.
204 0 : _timeScheduler.UpdateScheduler();
205 : }
206 :
207 0 : AudioFrameList mixList;
208 0 : AudioFrameList rampOutList;
209 0 : AudioFrameList additionalFramesList;
210 0 : std::map<int, MixerParticipant*> mixedParticipantsMap;
211 : {
212 0 : CriticalSectionScoped cs(_cbCrit.get());
213 :
214 0 : int32_t lowFreq = GetLowestMixingFrequency();
215 : // SILK can run in 12 kHz and 24 kHz. These frequencies are not
216 : // supported so use the closest higher frequency to not lose any
217 : // information.
218 : // TODO(henrike): this is probably more appropriate to do in
219 : // GetLowestMixingFrequency().
220 0 : if (lowFreq == 12000) {
221 0 : lowFreq = 16000;
222 0 : } else if (lowFreq == 24000) {
223 0 : lowFreq = 32000;
224 : }
225 0 : if(lowFreq <= 0) {
226 0 : CriticalSectionScoped cs(_crit.get());
227 0 : _processCalls--;
228 0 : return;
229 : } else {
230 0 : switch(lowFreq) {
231 : case 8000:
232 0 : if(OutputFrequency() != kNbInHz) {
233 0 : SetOutputFrequency(kNbInHz);
234 : }
235 0 : break;
236 : case 16000:
237 0 : if(OutputFrequency() != kWbInHz) {
238 0 : SetOutputFrequency(kWbInHz);
239 : }
240 0 : break;
241 : case 32000:
242 0 : if(OutputFrequency() != kSwbInHz) {
243 0 : SetOutputFrequency(kSwbInHz);
244 : }
245 0 : break;
246 : case 48000:
247 0 : if(OutputFrequency() != kFbInHz) {
248 0 : SetOutputFrequency(kFbInHz);
249 : }
250 0 : break;
251 : default:
252 0 : assert(false);
253 :
254 : CriticalSectionScoped cs(_crit.get());
255 : _processCalls--;
256 : return;
257 : }
258 : }
259 :
260 : UpdateToMix(&mixList, &rampOutList, &mixedParticipantsMap,
261 0 : &remainingParticipantsAllowedToMix);
262 :
263 0 : GetAdditionalAudio(&additionalFramesList);
264 0 : UpdateMixedStatus(mixedParticipantsMap);
265 : }
266 :
267 : // Get an AudioFrame for mixing from the memory pool.
268 0 : AudioFrame* mixedAudio = NULL;
269 0 : if(_audioFramePool->PopMemory(mixedAudio) == -1) {
270 : WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
271 : "failed PopMemory() call");
272 0 : assert(false);
273 : return;
274 : }
275 :
276 : {
277 0 : CriticalSectionScoped cs(_crit.get());
278 :
279 : // TODO(henrike): it might be better to decide the number of channels
280 : // with an API instead of dynamically.
281 :
282 : // Find the max channels over all mixing lists.
283 0 : const size_t num_mixed_channels = std::max(MaxNumChannels(&mixList),
284 0 : std::max(MaxNumChannels(&additionalFramesList),
285 0 : MaxNumChannels(&rampOutList)));
286 :
287 0 : mixedAudio->UpdateFrame(-1, _timeStamp, NULL, 0, _outputFrequency,
288 : AudioFrame::kNormalSpeech,
289 0 : AudioFrame::kVadPassive, num_mixed_channels);
290 :
291 0 : _timeStamp += static_cast<uint32_t>(_sampleSize);
292 :
293 : // We only use the limiter if it supports the output sample rate and
294 : // we're actually mixing multiple streams.
295 0 : use_limiter_ =
296 0 : _numMixedParticipants > 1 &&
297 0 : _outputFrequency <= AudioProcessing::kMaxNativeSampleRateHz;
298 :
299 0 : MixFromList(mixedAudio, mixList);
300 0 : MixAnonomouslyFromList(mixedAudio, additionalFramesList);
301 0 : MixAnonomouslyFromList(mixedAudio, rampOutList);
302 :
303 0 : if(mixedAudio->samples_per_channel_ == 0) {
304 : // Nothing was mixed, set the audio samples to silence.
305 0 : mixedAudio->samples_per_channel_ = _sampleSize;
306 0 : AudioFrameOperations::Mute(mixedAudio);
307 : } else {
308 : // Only call the limiter if we have something to mix.
309 0 : LimitMixedAudio(mixedAudio);
310 : }
311 : }
312 :
313 : {
314 0 : CriticalSectionScoped cs(_cbCrit.get());
315 0 : if(_mixReceiver != NULL) {
316 0 : const AudioFrame** dummy = NULL;
317 0 : _mixReceiver->NewMixedAudio(
318 : _id,
319 : *mixedAudio,
320 : dummy,
321 0 : 0);
322 : }
323 : }
324 :
325 : // Reclaim all outstanding memory.
326 0 : _audioFramePool->PushMemory(mixedAudio);
327 0 : ClearAudioFrameList(&mixList);
328 0 : ClearAudioFrameList(&rampOutList);
329 0 : ClearAudioFrameList(&additionalFramesList);
330 : {
331 0 : CriticalSectionScoped cs(_crit.get());
332 0 : _processCalls--;
333 : }
334 0 : return;
335 : }
336 :
337 0 : int32_t AudioConferenceMixerImpl::RegisterMixedStreamCallback(
338 : AudioMixerOutputReceiver* mixReceiver) {
339 0 : CriticalSectionScoped cs(_cbCrit.get());
340 0 : if(_mixReceiver != NULL) {
341 0 : return -1;
342 : }
343 0 : _mixReceiver = mixReceiver;
344 0 : return 0;
345 : }
346 :
347 0 : int32_t AudioConferenceMixerImpl::UnRegisterMixedStreamCallback() {
348 0 : CriticalSectionScoped cs(_cbCrit.get());
349 0 : if(_mixReceiver == NULL) {
350 0 : return -1;
351 : }
352 0 : _mixReceiver = NULL;
353 0 : return 0;
354 : }
355 :
356 0 : int32_t AudioConferenceMixerImpl::SetOutputFrequency(
357 : const Frequency& frequency) {
358 0 : CriticalSectionScoped cs(_crit.get());
359 :
360 0 : _outputFrequency = frequency;
361 0 : _sampleSize =
362 0 : static_cast<size_t>((_outputFrequency*kProcessPeriodicityInMs) / 1000);
363 :
364 0 : return 0;
365 : }
366 :
367 : AudioConferenceMixer::Frequency
368 0 : AudioConferenceMixerImpl::OutputFrequency() const {
369 0 : CriticalSectionScoped cs(_crit.get());
370 0 : return _outputFrequency;
371 : }
372 :
373 0 : int32_t AudioConferenceMixerImpl::SetMixabilityStatus(
374 : MixerParticipant* participant, bool mixable) {
375 0 : if (!mixable) {
376 : // Anonymous participants are in a separate list. Make sure that the
377 : // participant is in the _participantList if it is being mixed.
378 0 : SetAnonymousMixabilityStatus(participant, false);
379 : }
380 : size_t numMixedParticipants;
381 : {
382 0 : CriticalSectionScoped cs(_cbCrit.get());
383 : const bool isMixed =
384 0 : IsParticipantInList(*participant, _participantList);
385 : // API must be called with a new state.
386 0 : if(!(mixable ^ isMixed)) {
387 : WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
388 : "Mixable is aready %s",
389 : isMixed ? "ON" : "off");
390 0 : return -1;
391 : }
392 0 : bool success = false;
393 0 : if(mixable) {
394 0 : success = AddParticipantToList(participant, &_participantList);
395 : } else {
396 0 : success = RemoveParticipantFromList(participant, &_participantList);
397 : }
398 0 : if(!success) {
399 : WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
400 : "failed to %s participant",
401 : mixable ? "add" : "remove");
402 0 : assert(false);
403 : return -1;
404 : }
405 :
406 0 : size_t numMixedNonAnonymous = _participantList.size();
407 0 : if (numMixedNonAnonymous > kMaximumAmountOfMixedParticipants) {
408 0 : numMixedNonAnonymous = kMaximumAmountOfMixedParticipants;
409 : }
410 0 : numMixedParticipants =
411 0 : numMixedNonAnonymous + _additionalParticipantList.size();
412 : }
413 : // A MixerParticipant was added or removed. Make sure the scratch
414 : // buffer is updated if necessary.
415 : // Note: The scratch buffer may only be updated in Process().
416 0 : CriticalSectionScoped cs(_crit.get());
417 0 : _numMixedParticipants = numMixedParticipants;
418 0 : return 0;
419 : }
420 :
421 0 : bool AudioConferenceMixerImpl::MixabilityStatus(
422 : const MixerParticipant& participant) const {
423 0 : CriticalSectionScoped cs(_cbCrit.get());
424 0 : return IsParticipantInList(participant, _participantList);
425 : }
426 :
427 0 : int32_t AudioConferenceMixerImpl::SetAnonymousMixabilityStatus(
428 : MixerParticipant* participant, bool anonymous) {
429 0 : CriticalSectionScoped cs(_cbCrit.get());
430 0 : if(IsParticipantInList(*participant, _additionalParticipantList)) {
431 0 : if(anonymous) {
432 0 : return 0;
433 : }
434 0 : if(!RemoveParticipantFromList(participant,
435 : &_additionalParticipantList)) {
436 : WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
437 : "unable to remove participant from anonymous list");
438 0 : assert(false);
439 : return -1;
440 : }
441 0 : return AddParticipantToList(participant, &_participantList) ? 0 : -1;
442 : }
443 0 : if(!anonymous) {
444 0 : return 0;
445 : }
446 0 : const bool mixable = RemoveParticipantFromList(participant,
447 0 : &_participantList);
448 0 : if(!mixable) {
449 : WEBRTC_TRACE(
450 : kTraceWarning,
451 : kTraceAudioMixerServer,
452 : _id,
453 : "participant must be registered before turning it into anonymous");
454 : // Setting anonymous status is only possible if MixerParticipant is
455 : // already registered.
456 0 : return -1;
457 : }
458 0 : return AddParticipantToList(participant, &_additionalParticipantList) ?
459 0 : 0 : -1;
460 : }
461 :
462 0 : bool AudioConferenceMixerImpl::AnonymousMixabilityStatus(
463 : const MixerParticipant& participant) const {
464 0 : CriticalSectionScoped cs(_cbCrit.get());
465 0 : return IsParticipantInList(participant, _additionalParticipantList);
466 : }
467 :
468 0 : int32_t AudioConferenceMixerImpl::SetMinimumMixingFrequency(
469 : Frequency freq) {
470 : // Make sure that only allowed sampling frequencies are used. Use closest
471 : // higher sampling frequency to avoid losing information.
472 0 : if (static_cast<int>(freq) == 12000) {
473 0 : freq = kWbInHz;
474 0 : } else if (static_cast<int>(freq) == 24000) {
475 0 : freq = kSwbInHz;
476 : }
477 :
478 0 : if((freq == kNbInHz) || (freq == kWbInHz) || (freq == kSwbInHz) ||
479 : (freq == kLowestPossible)) {
480 0 : _minimumMixingFreq=freq;
481 0 : return 0;
482 : } else {
483 : WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
484 : "SetMinimumMixingFrequency incorrect frequency: %i",freq);
485 0 : assert(false);
486 : return -1;
487 : }
488 : }
489 :
490 : // Check all AudioFrames that are to be mixed. The highest sampling frequency
491 : // found is the lowest that can be used without losing information.
492 0 : int32_t AudioConferenceMixerImpl::GetLowestMixingFrequency() const {
493 : const int participantListFrequency =
494 0 : GetLowestMixingFrequencyFromList(_participantList);
495 : const int anonymousListFrequency =
496 0 : GetLowestMixingFrequencyFromList(_additionalParticipantList);
497 : const int highestFreq =
498 0 : (participantListFrequency > anonymousListFrequency) ?
499 0 : participantListFrequency : anonymousListFrequency;
500 : // Check if the user specified a lowest mixing frequency.
501 0 : if(_minimumMixingFreq != kLowestPossible) {
502 0 : if(_minimumMixingFreq > highestFreq) {
503 0 : return _minimumMixingFreq;
504 : }
505 : }
506 0 : return highestFreq;
507 : }
508 :
509 0 : int32_t AudioConferenceMixerImpl::GetLowestMixingFrequencyFromList(
510 : const MixerParticipantList& mixList) const {
511 0 : int32_t highestFreq = 8000;
512 0 : for (MixerParticipantList::const_iterator iter = mixList.begin();
513 0 : iter != mixList.end();
514 : ++iter) {
515 0 : const int32_t neededFrequency = (*iter)->NeededFrequency(_id);
516 0 : if(neededFrequency > highestFreq) {
517 0 : highestFreq = neededFrequency;
518 : }
519 : }
520 0 : return highestFreq;
521 : }
522 :
523 0 : void AudioConferenceMixerImpl::UpdateToMix(
524 : AudioFrameList* mixList,
525 : AudioFrameList* rampOutList,
526 : std::map<int, MixerParticipant*>* mixParticipantList,
527 : size_t* maxAudioFrameCounter) const {
528 : WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
529 : "UpdateToMix(mixList,rampOutList,mixParticipantList,%d)",
530 : *maxAudioFrameCounter);
531 0 : const size_t mixListStartSize = mixList->size();
532 0 : AudioFrameList activeList;
533 : // Struct needed by the passive lists to keep track of which AudioFrame
534 : // belongs to which MixerParticipant.
535 0 : ParticipantFrameStructList passiveWasNotMixedList;
536 0 : ParticipantFrameStructList passiveWasMixedList;
537 0 : for (MixerParticipantList::const_iterator participant =
538 0 : _participantList.begin(); participant != _participantList.end();
539 : ++participant) {
540 : // Stop keeping track of passive participants if there are already
541 : // enough participants available (they wont be mixed anyway).
542 0 : bool mustAddToPassiveList = (*maxAudioFrameCounter >
543 0 : (activeList.size() +
544 0 : passiveWasMixedList.size() +
545 0 : passiveWasNotMixedList.size()));
546 :
547 0 : bool wasMixed = false;
548 0 : wasMixed = (*participant)->_mixHistory->WasMixed();
549 0 : AudioFrame* audioFrame = NULL;
550 0 : if(_audioFramePool->PopMemory(audioFrame) == -1) {
551 : WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
552 : "failed PopMemory() call");
553 0 : assert(false);
554 : return;
555 : }
556 0 : audioFrame->sample_rate_hz_ = _outputFrequency;
557 :
558 0 : auto ret = (*participant)->GetAudioFrameWithMuted(_id, audioFrame);
559 0 : if (ret == MixerParticipant::AudioFrameInfo::kError) {
560 : WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
561 : "failed to GetAudioFrameWithMuted() from participant");
562 0 : _audioFramePool->PushMemory(audioFrame);
563 0 : continue;
564 : }
565 0 : const bool muted = (ret == MixerParticipant::AudioFrameInfo::kMuted);
566 0 : if (_participantList.size() != 1) {
567 : // TODO(wu): Issue 3390, add support for multiple participants case.
568 0 : audioFrame->ntp_time_ms_ = -1;
569 : }
570 :
571 : // TODO(henrike): this assert triggers in some test cases where SRTP is
572 : // used which prevents NetEQ from making a VAD. Temporarily disable this
573 : // assert until the problem is fixed on a higher level.
574 : // assert(audioFrame->vad_activity_ != AudioFrame::kVadUnknown);
575 0 : if (audioFrame->vad_activity_ == AudioFrame::kVadUnknown) {
576 : WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
577 : "invalid VAD state from participant");
578 : }
579 :
580 0 : if(audioFrame->vad_activity_ == AudioFrame::kVadActive) {
581 0 : if(!wasMixed && !muted) {
582 0 : RampIn(*audioFrame);
583 : }
584 :
585 0 : if(activeList.size() >= *maxAudioFrameCounter) {
586 : // There are already more active participants than should be
587 : // mixed. Only keep the ones with the highest energy.
588 0 : AudioFrameList::iterator replaceItem;
589 : uint32_t lowestEnergy =
590 0 : muted ? 0 : CalculateEnergy(*audioFrame);
591 :
592 0 : bool found_replace_item = false;
593 0 : for (AudioFrameList::iterator iter = activeList.begin();
594 0 : iter != activeList.end();
595 : ++iter) {
596 : const uint32_t energy =
597 0 : muted ? 0 : CalculateEnergy(*iter->frame);
598 0 : if(energy < lowestEnergy) {
599 0 : replaceItem = iter;
600 0 : lowestEnergy = energy;
601 0 : found_replace_item = true;
602 : }
603 : }
604 0 : if(found_replace_item) {
605 0 : RTC_DCHECK(!muted); // Cannot replace with a muted frame.
606 0 : FrameAndMuteInfo replaceFrame = *replaceItem;
607 :
608 0 : bool replaceWasMixed = false;
609 : std::map<int, MixerParticipant*>::const_iterator it =
610 0 : mixParticipantList->find(replaceFrame.frame->id_);
611 :
612 : // When a frame is pushed to |activeList| it is also pushed
613 : // to mixParticipantList with the frame's id. This means
614 : // that the Find call above should never fail.
615 0 : assert(it != mixParticipantList->end());
616 0 : replaceWasMixed = it->second->_mixHistory->WasMixed();
617 :
618 0 : mixParticipantList->erase(replaceFrame.frame->id_);
619 0 : activeList.erase(replaceItem);
620 :
621 0 : activeList.push_front(FrameAndMuteInfo(audioFrame, muted));
622 0 : (*mixParticipantList)[audioFrame->id_] = *participant;
623 0 : assert(mixParticipantList->size() <=
624 0 : kMaximumAmountOfMixedParticipants);
625 :
626 0 : if (replaceWasMixed) {
627 0 : if (!replaceFrame.muted) {
628 0 : RampOut(*replaceFrame.frame);
629 : }
630 0 : rampOutList->push_back(replaceFrame);
631 0 : assert(rampOutList->size() <=
632 0 : kMaximumAmountOfMixedParticipants);
633 : } else {
634 0 : _audioFramePool->PushMemory(replaceFrame.frame);
635 : }
636 : } else {
637 0 : if(wasMixed) {
638 0 : if (!muted) {
639 0 : RampOut(*audioFrame);
640 : }
641 0 : rampOutList->push_back(FrameAndMuteInfo(audioFrame,
642 0 : muted));
643 0 : assert(rampOutList->size() <=
644 0 : kMaximumAmountOfMixedParticipants);
645 : } else {
646 0 : _audioFramePool->PushMemory(audioFrame);
647 : }
648 : }
649 : } else {
650 0 : activeList.push_front(FrameAndMuteInfo(audioFrame, muted));
651 0 : (*mixParticipantList)[audioFrame->id_] = *participant;
652 0 : assert(mixParticipantList->size() <=
653 0 : kMaximumAmountOfMixedParticipants);
654 : }
655 : } else {
656 0 : if(wasMixed) {
657 : ParticipantFrameStruct* part_struct =
658 0 : new ParticipantFrameStruct(*participant, audioFrame, muted);
659 0 : passiveWasMixedList.push_back(part_struct);
660 0 : } else if(mustAddToPassiveList) {
661 0 : if (!muted) {
662 0 : RampIn(*audioFrame);
663 : }
664 : ParticipantFrameStruct* part_struct =
665 0 : new ParticipantFrameStruct(*participant, audioFrame, muted);
666 0 : passiveWasNotMixedList.push_back(part_struct);
667 : } else {
668 0 : _audioFramePool->PushMemory(audioFrame);
669 : }
670 : }
671 : }
672 0 : assert(activeList.size() <= *maxAudioFrameCounter);
673 : // At this point it is known which participants should be mixed. Transfer
674 : // this information to this functions output parameters.
675 0 : for (AudioFrameList::const_iterator iter = activeList.begin();
676 0 : iter != activeList.end();
677 : ++iter) {
678 0 : mixList->push_back(*iter);
679 : }
680 0 : activeList.clear();
681 : // Always mix a constant number of AudioFrames. If there aren't enough
682 : // active participants mix passive ones. Starting with those that was mixed
683 : // last iteration.
684 0 : for (ParticipantFrameStructList::const_iterator
685 0 : iter = passiveWasMixedList.begin(); iter != passiveWasMixedList.end();
686 : ++iter) {
687 0 : if(mixList->size() < *maxAudioFrameCounter + mixListStartSize) {
688 0 : mixList->push_back(FrameAndMuteInfo((*iter)->audioFrame,
689 0 : (*iter)->muted));
690 0 : (*mixParticipantList)[(*iter)->audioFrame->id_] =
691 0 : (*iter)->participant;
692 0 : assert(mixParticipantList->size() <=
693 0 : kMaximumAmountOfMixedParticipants);
694 : } else {
695 0 : _audioFramePool->PushMemory((*iter)->audioFrame);
696 : }
697 0 : delete *iter;
698 : }
699 : // And finally the ones that have not been mixed for a while.
700 0 : for (ParticipantFrameStructList::const_iterator iter =
701 0 : passiveWasNotMixedList.begin();
702 0 : iter != passiveWasNotMixedList.end();
703 : ++iter) {
704 0 : if(mixList->size() < *maxAudioFrameCounter + mixListStartSize) {
705 0 : mixList->push_back(FrameAndMuteInfo((*iter)->audioFrame,
706 0 : (*iter)->muted));
707 0 : (*mixParticipantList)[(*iter)->audioFrame->id_] =
708 0 : (*iter)->participant;
709 0 : assert(mixParticipantList->size() <=
710 0 : kMaximumAmountOfMixedParticipants);
711 : } else {
712 0 : _audioFramePool->PushMemory((*iter)->audioFrame);
713 : }
714 0 : delete *iter;
715 : }
716 0 : assert(*maxAudioFrameCounter + mixListStartSize >= mixList->size());
717 0 : *maxAudioFrameCounter += mixListStartSize - mixList->size();
718 : }
719 :
720 0 : void AudioConferenceMixerImpl::GetAdditionalAudio(
721 : AudioFrameList* additionalFramesList) const {
722 : WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
723 : "GetAdditionalAudio(additionalFramesList)");
724 : // The GetAudioFrameWithMuted() callback may result in the participant being
725 : // removed from additionalParticipantList_. If that happens it will
726 : // invalidate any iterators. Create a copy of the participants list such
727 : // that the list of participants can be traversed safely.
728 0 : MixerParticipantList additionalParticipantList;
729 0 : additionalParticipantList.insert(additionalParticipantList.begin(),
730 : _additionalParticipantList.begin(),
731 0 : _additionalParticipantList.end());
732 :
733 0 : for (MixerParticipantList::const_iterator participant =
734 0 : additionalParticipantList.begin();
735 0 : participant != additionalParticipantList.end();
736 : ++participant) {
737 0 : AudioFrame* audioFrame = NULL;
738 0 : if(_audioFramePool->PopMemory(audioFrame) == -1) {
739 : WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
740 : "failed PopMemory() call");
741 0 : assert(false);
742 : return;
743 : }
744 0 : audioFrame->sample_rate_hz_ = _outputFrequency;
745 0 : auto ret = (*participant)->GetAudioFrameWithMuted(_id, audioFrame);
746 0 : if (ret == MixerParticipant::AudioFrameInfo::kError) {
747 : WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
748 : "failed to GetAudioFrameWithMuted() from participant");
749 0 : _audioFramePool->PushMemory(audioFrame);
750 0 : continue;
751 : }
752 0 : if(audioFrame->samples_per_channel_ == 0) {
753 : // Empty frame. Don't use it.
754 0 : _audioFramePool->PushMemory(audioFrame);
755 0 : continue;
756 : }
757 0 : additionalFramesList->push_back(FrameAndMuteInfo(
758 0 : audioFrame, ret == MixerParticipant::AudioFrameInfo::kMuted));
759 : }
760 : }
761 :
762 0 : void AudioConferenceMixerImpl::UpdateMixedStatus(
763 : const std::map<int, MixerParticipant*>& mixedParticipantsMap) const {
764 : WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
765 : "UpdateMixedStatus(mixedParticipantsMap)");
766 0 : assert(mixedParticipantsMap.size() <= kMaximumAmountOfMixedParticipants);
767 :
768 : // Loop through all participants. If they are in the mix map they
769 : // were mixed.
770 0 : for (MixerParticipantList::const_iterator
771 0 : participant =_participantList.begin();
772 0 : participant != _participantList.end();
773 : ++participant) {
774 0 : bool isMixed = false;
775 0 : for (std::map<int, MixerParticipant*>::const_iterator it =
776 0 : mixedParticipantsMap.begin();
777 0 : it != mixedParticipantsMap.end();
778 : ++it) {
779 0 : if (it->second == *participant) {
780 0 : isMixed = true;
781 0 : break;
782 : }
783 : }
784 0 : (*participant)->_mixHistory->SetIsMixed(isMixed);
785 : }
786 0 : }
787 :
788 0 : void AudioConferenceMixerImpl::ClearAudioFrameList(
789 : AudioFrameList* audioFrameList) const {
790 : WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
791 : "ClearAudioFrameList(audioFrameList)");
792 0 : for (AudioFrameList::iterator iter = audioFrameList->begin();
793 0 : iter != audioFrameList->end();
794 : ++iter) {
795 0 : _audioFramePool->PushMemory(iter->frame);
796 : }
797 0 : audioFrameList->clear();
798 0 : }
799 :
800 0 : bool AudioConferenceMixerImpl::IsParticipantInList(
801 : const MixerParticipant& participant,
802 : const MixerParticipantList& participantList) const {
803 : WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
804 : "IsParticipantInList(participant,participantList)");
805 0 : for (MixerParticipantList::const_iterator iter = participantList.begin();
806 0 : iter != participantList.end();
807 : ++iter) {
808 0 : if(&participant == *iter) {
809 0 : return true;
810 : }
811 : }
812 0 : return false;
813 : }
814 :
815 0 : bool AudioConferenceMixerImpl::AddParticipantToList(
816 : MixerParticipant* participant,
817 : MixerParticipantList* participantList) const {
818 : WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
819 : "AddParticipantToList(participant, participantList)");
820 0 : participantList->push_back(participant);
821 : // Make sure that the mixed status is correct for new MixerParticipant.
822 0 : participant->_mixHistory->ResetMixedStatus();
823 0 : return true;
824 : }
825 :
826 0 : bool AudioConferenceMixerImpl::RemoveParticipantFromList(
827 : MixerParticipant* participant,
828 : MixerParticipantList* participantList) const {
829 : WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
830 : "RemoveParticipantFromList(participant, participantList)");
831 0 : for (MixerParticipantList::iterator iter = participantList->begin();
832 0 : iter != participantList->end();
833 : ++iter) {
834 0 : if(*iter == participant) {
835 0 : participantList->erase(iter);
836 : // Participant is no longer mixed, reset to default.
837 0 : participant->_mixHistory->ResetMixedStatus();
838 0 : return true;
839 : }
840 : }
841 0 : return false;
842 : }
843 :
844 0 : int32_t AudioConferenceMixerImpl::MixFromList(
845 : AudioFrame* mixedAudio,
846 : const AudioFrameList& audioFrameList) const {
847 : WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
848 : "MixFromList(mixedAudio, audioFrameList)");
849 0 : if(audioFrameList.empty()) return 0;
850 :
851 0 : uint32_t position = 0;
852 :
853 0 : if (_numMixedParticipants == 1) {
854 0 : mixedAudio->timestamp_ = audioFrameList.front().frame->timestamp_;
855 0 : mixedAudio->elapsed_time_ms_ =
856 0 : audioFrameList.front().frame->elapsed_time_ms_;
857 : } else {
858 : // TODO(wu): Issue 3390.
859 : // Audio frame timestamp is only supported in one channel case.
860 0 : mixedAudio->timestamp_ = 0;
861 0 : mixedAudio->elapsed_time_ms_ = -1;
862 : }
863 :
864 0 : for (AudioFrameList::const_iterator iter = audioFrameList.begin();
865 0 : iter != audioFrameList.end();
866 : ++iter) {
867 0 : if(position >= kMaximumAmountOfMixedParticipants) {
868 : WEBRTC_TRACE(
869 : kTraceMemory,
870 : kTraceAudioMixerServer,
871 : _id,
872 : "Trying to mix more than max amount of mixed participants:%d!",
873 : kMaximumAmountOfMixedParticipants);
874 : // Assert and avoid crash
875 0 : assert(false);
876 : position = 0;
877 : }
878 0 : if (!iter->muted) {
879 0 : MixFrames(mixedAudio, iter->frame, use_limiter_);
880 : }
881 :
882 0 : position++;
883 : }
884 :
885 0 : return 0;
886 : }
887 :
888 : // TODO(andrew): consolidate this function with MixFromList.
889 0 : int32_t AudioConferenceMixerImpl::MixAnonomouslyFromList(
890 : AudioFrame* mixedAudio,
891 : const AudioFrameList& audioFrameList) const {
892 : WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
893 : "MixAnonomouslyFromList(mixedAudio, audioFrameList)");
894 :
895 0 : if(audioFrameList.empty()) return 0;
896 :
897 0 : for (AudioFrameList::const_iterator iter = audioFrameList.begin();
898 0 : iter != audioFrameList.end();
899 : ++iter) {
900 0 : if (!iter->muted) {
901 0 : MixFrames(mixedAudio, iter->frame, use_limiter_);
902 : }
903 : }
904 0 : return 0;
905 : }
906 :
907 0 : bool AudioConferenceMixerImpl::LimitMixedAudio(AudioFrame* mixedAudio) const {
908 0 : if (!use_limiter_) {
909 0 : return true;
910 : }
911 :
912 : // Smoothly limit the mixed frame.
913 0 : const int error = _limiter->ProcessStream(mixedAudio);
914 :
915 : // And now we can safely restore the level. This procedure results in
916 : // some loss of resolution, deemed acceptable.
917 : //
918 : // It's possible to apply the gain in the AGC (with a target level of 0 dbFS
919 : // and compression gain of 6 dB). However, in the transition frame when this
920 : // is enabled (moving from one to two participants) it has the potential to
921 : // create discontinuities in the mixed frame.
922 : //
923 : // Instead we double the frame (with addition since left-shifting a
924 : // negative value is undefined).
925 0 : AudioFrameOperations::Add(*mixedAudio, mixedAudio);
926 :
927 0 : if(error != _limiter->kNoError) {
928 : WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
929 : "Error from AudioProcessing: %d", error);
930 0 : assert(false);
931 : return false;
932 : }
933 0 : return true;
934 : }
935 : } // namespace webrtc
|
