Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/voice_engine/output_mixer.h"
12 :
13 : #include "webrtc/audio/utility/audio_frame_operations.h"
14 : #include "webrtc/base/format_macros.h"
15 : #include "webrtc/modules/audio_processing/include/audio_processing.h"
16 : #include "webrtc/system_wrappers/include/file_wrapper.h"
17 : #include "webrtc/system_wrappers/include/trace.h"
18 : #include "webrtc/voice_engine/include/voe_external_media.h"
19 : #include "webrtc/voice_engine/statistics.h"
20 : #include "webrtc/voice_engine/utility.h"
21 :
22 : namespace webrtc {
23 : namespace voe {
24 :
25 : void
26 0 : OutputMixer::NewMixedAudio(int32_t id,
27 : const AudioFrame& generalAudioFrame,
28 : const AudioFrame** uniqueAudioFrames,
29 : uint32_t size)
30 : {
31 : WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
32 : "OutputMixer::NewMixedAudio(id=%d, size=%u)", id, size);
33 :
34 0 : _audioFrame.CopyFrom(generalAudioFrame);
35 0 : _audioFrame.id_ = id;
36 0 : }
37 :
38 0 : void OutputMixer::PlayNotification(int32_t id, uint32_t durationMs)
39 : {
40 : WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
41 : "OutputMixer::PlayNotification(id=%d, durationMs=%d)",
42 : id, durationMs);
43 : // Not implement yet
44 0 : }
45 :
46 0 : void OutputMixer::RecordNotification(int32_t id,
47 : uint32_t durationMs)
48 : {
49 : WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
50 : "OutputMixer::RecordNotification(id=%d, durationMs=%d)",
51 : id, durationMs);
52 :
53 : // Not implement yet
54 0 : }
55 :
56 0 : void OutputMixer::PlayFileEnded(int32_t id)
57 : {
58 : WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
59 : "OutputMixer::PlayFileEnded(id=%d)", id);
60 :
61 : // not needed
62 0 : }
63 :
64 0 : void OutputMixer::RecordFileEnded(int32_t id)
65 : {
66 : WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
67 : "OutputMixer::RecordFileEnded(id=%d)", id);
68 0 : assert(id == _instanceId);
69 :
70 0 : rtc::CritScope cs(&_fileCritSect);
71 0 : _outputFileRecording = false;
72 : WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId,-1),
73 : "OutputMixer::RecordFileEnded() =>"
74 : "output file recorder module is shutdown");
75 0 : }
76 :
77 : int32_t
78 0 : OutputMixer::Create(OutputMixer*& mixer, uint32_t instanceId)
79 : {
80 : WEBRTC_TRACE(kTraceMemory, kTraceVoice, instanceId,
81 : "OutputMixer::Create(instanceId=%d)", instanceId);
82 0 : mixer = new OutputMixer(instanceId);
83 0 : if (mixer == NULL)
84 : {
85 : WEBRTC_TRACE(kTraceMemory, kTraceVoice, instanceId,
86 : "OutputMixer::Create() unable to allocate memory for"
87 : "mixer");
88 0 : return -1;
89 : }
90 0 : return 0;
91 : }
92 :
93 0 : OutputMixer::OutputMixer(uint32_t instanceId) :
94 0 : _mixerModule(*AudioConferenceMixer::Create(instanceId)),
95 : _audioLevel(),
96 : _instanceId(instanceId),
97 : _externalMediaCallbackPtr(NULL),
98 : _externalMedia(false),
99 : _panLeft(1.0f),
100 : _panRight(1.0f),
101 : _mixingFrequencyHz(8000),
102 0 : _outputFileRecording(false)
103 : {
104 : WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(_instanceId,-1),
105 : "OutputMixer::OutputMixer() - ctor");
106 :
107 0 : if (_mixerModule.RegisterMixedStreamCallback(this) == -1)
108 : {
109 : WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId,-1),
110 : "OutputMixer::OutputMixer() failed to register mixer"
111 : "callbacks");
112 : }
113 0 : }
114 :
115 : void
116 0 : OutputMixer::Destroy(OutputMixer*& mixer)
117 : {
118 0 : if (mixer)
119 : {
120 0 : delete mixer;
121 0 : mixer = NULL;
122 : }
123 0 : }
124 :
125 0 : OutputMixer::~OutputMixer()
126 : {
127 : WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(_instanceId,-1),
128 : "OutputMixer::~OutputMixer() - dtor");
129 0 : if (_externalMedia)
130 : {
131 0 : DeRegisterExternalMediaProcessing();
132 : }
133 : {
134 0 : rtc::CritScope cs(&_fileCritSect);
135 0 : if (output_file_recorder_) {
136 0 : output_file_recorder_->RegisterModuleFileCallback(NULL);
137 0 : output_file_recorder_->StopRecording();
138 : }
139 : }
140 0 : _mixerModule.UnRegisterMixedStreamCallback();
141 0 : delete &_mixerModule;
142 0 : }
143 :
144 : int32_t
145 0 : OutputMixer::SetEngineInformation(voe::Statistics& engineStatistics)
146 : {
147 : WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
148 : "OutputMixer::SetEngineInformation()");
149 0 : _engineStatisticsPtr = &engineStatistics;
150 0 : return 0;
151 : }
152 :
153 : int32_t
154 0 : OutputMixer::SetAudioProcessingModule(AudioProcessing* audioProcessingModule)
155 : {
156 : WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
157 : "OutputMixer::SetAudioProcessingModule("
158 : "audioProcessingModule=0x%x)", audioProcessingModule);
159 0 : _audioProcessingModulePtr = audioProcessingModule;
160 0 : return 0;
161 : }
162 :
163 0 : int OutputMixer::RegisterExternalMediaProcessing(
164 : VoEMediaProcess& proccess_object)
165 : {
166 : WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
167 : "OutputMixer::RegisterExternalMediaProcessing()");
168 :
169 0 : rtc::CritScope cs(&_callbackCritSect);
170 0 : _externalMediaCallbackPtr = &proccess_object;
171 0 : _externalMedia = true;
172 :
173 0 : return 0;
174 : }
175 :
176 0 : int OutputMixer::DeRegisterExternalMediaProcessing()
177 : {
178 : WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
179 : "OutputMixer::DeRegisterExternalMediaProcessing()");
180 :
181 0 : rtc::CritScope cs(&_callbackCritSect);
182 0 : _externalMedia = false;
183 0 : _externalMediaCallbackPtr = NULL;
184 :
185 0 : return 0;
186 : }
187 :
188 : int32_t
189 0 : OutputMixer::SetMixabilityStatus(MixerParticipant& participant,
190 : bool mixable)
191 : {
192 0 : return _mixerModule.SetMixabilityStatus(&participant, mixable);
193 : }
194 :
195 : int32_t
196 0 : OutputMixer::SetAnonymousMixabilityStatus(MixerParticipant& participant,
197 : bool mixable)
198 : {
199 0 : return _mixerModule.SetAnonymousMixabilityStatus(&participant, mixable);
200 : }
201 :
202 : int32_t
203 0 : OutputMixer::MixActiveChannels()
204 : {
205 0 : _mixerModule.Process();
206 0 : return 0;
207 : }
208 :
209 : int
210 0 : OutputMixer::GetSpeechOutputLevel(uint32_t& level)
211 : {
212 0 : int8_t currentLevel = _audioLevel.Level();
213 0 : level = static_cast<uint32_t> (currentLevel);
214 : WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId,-1),
215 : "GetSpeechOutputLevel() => level=%u", level);
216 0 : return 0;
217 : }
218 :
219 : int
220 0 : OutputMixer::GetSpeechOutputLevelFullRange(uint32_t& level)
221 : {
222 0 : int16_t currentLevel = _audioLevel.LevelFullRange();
223 0 : level = static_cast<uint32_t> (currentLevel);
224 : WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId,-1),
225 : "GetSpeechOutputLevelFullRange() => level=%u", level);
226 0 : return 0;
227 : }
228 :
229 : int
230 0 : OutputMixer::SetOutputVolumePan(float left, float right)
231 : {
232 : WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
233 : "OutputMixer::SetOutputVolumePan()");
234 0 : _panLeft = left;
235 0 : _panRight = right;
236 0 : return 0;
237 : }
238 :
239 : int
240 0 : OutputMixer::GetOutputVolumePan(float& left, float& right)
241 : {
242 0 : left = _panLeft;
243 0 : right = _panRight;
244 : WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId,-1),
245 : "GetOutputVolumePan() => left=%2.1f, right=%2.1f",
246 : left, right);
247 0 : return 0;
248 : }
249 :
250 0 : int OutputMixer::GetOutputChannelCount()
251 : {
252 0 : return _audioFrame.num_channels_;
253 : }
254 :
255 0 : int OutputMixer::StartRecordingPlayout(const char* fileName,
256 : const CodecInst* codecInst)
257 : {
258 : WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
259 : "OutputMixer::StartRecordingPlayout(fileName=%s)", fileName);
260 :
261 0 : if (_outputFileRecording)
262 : {
263 : WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId,-1),
264 : "StartRecordingPlayout() is already recording");
265 0 : return 0;
266 : }
267 :
268 : FileFormats format;
269 0 : const uint32_t notificationTime(0);
270 0 : CodecInst dummyCodec={100,"L16",16000,320,1,320000};
271 :
272 0 : if ((codecInst != NULL) &&
273 0 : ((codecInst->channels < 1) || (codecInst->channels > 2)))
274 : {
275 0 : _engineStatisticsPtr->SetLastError(
276 : VE_BAD_ARGUMENT, kTraceError,
277 0 : "StartRecordingPlayout() invalid compression");
278 0 : return(-1);
279 : }
280 0 : if(codecInst == NULL)
281 : {
282 0 : format = kFileFormatPcm16kHzFile;
283 0 : codecInst=&dummyCodec;
284 : }
285 0 : else if((STR_CASE_CMP(codecInst->plname,"L16") == 0) ||
286 0 : (STR_CASE_CMP(codecInst->plname,"PCMU") == 0) ||
287 0 : (STR_CASE_CMP(codecInst->plname,"PCMA") == 0))
288 : {
289 0 : format = kFileFormatWavFile;
290 : }
291 : else
292 : {
293 0 : format = kFileFormatCompressedFile;
294 : }
295 :
296 0 : rtc::CritScope cs(&_fileCritSect);
297 :
298 : // Destroy the old instance
299 0 : if (output_file_recorder_) {
300 0 : output_file_recorder_->RegisterModuleFileCallback(NULL);
301 0 : output_file_recorder_.reset();
302 : }
303 :
304 0 : output_file_recorder_ = FileRecorder::CreateFileRecorder(
305 0 : _instanceId, (const FileFormats)format);
306 0 : if (!output_file_recorder_) {
307 0 : _engineStatisticsPtr->SetLastError(
308 : VE_INVALID_ARGUMENT, kTraceError,
309 0 : "StartRecordingPlayout() fileRecorder format isnot correct");
310 0 : return -1;
311 : }
312 :
313 0 : if (output_file_recorder_->StartRecordingAudioFile(
314 0 : fileName, (const CodecInst&)*codecInst, notificationTime) != 0) {
315 0 : _engineStatisticsPtr->SetLastError(
316 : VE_BAD_FILE, kTraceError,
317 0 : "StartRecordingAudioFile() failed to start file recording");
318 0 : output_file_recorder_->StopRecording();
319 0 : output_file_recorder_.reset();
320 0 : return -1;
321 : }
322 0 : output_file_recorder_->RegisterModuleFileCallback(this);
323 0 : _outputFileRecording = true;
324 :
325 0 : return 0;
326 : }
327 :
328 0 : int OutputMixer::StartRecordingPlayout(OutStream* stream,
329 : const CodecInst* codecInst)
330 : {
331 : WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
332 : "OutputMixer::StartRecordingPlayout()");
333 :
334 0 : if (_outputFileRecording)
335 : {
336 : WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId,-1),
337 : "StartRecordingPlayout() is already recording");
338 0 : return 0;
339 : }
340 :
341 : FileFormats format;
342 0 : const uint32_t notificationTime(0);
343 0 : CodecInst dummyCodec={100,"L16",16000,320,1,320000};
344 :
345 0 : if (codecInst != NULL && codecInst->channels != 1)
346 : {
347 0 : _engineStatisticsPtr->SetLastError(
348 : VE_BAD_ARGUMENT, kTraceError,
349 0 : "StartRecordingPlayout() invalid compression");
350 0 : return(-1);
351 : }
352 0 : if(codecInst == NULL)
353 : {
354 0 : format = kFileFormatPcm16kHzFile;
355 0 : codecInst=&dummyCodec;
356 : }
357 0 : else if((STR_CASE_CMP(codecInst->plname,"L16") == 0) ||
358 0 : (STR_CASE_CMP(codecInst->plname,"PCMU") == 0) ||
359 0 : (STR_CASE_CMP(codecInst->plname,"PCMA") == 0))
360 : {
361 0 : format = kFileFormatWavFile;
362 : }
363 : else
364 : {
365 0 : format = kFileFormatCompressedFile;
366 : }
367 :
368 0 : rtc::CritScope cs(&_fileCritSect);
369 :
370 : // Destroy the old instance
371 0 : if (output_file_recorder_) {
372 0 : output_file_recorder_->RegisterModuleFileCallback(NULL);
373 0 : output_file_recorder_.reset();
374 : }
375 :
376 0 : output_file_recorder_ = FileRecorder::CreateFileRecorder(
377 0 : _instanceId, (const FileFormats)format);
378 0 : if (!output_file_recorder_) {
379 0 : _engineStatisticsPtr->SetLastError(
380 : VE_INVALID_ARGUMENT, kTraceError,
381 0 : "StartRecordingPlayout() fileRecorder format isnot correct");
382 0 : return -1;
383 : }
384 :
385 0 : if (output_file_recorder_->StartRecordingAudioFile(stream, *codecInst,
386 0 : notificationTime) != 0) {
387 0 : _engineStatisticsPtr->SetLastError(
388 : VE_BAD_FILE, kTraceError,
389 0 : "StartRecordingAudioFile() failed to start file recording");
390 0 : output_file_recorder_->StopRecording();
391 0 : output_file_recorder_.reset();
392 0 : return -1;
393 : }
394 :
395 0 : output_file_recorder_->RegisterModuleFileCallback(this);
396 0 : _outputFileRecording = true;
397 :
398 0 : return 0;
399 : }
400 :
401 0 : int OutputMixer::StopRecordingPlayout()
402 : {
403 : WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
404 : "OutputMixer::StopRecordingPlayout()");
405 :
406 0 : if (!_outputFileRecording)
407 : {
408 : WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId,-1),
409 : "StopRecordingPlayout() file isnot recording");
410 0 : return -1;
411 : }
412 :
413 0 : rtc::CritScope cs(&_fileCritSect);
414 :
415 0 : if (output_file_recorder_->StopRecording() != 0) {
416 0 : _engineStatisticsPtr->SetLastError(
417 : VE_STOP_RECORDING_FAILED, kTraceError,
418 0 : "StopRecording(), could not stop recording");
419 0 : return -1;
420 : }
421 0 : output_file_recorder_->RegisterModuleFileCallback(NULL);
422 0 : output_file_recorder_.reset();
423 0 : _outputFileRecording = false;
424 :
425 0 : return 0;
426 : }
427 :
428 0 : int OutputMixer::GetMixedAudio(int sample_rate_hz,
429 : size_t num_channels,
430 : AudioFrame* frame) {
431 : WEBRTC_TRACE(
432 : kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
433 : "OutputMixer::GetMixedAudio(sample_rate_hz=%d, num_channels=%" PRIuS ")",
434 : sample_rate_hz, num_channels);
435 :
436 : // --- Record playout if enabled
437 : {
438 0 : rtc::CritScope cs(&_fileCritSect);
439 0 : if (_outputFileRecording && output_file_recorder_)
440 0 : output_file_recorder_->RecordAudioToFile(_audioFrame);
441 : }
442 :
443 0 : frame->num_channels_ = num_channels;
444 0 : frame->sample_rate_hz_ = sample_rate_hz;
445 : // TODO(andrew): Ideally the downmixing would occur much earlier, in
446 : // AudioCodingModule.
447 0 : RemixAndResample(_audioFrame, &resampler_, frame);
448 0 : return 0;
449 : }
450 :
451 : int32_t
452 0 : OutputMixer::DoOperationsOnCombinedSignal(bool feed_data_to_apm)
453 : {
454 0 : if (_audioFrame.sample_rate_hz_ != _mixingFrequencyHz)
455 : {
456 : WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
457 : "OutputMixer::DoOperationsOnCombinedSignal() => "
458 : "mixing frequency = %d", _audioFrame.sample_rate_hz_);
459 0 : _mixingFrequencyHz = _audioFrame.sample_rate_hz_;
460 : }
461 :
462 : // Scale left and/or right channel(s) if balance is active
463 0 : if (_panLeft != 1.0 || _panRight != 1.0)
464 : {
465 0 : if (_audioFrame.num_channels_ == 1)
466 : {
467 0 : AudioFrameOperations::MonoToStereo(&_audioFrame);
468 : }
469 : else
470 : {
471 : // Pure stereo mode (we are receiving a stereo signal).
472 : }
473 :
474 0 : assert(_audioFrame.num_channels_ == 2);
475 0 : AudioFrameOperations::Scale(_panLeft, _panRight, _audioFrame);
476 : }
477 :
478 : // --- Far-end Voice Quality Enhancement (AudioProcessing Module)
479 0 : if (feed_data_to_apm) {
480 0 : APMAnalyzeReverseStream(_audioFrame);
481 : }
482 :
483 : // --- External media processing
484 : {
485 0 : rtc::CritScope cs(&_callbackCritSect);
486 0 : if (_externalMedia)
487 : {
488 0 : const bool is_stereo = (_audioFrame.num_channels_ == 2);
489 0 : if (_externalMediaCallbackPtr)
490 : {
491 0 : _externalMediaCallbackPtr->Process(
492 : -1,
493 : kPlaybackAllChannelsMixed,
494 : (int16_t*)_audioFrame.data_,
495 : _audioFrame.samples_per_channel_,
496 : _audioFrame.sample_rate_hz_,
497 0 : is_stereo);
498 : }
499 : }
500 : }
501 :
502 : // --- Measure audio level (0-9) for the combined signal
503 0 : _audioLevel.ComputeLevel(_audioFrame);
504 :
505 0 : return 0;
506 : }
507 :
508 : // Brought back by Mozilla so we can insert the reverse stream
509 0 : void OutputMixer::APMAnalyzeReverseStream(AudioFrame &audioFrame) {
510 : // Convert 44100Hz to 32000Hz since Processing doesn't support 44100
511 : // directly.
512 : // XXX Bug 1367510 -- convert to 48000? Or modify Processing to
513 : // support 44100 directly? (that's probably the best)
514 0 : AudioFrame *frame = &audioFrame;
515 0 : AudioFrame tempframe;
516 0 : if (frame->sample_rate_hz_ == AudioProcessing::NativeRate::kSampleRate44_1kHz) {
517 0 : tempframe.num_channels_ = 1;
518 0 : tempframe.sample_rate_hz_ = AudioProcessing::NativeRate::kSampleRate32kHz;
519 0 : RemixAndResample(audioFrame, &audioproc_resampler_, &tempframe);
520 0 : frame = &tempframe;
521 : }
522 :
523 0 : if (_audioProcessingModulePtr->ProcessReverseStream(frame) != 0) {
524 : WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId,-1),
525 : "AudioProcessingModule::ProcessReverseStream() => error");
526 0 : RTC_DCHECK(false);
527 : }
528 0 : }
529 :
530 : } // namespace voe
531 : } // namespace webrtc
|
