Line data Source code
1 : /* This Source Code Form is subject to the terms of the Mozilla Public
2 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
4 :
5 : #include "MediaEngineWebRTC.h"
6 : #include <stdio.h>
7 : #include <algorithm>
8 : #include "mozilla/Assertions.h"
9 : #include "MediaTrackConstraints.h"
10 : #include "mtransport/runnable_utils.h"
11 : #include "nsAutoPtr.h"
12 :
13 : // scoped_ptr.h uses FF
14 : #ifdef FF
15 : #undef FF
16 : #endif
17 : #include "webrtc/modules/audio_device/opensl/single_rw_fifo.h"
18 :
19 : #define CHANNELS 1
20 : #define ENCODING "L16"
21 : #define DEFAULT_PORT 5555
22 :
23 : #define SAMPLE_RATE(freq) ((freq)*2*8) // bps, 16-bit samples
24 : #define SAMPLE_LENGTH(freq) (((freq)*10)/1000)
25 :
26 : // These are restrictions from the webrtc.org code
27 : #define MAX_CHANNELS 2
28 : #define MAX_SAMPLING_FREQ 48000 // Hz - multiple of 100
29 :
30 : #define MAX_AEC_FIFO_DEPTH 200 // ms - multiple of 10
31 : static_assert(!(MAX_AEC_FIFO_DEPTH % 10), "Invalid MAX_AEC_FIFO_DEPTH");
32 :
33 : namespace mozilla {
34 :
35 : #ifdef LOG
36 : #undef LOG
37 : #endif
38 :
39 : extern LogModule* GetMediaManagerLog();
40 : #define LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg)
41 : #define LOG_FRAMES(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Verbose, msg)
42 :
43 0 : LogModule* AudioLogModule() {
44 : static mozilla::LazyLogModule log("AudioLatency");
45 0 : return static_cast<LogModule*>(log);
46 : }
47 :
48 : /**
49 : * Webrtc microphone source source.
50 : */
51 0 : NS_IMPL_ISUPPORTS0(MediaEngineWebRTCMicrophoneSource)
52 0 : NS_IMPL_ISUPPORTS0(MediaEngineWebRTCAudioCaptureSource)
53 :
54 : int MediaEngineWebRTCMicrophoneSource::sChannelsOpen = 0;
55 3 : ScopedCustomReleasePtr<webrtc::VoEBase> MediaEngineWebRTCMicrophoneSource::mVoEBase;
56 3 : ScopedCustomReleasePtr<webrtc::VoEExternalMedia> MediaEngineWebRTCMicrophoneSource::mVoERender;
57 3 : ScopedCustomReleasePtr<webrtc::VoENetwork> MediaEngineWebRTCMicrophoneSource::mVoENetwork;
58 3 : ScopedCustomReleasePtr<webrtc::VoEAudioProcessing> MediaEngineWebRTCMicrophoneSource::mVoEProcessing;
59 :
60 0 : AudioOutputObserver::AudioOutputObserver()
61 : : mPlayoutFreq(0)
62 : , mPlayoutChannels(0)
63 : , mChunkSize(0)
64 : , mSaved(nullptr)
65 0 : , mSamplesSaved(0)
66 : {
67 : // Buffers of 10ms chunks
68 0 : mPlayoutFifo = new webrtc::SingleRwFifo(MAX_AEC_FIFO_DEPTH/10);
69 0 : }
70 :
71 0 : AudioOutputObserver::~AudioOutputObserver()
72 : {
73 0 : Clear();
74 0 : free(mSaved);
75 0 : mSaved = nullptr;
76 0 : }
77 :
78 : void
79 0 : AudioOutputObserver::Clear()
80 : {
81 0 : while (mPlayoutFifo->size() > 0) {
82 0 : free(mPlayoutFifo->Pop());
83 : }
84 : // we'd like to touch mSaved here, but we can't if we might still be getting callbacks
85 0 : }
86 :
87 : FarEndAudioChunk *
88 0 : AudioOutputObserver::Pop()
89 : {
90 0 : return (FarEndAudioChunk *) mPlayoutFifo->Pop();
91 : }
92 :
93 : uint32_t
94 0 : AudioOutputObserver::Size()
95 : {
96 0 : return mPlayoutFifo->size();
97 : }
98 :
99 : // static
100 : void
101 0 : AudioOutputObserver::InsertFarEnd(const AudioDataValue *aBuffer, uint32_t aFrames, bool aOverran,
102 : int aFreq, int aChannels)
103 : {
104 0 : if (mPlayoutChannels != 0) {
105 0 : if (mPlayoutChannels != static_cast<uint32_t>(aChannels)) {
106 0 : MOZ_CRASH();
107 : }
108 : } else {
109 0 : MOZ_ASSERT(aChannels <= MAX_CHANNELS);
110 0 : mPlayoutChannels = static_cast<uint32_t>(aChannels);
111 : }
112 0 : if (mPlayoutFreq != 0) {
113 0 : if (mPlayoutFreq != static_cast<uint32_t>(aFreq)) {
114 0 : MOZ_CRASH();
115 : }
116 : } else {
117 0 : MOZ_ASSERT(aFreq <= MAX_SAMPLING_FREQ);
118 0 : MOZ_ASSERT(!(aFreq % 100), "Sampling rate for far end data should be multiple of 100.");
119 0 : mPlayoutFreq = aFreq;
120 0 : mChunkSize = aFreq/100; // 10ms
121 : }
122 :
123 : #ifdef LOG_FAREND_INSERTION
124 : static FILE *fp = fopen("insertfarend.pcm","wb");
125 : #endif
126 :
127 0 : if (mSaved) {
128 : // flag overrun as soon as possible, and only once
129 0 : mSaved->mOverrun = aOverran;
130 0 : aOverran = false;
131 : }
132 : // Rechunk to 10ms.
133 : // The AnalyzeReverseStream() and WebRtcAec_BufferFarend() functions insist on 10ms
134 : // samples per call. Annoying...
135 0 : while (aFrames) {
136 0 : if (!mSaved) {
137 0 : mSaved = (FarEndAudioChunk *) moz_xmalloc(sizeof(FarEndAudioChunk) +
138 0 : (mChunkSize * aChannels - 1)*sizeof(int16_t));
139 0 : mSaved->mSamples = mChunkSize;
140 0 : mSaved->mOverrun = aOverran;
141 0 : aOverran = false;
142 : }
143 0 : uint32_t to_copy = mChunkSize - mSamplesSaved;
144 0 : if (to_copy > aFrames) {
145 0 : to_copy = aFrames;
146 : }
147 :
148 0 : int16_t *dest = &(mSaved->mData[mSamplesSaved * aChannels]);
149 0 : ConvertAudioSamples(aBuffer, dest, to_copy * aChannels);
150 :
151 : #ifdef LOG_FAREND_INSERTION
152 : if (fp) {
153 : fwrite(&(mSaved->mData[mSamplesSaved * aChannels]), to_copy * aChannels, sizeof(int16_t), fp);
154 : }
155 : #endif
156 0 : aFrames -= to_copy;
157 0 : mSamplesSaved += to_copy;
158 0 : aBuffer += to_copy * aChannels;
159 :
160 0 : if (mSamplesSaved >= mChunkSize) {
161 0 : int free_slots = mPlayoutFifo->capacity() - mPlayoutFifo->size();
162 0 : if (free_slots <= 0) {
163 : // XXX We should flag an overrun for the reader. We can't drop data from it due to
164 : // thread safety issues.
165 0 : break;
166 : } else {
167 0 : mPlayoutFifo->Push((int8_t *) mSaved); // takes ownership
168 0 : mSaved = nullptr;
169 0 : mSamplesSaved = 0;
170 : }
171 : }
172 : }
173 0 : }
174 :
175 0 : MediaEngineWebRTCMicrophoneSource::MediaEngineWebRTCMicrophoneSource(
176 : webrtc::VoiceEngine* aVoiceEnginePtr,
177 : mozilla::AudioInput* aAudioInput,
178 : int aIndex,
179 : const char* name,
180 0 : const char* uuid)
181 : : MediaEngineAudioSource(kReleased)
182 : , mVoiceEngine(aVoiceEnginePtr)
183 : , mAudioInput(aAudioInput)
184 : , mMonitor("WebRTCMic.Monitor")
185 : , mCapIndex(aIndex)
186 : , mChannel(-1)
187 : , mTrackID(TRACK_NONE)
188 : , mStarted(false)
189 : , mSampleFrequency(MediaEngine::DEFAULT_SAMPLE_RATE)
190 : , mTotalFrames(0)
191 : , mLastLogFrames(0)
192 : , mPlayoutDelay(0)
193 : , mNullTransport(nullptr)
194 0 : , mSkipProcessing(false)
195 : {
196 0 : MOZ_ASSERT(aVoiceEnginePtr);
197 0 : MOZ_ASSERT(aAudioInput);
198 0 : mDeviceName.Assign(NS_ConvertUTF8toUTF16(name));
199 0 : mDeviceUUID.Assign(uuid);
200 0 : mListener = new mozilla::WebRTCAudioDataListener(this);
201 0 : mSettings.mEchoCancellation.Construct(0);
202 0 : mSettings.mAutoGainControl.Construct(0);
203 0 : mSettings.mNoiseSuppression.Construct(0);
204 0 : mSettings.mChannelCount.Construct(0);
205 : // We'll init lazily as needed
206 0 : }
207 :
208 : void
209 0 : MediaEngineWebRTCMicrophoneSource::GetName(nsAString& aName) const
210 : {
211 0 : aName.Assign(mDeviceName);
212 0 : return;
213 : }
214 :
215 : void
216 0 : MediaEngineWebRTCMicrophoneSource::GetUUID(nsACString& aUUID) const
217 : {
218 0 : aUUID.Assign(mDeviceUUID);
219 0 : return;
220 : }
221 :
222 : // GetBestFitnessDistance returns the best distance the capture device can offer
223 : // as a whole, given an accumulated number of ConstraintSets.
224 : // Ideal values are considered in the first ConstraintSet only.
225 : // Plain values are treated as Ideal in the first ConstraintSet.
226 : // Plain values are treated as Exact in subsequent ConstraintSets.
227 : // Infinity = UINT32_MAX e.g. device cannot satisfy accumulated ConstraintSets.
228 : // A finite result may be used to calculate this device's ranking as a choice.
229 :
230 0 : uint32_t MediaEngineWebRTCMicrophoneSource::GetBestFitnessDistance(
231 : const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
232 : const nsString& aDeviceId) const
233 : {
234 0 : uint32_t distance = 0;
235 :
236 0 : for (const auto* cs : aConstraintSets) {
237 0 : distance = GetMinimumFitnessDistance(*cs, aDeviceId);
238 0 : break; // distance is read from first entry only
239 : }
240 0 : return distance;
241 : }
242 :
243 : nsresult
244 0 : MediaEngineWebRTCMicrophoneSource::Restart(AllocationHandle* aHandle,
245 : const dom::MediaTrackConstraints& aConstraints,
246 : const MediaEnginePrefs &aPrefs,
247 : const nsString& aDeviceId,
248 : const char** aOutBadConstraint)
249 : {
250 0 : AssertIsOnOwningThread();
251 0 : MOZ_ASSERT(aHandle);
252 0 : NormalizedConstraints constraints(aConstraints);
253 0 : return ReevaluateAllocation(aHandle, &constraints, aPrefs, aDeviceId,
254 0 : aOutBadConstraint);
255 : }
256 :
257 0 : bool operator == (const MediaEnginePrefs& a, const MediaEnginePrefs& b)
258 : {
259 0 : return !memcmp(&a, &b, sizeof(MediaEnginePrefs));
260 : };
261 :
262 : nsresult
263 0 : MediaEngineWebRTCMicrophoneSource::UpdateSingleSource(
264 : const AllocationHandle* aHandle,
265 : const NormalizedConstraints& aNetConstraints,
266 : const MediaEnginePrefs& aPrefs,
267 : const nsString& aDeviceId,
268 : const char** aOutBadConstraint)
269 : {
270 0 : FlattenedConstraints c(aNetConstraints);
271 :
272 0 : MediaEnginePrefs prefs = aPrefs;
273 0 : prefs.mAecOn = c.mEchoCancellation.Get(prefs.mAecOn);
274 0 : prefs.mAgcOn = c.mAutoGainControl.Get(prefs.mAgcOn);
275 0 : prefs.mNoiseOn = c.mNoiseSuppression.Get(prefs.mNoiseOn);
276 0 : uint32_t maxChannels = 1;
277 0 : if (mAudioInput->GetMaxAvailableChannels(maxChannels) != 0) {
278 0 : return NS_ERROR_FAILURE;
279 : }
280 : // Check channelCount violation
281 0 : if (static_cast<int32_t>(maxChannels) < c.mChannelCount.mMin ||
282 0 : static_cast<int32_t>(maxChannels) > c.mChannelCount.mMax) {
283 0 : *aOutBadConstraint = "channelCount";
284 0 : return NS_ERROR_FAILURE;
285 : }
286 : // Clamp channelCount to a valid value
287 0 : if (prefs.mChannels <= 0) {
288 0 : prefs.mChannels = static_cast<int32_t>(maxChannels);
289 : }
290 0 : prefs.mChannels = c.mChannelCount.Get(std::min(prefs.mChannels,
291 0 : static_cast<int32_t>(maxChannels)));
292 : // Clamp channelCount to a valid value
293 0 : prefs.mChannels = std::max(1, std::min(prefs.mChannels, static_cast<int32_t>(maxChannels)));
294 :
295 0 : LOG(("Audio config: aec: %d, agc: %d, noise: %d, delay: %d, channels: %d",
296 : prefs.mAecOn ? prefs.mAec : -1,
297 : prefs.mAgcOn ? prefs.mAgc : -1,
298 : prefs.mNoiseOn ? prefs.mNoise : -1,
299 : prefs.mPlayoutDelay,
300 : prefs.mChannels));
301 :
302 0 : mPlayoutDelay = prefs.mPlayoutDelay;
303 :
304 0 : switch (mState) {
305 : case kReleased:
306 0 : MOZ_ASSERT(aHandle);
307 0 : if (sChannelsOpen == 0) {
308 0 : if (!InitEngine()) {
309 0 : LOG(("Audio engine is not initalized"));
310 0 : return NS_ERROR_FAILURE;
311 : }
312 : } else {
313 : // Until we fix (or wallpaper) support for multiple mic input
314 : // (Bug 1238038) fail allocation for a second device
315 0 : return NS_ERROR_FAILURE;
316 : }
317 0 : if (mAudioInput->SetRecordingDevice(mCapIndex)) {
318 0 : return NS_ERROR_FAILURE;
319 : }
320 0 : mAudioInput->SetUserChannelCount(prefs.mChannels);
321 0 : if (!AllocChannel()) {
322 0 : FreeChannel();
323 0 : LOG(("Audio device is not initalized"));
324 0 : return NS_ERROR_FAILURE;
325 : }
326 0 : LOG(("Audio device %d allocated", mCapIndex));
327 : {
328 : // Update with the actual applied channelCount in order
329 : // to store it in settings.
330 0 : uint32_t channelCount = 0;
331 0 : mAudioInput->GetChannelCount(channelCount);
332 0 : MOZ_ASSERT(channelCount > 0);
333 0 : prefs.mChannels = channelCount;
334 : }
335 0 : break;
336 :
337 : case kStarted:
338 0 : if (prefs == mLastPrefs) {
339 0 : return NS_OK;
340 : }
341 :
342 0 : if (prefs.mChannels != mLastPrefs.mChannels) {
343 0 : MOZ_ASSERT(mSources.Length() > 0);
344 0 : auto& source = mSources.LastElement();
345 0 : mAudioInput->SetUserChannelCount(prefs.mChannels);
346 : // Get validated number of channel
347 0 : uint32_t channelCount = 0;
348 0 : mAudioInput->GetChannelCount(channelCount);
349 0 : MOZ_ASSERT(channelCount > 0 && mLastPrefs.mChannels > 0);
350 : // Check if new validated channels is the same as previous
351 0 : if (static_cast<uint32_t>(mLastPrefs.mChannels) != channelCount
352 0 : && !source->OpenNewAudioCallbackDriver(mListener)) {
353 0 : return NS_ERROR_FAILURE;
354 : }
355 : // Update settings
356 0 : prefs.mChannels = channelCount;
357 : }
358 :
359 0 : if (MOZ_LOG_TEST(GetMediaManagerLog(), LogLevel::Debug)) {
360 0 : MonitorAutoLock lock(mMonitor);
361 0 : if (mSources.IsEmpty()) {
362 0 : LOG(("Audio device %d reallocated", mCapIndex));
363 : } else {
364 0 : LOG(("Audio device %d allocated shared", mCapIndex));
365 : }
366 : }
367 0 : break;
368 :
369 : default:
370 0 : LOG(("Audio device %d in ignored state %d", mCapIndex, mState));
371 0 : break;
372 : }
373 :
374 0 : if (sChannelsOpen > 0) {
375 : int error;
376 :
377 0 : error = mVoEProcessing->SetEcStatus(prefs.mAecOn, (webrtc::EcModes)prefs.mAec);
378 0 : if (error) {
379 0 : LOG(("%s Error setting Echo Status: %d ",__FUNCTION__, error));
380 : // Overhead of capturing all the time is very low (<0.1% of an audio only call)
381 0 : if (prefs.mAecOn) {
382 0 : error = mVoEProcessing->SetEcMetricsStatus(true);
383 0 : if (error) {
384 0 : LOG(("%s Error setting Echo Metrics: %d ",__FUNCTION__, error));
385 : }
386 : }
387 : }
388 0 : error = mVoEProcessing->SetAgcStatus(prefs.mAgcOn, (webrtc::AgcModes)prefs.mAgc);
389 0 : if (error) {
390 0 : LOG(("%s Error setting AGC Status: %d ",__FUNCTION__, error));
391 : }
392 0 : error = mVoEProcessing->SetNsStatus(prefs.mNoiseOn, (webrtc::NsModes)prefs.mNoise);
393 0 : if (error) {
394 0 : LOG(("%s Error setting NoiseSuppression Status: %d ",__FUNCTION__, error));
395 : }
396 : }
397 :
398 0 : mSkipProcessing = !(prefs.mAecOn || prefs.mAgcOn || prefs.mNoiseOn);
399 0 : if (mSkipProcessing) {
400 0 : mSampleFrequency = MediaEngine::USE_GRAPH_RATE;
401 0 : mAudioOutputObserver = nullptr;
402 : } else {
403 : // make sure we route a copy of the mixed audio output of this MSG to the
404 : // AEC
405 0 : mAudioOutputObserver = new AudioOutputObserver();
406 : }
407 0 : SetLastPrefs(prefs);
408 0 : return NS_OK;
409 : }
410 :
411 : void
412 0 : MediaEngineWebRTCMicrophoneSource::SetLastPrefs(
413 : const MediaEnginePrefs& aPrefs)
414 : {
415 0 : mLastPrefs = aPrefs;
416 :
417 0 : RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
418 :
419 0 : NS_DispatchToMainThread(media::NewRunnableFrom([that, aPrefs]() mutable {
420 0 : that->mSettings.mEchoCancellation.Value() = aPrefs.mAecOn;
421 0 : that->mSettings.mAutoGainControl.Value() = aPrefs.mAgcOn;
422 0 : that->mSettings.mNoiseSuppression.Value() = aPrefs.mNoiseOn;
423 0 : that->mSettings.mChannelCount.Value() = aPrefs.mChannels;
424 0 : return NS_OK;
425 0 : }));
426 0 : }
427 :
428 :
429 : nsresult
430 0 : MediaEngineWebRTCMicrophoneSource::Deallocate(AllocationHandle* aHandle)
431 : {
432 0 : AssertIsOnOwningThread();
433 :
434 0 : Super::Deallocate(aHandle);
435 :
436 0 : if (!mRegisteredHandles.Length()) {
437 : // If empty, no callbacks to deliver data should be occuring
438 0 : if (mState != kStopped && mState != kAllocated) {
439 0 : return NS_ERROR_FAILURE;
440 : }
441 :
442 0 : FreeChannel();
443 0 : LOG(("Audio device %d deallocated", mCapIndex));
444 : } else {
445 0 : LOG(("Audio device %d deallocated but still in use", mCapIndex));
446 : }
447 0 : return NS_OK;
448 : }
449 :
450 : nsresult
451 0 : MediaEngineWebRTCMicrophoneSource::Start(SourceMediaStream *aStream,
452 : TrackID aID,
453 : const PrincipalHandle& aPrincipalHandle)
454 : {
455 0 : AssertIsOnOwningThread();
456 0 : if (sChannelsOpen == 0 || !aStream) {
457 0 : return NS_ERROR_FAILURE;
458 : }
459 :
460 : {
461 0 : MonitorAutoLock lock(mMonitor);
462 0 : mSources.AppendElement(aStream);
463 0 : mPrincipalHandles.AppendElement(aPrincipalHandle);
464 0 : MOZ_ASSERT(mSources.Length() == mPrincipalHandles.Length());
465 : }
466 :
467 0 : AudioSegment* segment = new AudioSegment();
468 0 : if (mSampleFrequency == MediaEngine::USE_GRAPH_RATE) {
469 0 : mSampleFrequency = aStream->GraphRate();
470 : }
471 0 : aStream->AddAudioTrack(aID, mSampleFrequency, 0, segment, SourceMediaStream::ADDTRACK_QUEUED);
472 :
473 : // XXX Make this based on the pref.
474 0 : aStream->RegisterForAudioMixing();
475 0 : LOG(("Start audio for stream %p", aStream));
476 :
477 0 : if (!mListener) {
478 0 : mListener = new mozilla::WebRTCAudioDataListener(this);
479 : }
480 0 : if (mState == kStarted) {
481 0 : MOZ_ASSERT(aID == mTrackID);
482 : // Make sure we're associated with this stream
483 0 : mAudioInput->StartRecording(aStream, mListener);
484 0 : return NS_OK;
485 : }
486 0 : mState = kStarted;
487 0 : mTrackID = aID;
488 :
489 : // Make sure logger starts before capture
490 0 : AsyncLatencyLogger::Get(true);
491 :
492 0 : MOZ_ASSERT(mAudioOutputObserver);
493 0 : mAudioOutputObserver->Clear();
494 :
495 0 : if (mVoEBase->StartReceive(mChannel)) {
496 0 : return NS_ERROR_FAILURE;
497 : }
498 :
499 : // Must be *before* StartSend() so it will notice we selected external input (full_duplex)
500 0 : mAudioInput->StartRecording(aStream, mListener);
501 :
502 0 : if (mVoEBase->StartSend(mChannel)) {
503 0 : return NS_ERROR_FAILURE;
504 : }
505 :
506 : // Attach external media processor, so this::Process will be called.
507 0 : mVoERender->RegisterExternalMediaProcessing(mChannel, webrtc::kRecordingPerChannel, *this);
508 :
509 0 : return NS_OK;
510 : }
511 :
512 : nsresult
513 0 : MediaEngineWebRTCMicrophoneSource::Stop(SourceMediaStream *aSource, TrackID aID)
514 : {
515 0 : AssertIsOnOwningThread();
516 : {
517 0 : MonitorAutoLock lock(mMonitor);
518 :
519 0 : size_t sourceIndex = mSources.IndexOf(aSource);
520 0 : if (sourceIndex == mSources.NoIndex) {
521 : // Already stopped - this is allowed
522 0 : return NS_OK;
523 : }
524 0 : mSources.RemoveElementAt(sourceIndex);
525 0 : mPrincipalHandles.RemoveElementAt(sourceIndex);
526 0 : MOZ_ASSERT(mSources.Length() == mPrincipalHandles.Length());
527 :
528 0 : aSource->EndTrack(aID);
529 :
530 0 : if (!mSources.IsEmpty()) {
531 0 : mAudioInput->StopRecording(aSource);
532 0 : return NS_OK;
533 : }
534 0 : if (mState != kStarted) {
535 0 : return NS_ERROR_FAILURE;
536 : }
537 0 : if (!mVoEBase) {
538 0 : return NS_ERROR_FAILURE;
539 : }
540 :
541 0 : mState = kStopped;
542 : }
543 0 : if (mListener) {
544 : // breaks a cycle, since the WebRTCAudioDataListener has a RefPtr to us
545 0 : mListener->Shutdown();
546 0 : mListener = nullptr;
547 : }
548 :
549 0 : mAudioInput->StopRecording(aSource);
550 :
551 0 : mVoERender->DeRegisterExternalMediaProcessing(mChannel, webrtc::kRecordingPerChannel);
552 :
553 0 : if (mVoEBase->StopSend(mChannel)) {
554 0 : return NS_ERROR_FAILURE;
555 : }
556 0 : if (mVoEBase->StopReceive(mChannel)) {
557 0 : return NS_ERROR_FAILURE;
558 : }
559 0 : return NS_OK;
560 : }
561 :
562 : void
563 0 : MediaEngineWebRTCMicrophoneSource::NotifyPull(MediaStreamGraph *aGraph,
564 : SourceMediaStream *aSource,
565 : TrackID aID,
566 : StreamTime aDesiredTime,
567 : const PrincipalHandle& aPrincipalHandle)
568 : {
569 : // Ignore - we push audio data
570 0 : LOG_FRAMES(("NotifyPull, desired = %" PRId64, (int64_t) aDesiredTime));
571 0 : }
572 :
573 : void
574 0 : MediaEngineWebRTCMicrophoneSource::NotifyOutputData(MediaStreamGraph* aGraph,
575 : AudioDataValue* aBuffer,
576 : size_t aFrames,
577 : TrackRate aRate,
578 : uint32_t aChannels)
579 : {
580 0 : if (mAudioOutputObserver) {
581 0 : mAudioOutputObserver->InsertFarEnd(aBuffer, aFrames, false,
582 0 : aRate, aChannels);
583 : }
584 0 : }
585 :
586 : void
587 0 : MediaEngineWebRTCMicrophoneSource::PacketizeAndProcess(MediaStreamGraph* aGraph,
588 : const AudioDataValue* aBuffer,
589 : size_t aFrames,
590 : TrackRate aRate,
591 : uint32_t aChannels)
592 : {
593 : // This will call Process() with data coming out of the AEC/NS/AGC/etc chain
594 0 : if (!mPacketizer ||
595 0 : mPacketizer->PacketSize() != aRate/100u ||
596 0 : mPacketizer->Channels() != aChannels) {
597 : // It's ok to drop the audio still in the packetizer here.
598 : mPacketizer =
599 0 : new AudioPacketizer<AudioDataValue, int16_t>(aRate/100, aChannels);
600 : }
601 :
602 0 : mPacketizer->Input(aBuffer, static_cast<uint32_t>(aFrames));
603 :
604 0 : while (mPacketizer->PacketsAvailable()) {
605 0 : uint32_t samplesPerPacket = mPacketizer->PacketSize() *
606 0 : mPacketizer->Channels();
607 0 : if (mInputBuffer.Length() < samplesPerPacket) {
608 0 : mInputBuffer.SetLength(samplesPerPacket);
609 : }
610 0 : int16_t* packet = mInputBuffer.Elements();
611 0 : mPacketizer->Output(packet);
612 :
613 0 : mVoERender->ExternalRecordingInsertData(packet, samplesPerPacket, aRate, 0);
614 : }
615 0 : }
616 :
617 : template<typename T>
618 : void
619 0 : MediaEngineWebRTCMicrophoneSource::InsertInGraph(const T* aBuffer,
620 : size_t aFrames,
621 : uint32_t aChannels)
622 : {
623 0 : if (mState != kStarted) {
624 0 : return;
625 : }
626 :
627 0 : if (MOZ_LOG_TEST(AudioLogModule(), LogLevel::Debug)) {
628 0 : mTotalFrames += aFrames;
629 0 : if (mTotalFrames > mLastLogFrames + mSampleFrequency) { // ~ 1 second
630 0 : MOZ_LOG(AudioLogModule(), LogLevel::Debug,
631 : ("%p: Inserting %" PRIuSIZE " samples into graph, total frames = %" PRIu64,
632 : (void*)this, aFrames, mTotalFrames));
633 0 : mLastLogFrames = mTotalFrames;
634 : }
635 : }
636 :
637 0 : size_t len = mSources.Length();
638 0 : for (size_t i = 0; i < len; i++) {
639 0 : if (!mSources[i]) {
640 0 : continue;
641 : }
642 :
643 0 : TimeStamp insertTime;
644 : // Make sure we include the stream and the track.
645 : // The 0:1 is a flag to note when we've done the final insert for a given input block.
646 0 : LogTime(AsyncLatencyLogger::AudioTrackInsertion,
647 0 : LATENCY_STREAM_ID(mSources[i].get(), mTrackID),
648 0 : (i+1 < len) ? 0 : 1, insertTime);
649 :
650 : // Bug 971528 - Support stereo capture in gUM
651 0 : MOZ_ASSERT(aChannels == 1 || aChannels == 2,
652 : "GraphDriver only supports mono and stereo audio for now");
653 :
654 0 : nsAutoPtr<AudioSegment> segment(new AudioSegment());
655 : RefPtr<SharedBuffer> buffer =
656 0 : SharedBuffer::Create(aFrames * aChannels * sizeof(T));
657 0 : AutoTArray<const T*, 8> channels;
658 0 : if (aChannels == 1) {
659 0 : PodCopy(static_cast<T*>(buffer->Data()), aBuffer, aFrames);
660 0 : channels.AppendElement(static_cast<T*>(buffer->Data()));
661 : } else {
662 0 : channels.SetLength(aChannels);
663 0 : AutoTArray<T*, 8> write_channels;
664 0 : write_channels.SetLength(aChannels);
665 0 : T * samples = static_cast<T*>(buffer->Data());
666 :
667 0 : size_t offset = 0;
668 0 : for(uint32_t i = 0; i < aChannels; ++i) {
669 0 : channels[i] = write_channels[i] = samples + offset;
670 0 : offset += aFrames;
671 : }
672 :
673 0 : DeinterleaveAndConvertBuffer(aBuffer,
674 : aFrames,
675 : aChannels,
676 : write_channels.Elements());
677 : }
678 :
679 0 : MOZ_ASSERT(aChannels == channels.Length());
680 0 : segment->AppendFrames(buffer.forget(), channels, aFrames,
681 : mPrincipalHandles[i]);
682 0 : segment->GetStartTime(insertTime);
683 :
684 0 : mSources[i]->AppendToTrack(mTrackID, segment);
685 : }
686 : }
687 :
688 : // Called back on GraphDriver thread!
689 : // Note this can be called back after ::Shutdown()
690 : void
691 0 : MediaEngineWebRTCMicrophoneSource::NotifyInputData(MediaStreamGraph* aGraph,
692 : const AudioDataValue* aBuffer,
693 : size_t aFrames,
694 : TrackRate aRate,
695 : uint32_t aChannels)
696 : {
697 : // If some processing is necessary, packetize and insert in the WebRTC.org
698 : // code. Otherwise, directly insert the mic data in the MSG, bypassing all processing.
699 0 : if (PassThrough()) {
700 0 : InsertInGraph<AudioDataValue>(aBuffer, aFrames, aChannels);
701 : } else {
702 0 : PacketizeAndProcess(aGraph, aBuffer, aFrames, aRate, aChannels);
703 : }
704 0 : }
705 :
706 : #define ResetProcessingIfNeeded(_processing) \
707 : do { \
708 : webrtc::_processing##Modes mode; \
709 : int rv = mVoEProcessing->Get##_processing##Status(enabled, mode); \
710 : if (rv) { \
711 : NS_WARNING("Could not get the status of the " \
712 : #_processing " on device change."); \
713 : return; \
714 : } \
715 : \
716 : if (enabled) { \
717 : rv = mVoEProcessing->Set##_processing##Status(!enabled); \
718 : if (rv) { \
719 : NS_WARNING("Could not reset the status of the " \
720 : #_processing " on device change."); \
721 : return; \
722 : } \
723 : \
724 : rv = mVoEProcessing->Set##_processing##Status(enabled); \
725 : if (rv) { \
726 : NS_WARNING("Could not reset the status of the " \
727 : #_processing " on device change."); \
728 : return; \
729 : } \
730 : } \
731 : } while(0)
732 :
733 : void
734 0 : MediaEngineWebRTCMicrophoneSource::DeviceChanged() {
735 : // Reset some processing
736 : bool enabled;
737 0 : ResetProcessingIfNeeded(Agc);
738 0 : ResetProcessingIfNeeded(Ec);
739 0 : ResetProcessingIfNeeded(Ns);
740 : }
741 :
742 : bool
743 0 : MediaEngineWebRTCMicrophoneSource::InitEngine()
744 : {
745 0 : MOZ_ASSERT(!mVoEBase);
746 0 : mVoEBase = webrtc::VoEBase::GetInterface(mVoiceEngine);
747 :
748 0 : mVoEBase->Init();
749 :
750 0 : mVoERender = webrtc::VoEExternalMedia::GetInterface(mVoiceEngine);
751 0 : if (mVoERender) {
752 0 : mVoENetwork = webrtc::VoENetwork::GetInterface(mVoiceEngine);
753 0 : if (mVoENetwork) {
754 0 : mVoEProcessing = webrtc::VoEAudioProcessing::GetInterface(mVoiceEngine);
755 0 : if (mVoEProcessing) {
756 0 : mNullTransport = new NullTransport();
757 0 : return true;
758 : }
759 : }
760 : }
761 0 : DeInitEngine();
762 0 : return false;
763 : }
764 :
765 : // This shuts down the engine when no channel is open
766 : void
767 0 : MediaEngineWebRTCMicrophoneSource::DeInitEngine()
768 : {
769 0 : if (mVoEBase) {
770 0 : mVoEBase->Terminate();
771 0 : delete mNullTransport;
772 0 : mNullTransport = nullptr;
773 :
774 0 : mVoEProcessing = nullptr;
775 0 : mVoENetwork = nullptr;
776 0 : mVoERender = nullptr;
777 0 : mVoEBase = nullptr;
778 : }
779 0 : }
780 :
781 : // This shuts down the engine when no channel is open.
782 : // mState records if a channel is allocated (slightly redundantly to mChannel)
783 : void
784 0 : MediaEngineWebRTCMicrophoneSource::FreeChannel()
785 : {
786 0 : if (mState != kReleased) {
787 0 : if (mChannel != -1) {
788 0 : MOZ_ASSERT(mVoENetwork && mVoEBase);
789 0 : if (mVoENetwork) {
790 0 : mVoENetwork->DeRegisterExternalTransport(mChannel);
791 : }
792 0 : if (mVoEBase) {
793 0 : mVoEBase->DeleteChannel(mChannel);
794 : }
795 0 : mChannel = -1;
796 : }
797 0 : mState = kReleased;
798 :
799 0 : MOZ_ASSERT(sChannelsOpen > 0);
800 0 : if (--sChannelsOpen == 0) {
801 0 : DeInitEngine();
802 : }
803 : }
804 0 : }
805 :
806 : bool
807 0 : MediaEngineWebRTCMicrophoneSource::AllocChannel()
808 : {
809 0 : MOZ_ASSERT(mVoEBase);
810 :
811 0 : mChannel = mVoEBase->CreateChannel();
812 0 : if (mChannel >= 0) {
813 0 : if (!mVoENetwork->RegisterExternalTransport(mChannel, *mNullTransport)) {
814 0 : mSampleFrequency = MediaEngine::DEFAULT_SAMPLE_RATE;
815 0 : LOG(("%s: sampling rate %u", __FUNCTION__, mSampleFrequency));
816 :
817 : // Check for availability.
818 0 : if (!mAudioInput->SetRecordingDevice(mCapIndex)) {
819 : #ifndef MOZ_B2G
820 : // Because of the permission mechanism of B2G, we need to skip the status
821 : // check here.
822 0 : bool avail = false;
823 0 : mAudioInput->GetRecordingDeviceStatus(avail);
824 0 : if (!avail) {
825 0 : if (sChannelsOpen == 0) {
826 0 : DeInitEngine();
827 : }
828 0 : return false;
829 : }
830 : #endif // MOZ_B2G
831 :
832 : // Set "codec" to PCM, 32kHz on device's channels
833 0 : ScopedCustomReleasePtr<webrtc::VoECodec> ptrVoECodec(webrtc::VoECodec::GetInterface(mVoiceEngine));
834 0 : if (ptrVoECodec) {
835 : webrtc::CodecInst codec;
836 0 : strcpy(codec.plname, ENCODING);
837 0 : codec.channels = CHANNELS;
838 0 : uint32_t maxChannels = 0;
839 0 : if (mAudioInput->GetMaxAvailableChannels(maxChannels) == 0) {
840 0 : codec.channels = maxChannels;
841 : }
842 0 : MOZ_ASSERT(mSampleFrequency == 16000 || mSampleFrequency == 32000);
843 0 : codec.rate = SAMPLE_RATE(mSampleFrequency);
844 0 : codec.plfreq = mSampleFrequency;
845 0 : codec.pacsize = SAMPLE_LENGTH(mSampleFrequency);
846 0 : codec.pltype = 0; // Default payload type
847 :
848 0 : if (!ptrVoECodec->SetSendCodec(mChannel, codec)) {
849 0 : mState = kAllocated;
850 0 : sChannelsOpen++;
851 0 : return true;
852 : }
853 : }
854 : }
855 : }
856 : }
857 0 : mVoEBase->DeleteChannel(mChannel);
858 0 : mChannel = -1;
859 0 : if (sChannelsOpen == 0) {
860 0 : DeInitEngine();
861 : }
862 0 : return false;
863 : }
864 :
865 : void
866 0 : MediaEngineWebRTCMicrophoneSource::Shutdown()
867 : {
868 0 : Super::Shutdown();
869 0 : if (mListener) {
870 : // breaks a cycle, since the WebRTCAudioDataListener has a RefPtr to us
871 0 : mListener->Shutdown();
872 : // Don't release the webrtc.org pointers yet until the Listener is (async) shutdown
873 0 : mListener = nullptr;
874 : }
875 :
876 0 : if (mState == kStarted) {
877 : SourceMediaStream *source;
878 : bool empty;
879 :
880 : while (1) {
881 : {
882 0 : MonitorAutoLock lock(mMonitor);
883 0 : empty = mSources.IsEmpty();
884 0 : if (empty) {
885 0 : break;
886 : }
887 0 : source = mSources[0];
888 : }
889 0 : Stop(source, kAudioTrack); // XXX change to support multiple tracks
890 0 : }
891 0 : MOZ_ASSERT(mState == kStopped);
892 : }
893 :
894 0 : while (mRegisteredHandles.Length()) {
895 0 : MOZ_ASSERT(mState == kAllocated || mState == kStopped);
896 : // on last Deallocate(), FreeChannel()s and DeInit()s if all channels are released
897 0 : Deallocate(mRegisteredHandles[0].get());
898 : }
899 0 : MOZ_ASSERT(mState == kReleased);
900 :
901 0 : mAudioInput = nullptr;
902 0 : }
903 :
904 : typedef int16_t sample;
905 :
906 : void
907 0 : MediaEngineWebRTCMicrophoneSource::Process(int channel,
908 : webrtc::ProcessingTypes type,
909 : sample *audio10ms, size_t length,
910 : int samplingFreq, bool isStereo)
911 : {
912 0 : MOZ_ASSERT(!PassThrough(), "This should be bypassed when in PassThrough mode.");
913 : // On initial capture, throw away all far-end data except the most recent sample
914 : // since it's already irrelevant and we want to keep avoid confusing the AEC far-end
915 : // input code with "old" audio.
916 0 : if (!mStarted) {
917 0 : mStarted = true;
918 0 : while (mAudioOutputObserver->Size() > 1) {
919 0 : free(mAudioOutputObserver->Pop()); // only call if size() > 0
920 : }
921 : }
922 :
923 0 : while (mAudioOutputObserver->Size() > 0) {
924 0 : FarEndAudioChunk *buffer = mAudioOutputObserver->Pop(); // only call if size() > 0
925 0 : if (buffer) {
926 0 : int length = buffer->mSamples;
927 0 : int res = mVoERender->ExternalPlayoutData(buffer->mData,
928 0 : mAudioOutputObserver->PlayoutFrequency(),
929 0 : mAudioOutputObserver->PlayoutChannels(),
930 : mPlayoutDelay,
931 0 : length);
932 0 : free(buffer);
933 0 : if (res == -1) {
934 0 : return;
935 : }
936 : }
937 : }
938 :
939 0 : MonitorAutoLock lock(mMonitor);
940 0 : if (mState != kStarted)
941 0 : return;
942 :
943 0 : uint32_t channels = isStereo ? 2 : 1;
944 0 : InsertInGraph<int16_t>(audio10ms, length, channels);
945 0 : return;
946 : }
947 :
948 : void
949 0 : MediaEngineWebRTCAudioCaptureSource::GetName(nsAString &aName) const
950 : {
951 0 : aName.AssignLiteral("AudioCapture");
952 0 : }
953 :
954 : void
955 0 : MediaEngineWebRTCAudioCaptureSource::GetUUID(nsACString &aUUID) const
956 : {
957 : nsID uuid;
958 : char uuidBuffer[NSID_LENGTH];
959 0 : nsCString asciiString;
960 0 : ErrorResult rv;
961 :
962 0 : rv = nsContentUtils::GenerateUUIDInPlace(uuid);
963 0 : if (rv.Failed()) {
964 0 : aUUID.AssignLiteral("");
965 0 : return;
966 : }
967 :
968 :
969 0 : uuid.ToProvidedString(uuidBuffer);
970 0 : asciiString.AssignASCII(uuidBuffer);
971 :
972 : // Remove {} and the null terminator
973 0 : aUUID.Assign(Substring(asciiString, 1, NSID_LENGTH - 3));
974 : }
975 :
976 : nsresult
977 0 : MediaEngineWebRTCAudioCaptureSource::Start(SourceMediaStream *aMediaStream,
978 : TrackID aId,
979 : const PrincipalHandle& aPrincipalHandle)
980 : {
981 0 : AssertIsOnOwningThread();
982 0 : aMediaStream->AddTrack(aId, 0, new AudioSegment());
983 0 : return NS_OK;
984 : }
985 :
986 : nsresult
987 0 : MediaEngineWebRTCAudioCaptureSource::Stop(SourceMediaStream *aMediaStream,
988 : TrackID aId)
989 : {
990 0 : AssertIsOnOwningThread();
991 0 : aMediaStream->EndAllTrackAndFinish();
992 0 : return NS_OK;
993 : }
994 :
995 : nsresult
996 0 : MediaEngineWebRTCAudioCaptureSource::Restart(
997 : AllocationHandle* aHandle,
998 : const dom::MediaTrackConstraints& aConstraints,
999 : const MediaEnginePrefs &aPrefs,
1000 : const nsString& aDeviceId,
1001 : const char** aOutBadConstraint)
1002 : {
1003 0 : MOZ_ASSERT(!aHandle);
1004 0 : return NS_OK;
1005 : }
1006 :
1007 : uint32_t
1008 0 : MediaEngineWebRTCAudioCaptureSource::GetBestFitnessDistance(
1009 : const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
1010 : const nsString& aDeviceId) const
1011 : {
1012 : // There is only one way of capturing audio for now, and it's always adequate.
1013 0 : return 0;
1014 : }
1015 :
1016 : }
|
