Line data Source code
1 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 : /* This Source Code Form is subject to the terms of the Mozilla Public
3 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
5 : #include "TrackEncoder.h"
6 : #include "AudioChannelFormat.h"
7 : #include "MediaStreamGraph.h"
8 : #include "MediaStreamListener.h"
9 : #include "mozilla/Logging.h"
10 : #include "VideoUtils.h"
11 : #include "mozilla/Logging.h"
12 :
13 : namespace mozilla {
14 :
15 : LazyLogModule gTrackEncoderLog("TrackEncoder");
16 : #define TRACK_LOG(type, msg) MOZ_LOG(gTrackEncoderLog, type, msg)
17 :
18 : static const int DEFAULT_CHANNELS = 1;
19 : static const int DEFAULT_SAMPLING_RATE = 16000;
20 : static const int DEFAULT_FRAME_WIDTH = 640;
21 : static const int DEFAULT_FRAME_HEIGHT = 480;
22 : static const int DEFAULT_TRACK_RATE = USECS_PER_S;
23 : // 30 seconds threshold if the encoder still can't not be initialized.
24 : static const int INIT_FAILED_DURATION = 30;
25 :
26 0 : TrackEncoder::TrackEncoder()
27 : : mReentrantMonitor("media.TrackEncoder")
28 : , mEncodingComplete(false)
29 : , mEosSetInEncoder(false)
30 : , mInitialized(false)
31 : , mEndOfStream(false)
32 : , mCanceled(false)
33 : , mInitCounter(0)
34 0 : , mNotInitDuration(0)
35 : {
36 0 : }
37 :
38 0 : void TrackEncoder::NotifyEvent(MediaStreamGraph* aGraph,
39 : MediaStreamGraphEvent event)
40 : {
41 0 : if (event == MediaStreamGraphEvent::EVENT_REMOVED) {
42 0 : NotifyEndOfStream();
43 : }
44 0 : }
45 :
46 : void
47 0 : AudioTrackEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
48 : TrackID aID,
49 : StreamTime aTrackOffset,
50 : uint32_t aTrackEvents,
51 : const MediaSegment& aQueuedMedia)
52 : {
53 0 : ReentrantMonitorAutoEnter mon(mReentrantMonitor);
54 :
55 0 : if (mCanceled) {
56 0 : return;
57 : }
58 :
59 0 : const AudioSegment& audio = static_cast<const AudioSegment&>(aQueuedMedia);
60 :
61 : // Check and initialize parameters for codec encoder.
62 0 : if (!mInitialized) {
63 0 : mInitCounter++;
64 0 : TRACK_LOG(LogLevel::Debug, ("Init the audio encoder %d times", mInitCounter));
65 0 : AudioSegment::ChunkIterator iter(const_cast<AudioSegment&>(audio));
66 0 : while (!iter.IsEnded()) {
67 0 : AudioChunk chunk = *iter;
68 :
69 : // The number of channels is determined by the first non-null chunk, and
70 : // thus the audio encoder is initialized at this time.
71 0 : if (!chunk.IsNull()) {
72 0 : nsresult rv = Init(chunk.mChannelData.Length(), aGraph->GraphRate());
73 0 : if (NS_FAILED(rv)) {
74 0 : TRACK_LOG(LogLevel::Error, ("[AudioTrackEncoder]: Fail to initialize the encoder!"));
75 0 : NotifyCancel();
76 : }
77 0 : break;
78 : }
79 :
80 0 : iter.Next();
81 : }
82 :
83 0 : mNotInitDuration += aQueuedMedia.GetDuration();
84 0 : if (!mInitialized &&
85 0 : (mNotInitDuration / aGraph->GraphRate() > INIT_FAILED_DURATION) &&
86 0 : mInitCounter > 1) {
87 0 : TRACK_LOG(LogLevel::Warning, ("[AudioTrackEncoder]: Initialize failed for 30s."));
88 0 : NotifyEndOfStream();
89 0 : return;
90 : }
91 : }
92 :
93 : // Append and consume this raw segment.
94 0 : AppendAudioSegment(audio);
95 :
96 :
97 : // The stream has stopped and reached the end of track.
98 0 : if (aTrackEvents == TrackEventCommand::TRACK_EVENT_ENDED) {
99 0 : TRACK_LOG(LogLevel::Info, ("[AudioTrackEncoder]: Receive TRACK_EVENT_ENDED ."));
100 0 : NotifyEndOfStream();
101 : }
102 : }
103 :
104 : void
105 0 : AudioTrackEncoder::NotifyEndOfStream()
106 : {
107 0 : ReentrantMonitorAutoEnter mon(mReentrantMonitor);
108 :
109 : // If source audio track is completely silent till the end of encoding,
110 : // initialize the encoder with default channel counts and sampling rate.
111 0 : if (!mCanceled && !mInitialized) {
112 0 : Init(DEFAULT_CHANNELS, DEFAULT_SAMPLING_RATE);
113 : }
114 :
115 0 : mEndOfStream = true;
116 0 : mReentrantMonitor.NotifyAll();
117 0 : }
118 :
119 : nsresult
120 0 : AudioTrackEncoder::AppendAudioSegment(const AudioSegment& aSegment)
121 : {
122 0 : ReentrantMonitorAutoEnter mon(mReentrantMonitor);
123 :
124 0 : AudioSegment::ChunkIterator iter(const_cast<AudioSegment&>(aSegment));
125 0 : while (!iter.IsEnded()) {
126 0 : AudioChunk chunk = *iter;
127 : // Append and consume both non-null and null chunks.
128 0 : mRawSegment.AppendAndConsumeChunk(&chunk);
129 0 : iter.Next();
130 : }
131 :
132 0 : if (mRawSegment.GetDuration() >= GetPacketDuration()) {
133 0 : mReentrantMonitor.NotifyAll();
134 : }
135 :
136 0 : return NS_OK;
137 : }
138 :
139 : /*static*/
140 : void
141 0 : AudioTrackEncoder::InterleaveTrackData(AudioChunk& aChunk,
142 : int32_t aDuration,
143 : uint32_t aOutputChannels,
144 : AudioDataValue* aOutput)
145 : {
146 : uint32_t numChannelsToCopy = std::min(aOutputChannels,
147 0 : static_cast<uint32_t>(aChunk.mChannelData.Length()));
148 0 : switch(aChunk.mBufferFormat) {
149 : case AUDIO_FORMAT_S16: {
150 0 : AutoTArray<const int16_t*, 2> array;
151 0 : array.SetLength(numChannelsToCopy);
152 0 : for (uint32_t i = 0; i < array.Length(); i++) {
153 0 : array[i] = static_cast<const int16_t*>(aChunk.mChannelData[i]);
154 : }
155 0 : InterleaveTrackData(array, aDuration, aOutputChannels, aOutput, aChunk.mVolume);
156 0 : break;
157 : }
158 : case AUDIO_FORMAT_FLOAT32: {
159 0 : AutoTArray<const float*, 2> array;
160 0 : array.SetLength(numChannelsToCopy);
161 0 : for (uint32_t i = 0; i < array.Length(); i++) {
162 0 : array[i] = static_cast<const float*>(aChunk.mChannelData[i]);
163 : }
164 0 : InterleaveTrackData(array, aDuration, aOutputChannels, aOutput, aChunk.mVolume);
165 0 : break;
166 : }
167 : case AUDIO_FORMAT_SILENCE: {
168 0 : MOZ_ASSERT(false, "To implement.");
169 : }
170 : };
171 0 : }
172 :
173 : /*static*/
174 : void
175 0 : AudioTrackEncoder::DeInterleaveTrackData(AudioDataValue* aInput,
176 : int32_t aDuration,
177 : int32_t aChannels,
178 : AudioDataValue* aOutput)
179 : {
180 0 : for (int32_t i = 0; i < aChannels; ++i) {
181 0 : for(int32_t j = 0; j < aDuration; ++j) {
182 0 : aOutput[i * aDuration + j] = aInput[i + j * aChannels];
183 : }
184 : }
185 0 : }
186 :
187 : size_t
188 0 : AudioTrackEncoder::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
189 : {
190 0 : return mRawSegment.SizeOfExcludingThis(aMallocSizeOf);
191 : }
192 :
193 : void
194 0 : VideoTrackEncoder::Init(const VideoSegment& aSegment)
195 : {
196 0 : ReentrantMonitorAutoEnter mon(mReentrantMonitor);
197 :
198 0 : if (mInitialized) {
199 0 : return;
200 : }
201 :
202 0 : mInitCounter++;
203 0 : TRACK_LOG(LogLevel::Debug, ("Init the video encoder %d times", mInitCounter));
204 0 : VideoSegment::ConstChunkIterator iter(aSegment);
205 0 : while (!iter.IsEnded()) {
206 0 : VideoChunk chunk = *iter;
207 0 : if (!chunk.IsNull()) {
208 0 : gfx::IntSize imgsize = chunk.mFrame.GetImage()->GetSize();
209 0 : gfx::IntSize intrinsicSize = chunk.mFrame.GetIntrinsicSize();
210 0 : nsresult rv = Init(imgsize.width, imgsize.height,
211 0 : intrinsicSize.width, intrinsicSize.height);
212 :
213 0 : if (NS_FAILED(rv)) {
214 0 : TRACK_LOG(LogLevel::Error, ("[VideoTrackEncoder]: Fail to initialize the encoder!"));
215 0 : NotifyCancel();
216 : }
217 0 : break;
218 : }
219 :
220 0 : iter.Next();
221 : }
222 :
223 0 : mNotInitDuration += aSegment.GetDuration();
224 0 : if ((mNotInitDuration / mTrackRate > INIT_FAILED_DURATION) &&
225 0 : mInitCounter > 1) {
226 0 : TRACK_LOG(LogLevel::Debug, ("[VideoTrackEncoder]: Initialize failed for %ds.", INIT_FAILED_DURATION));
227 0 : NotifyEndOfStream();
228 0 : return;
229 : }
230 : }
231 :
232 : void
233 0 : VideoTrackEncoder::SetCurrentFrames(const VideoSegment& aSegment)
234 : {
235 0 : ReentrantMonitorAutoEnter mon(mReentrantMonitor);
236 :
237 0 : if (mCanceled) {
238 0 : return;
239 : }
240 :
241 0 : Init(aSegment);
242 0 : AppendVideoSegment(aSegment);
243 : }
244 :
245 : void
246 0 : VideoTrackEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
247 : TrackID aID,
248 : StreamTime aTrackOffset,
249 : uint32_t aTrackEvents,
250 : const MediaSegment& aQueuedMedia)
251 : {
252 0 : ReentrantMonitorAutoEnter mon(mReentrantMonitor);
253 :
254 0 : if (mCanceled) {
255 0 : return;
256 : }
257 :
258 0 : if (!(aTrackEvents == TRACK_EVENT_CREATED ||
259 : aTrackEvents == TRACK_EVENT_ENDED)) {
260 0 : return;
261 : }
262 :
263 0 : const VideoSegment& video = static_cast<const VideoSegment&>(aQueuedMedia);
264 :
265 : // Check and initialize parameters for codec encoder.
266 0 : Init(video);
267 :
268 0 : AppendVideoSegment(video);
269 :
270 : // The stream has stopped and reached the end of track.
271 0 : if (aTrackEvents == TrackEventCommand::TRACK_EVENT_ENDED) {
272 0 : TRACK_LOG(LogLevel::Info, ("[VideoTrackEncoder]: Receive TRACK_EVENT_ENDED ."));
273 0 : NotifyEndOfStream();
274 : }
275 :
276 : }
277 :
278 : nsresult
279 0 : VideoTrackEncoder::AppendVideoSegment(const VideoSegment& aSegment)
280 : {
281 0 : ReentrantMonitorAutoEnter mon(mReentrantMonitor);
282 :
283 0 : if (mEndOfStream) {
284 0 : MOZ_ASSERT(false);
285 : return NS_OK;
286 : }
287 :
288 : // Append all video segments from MediaStreamGraph, including null an
289 : // non-null frames.
290 0 : VideoSegment::ConstChunkIterator iter(aSegment);
291 0 : for (; !iter.IsEnded(); iter.Next()) {
292 0 : VideoChunk chunk = *iter;
293 :
294 0 : if (mLastChunk.mTimeStamp.IsNull()) {
295 0 : if (chunk.IsNull()) {
296 : // The start of this track is frameless. We need to track the time
297 : // it takes to get the first frame.
298 0 : mLastChunk.mDuration += chunk.mDuration;
299 0 : continue;
300 : }
301 :
302 : // This is the first real chunk in the track. Use its timestamp as the
303 : // starting point for this track.
304 0 : MOZ_ASSERT(!chunk.mTimeStamp.IsNull());
305 0 : const StreamTime nullDuration = mLastChunk.mDuration;
306 0 : mLastChunk = chunk;
307 0 : chunk.mDuration = 0;
308 :
309 0 : TRACK_LOG(LogLevel::Verbose,
310 : ("[VideoTrackEncoder]: Got first video chunk after %" PRId64 " ticks.",
311 : nullDuration));
312 : // Adapt to the time before the first frame. This extends the first frame
313 : // from [start, end] to [0, end], but it'll do for now.
314 0 : auto diff = FramesToTimeUnit(nullDuration, mTrackRate);
315 0 : if (!diff.IsValid()) {
316 0 : NS_ERROR("null duration overflow");
317 0 : return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
318 : }
319 :
320 0 : mLastChunk.mTimeStamp -= diff.ToTimeDuration();
321 0 : mLastChunk.mDuration += nullDuration;
322 : }
323 :
324 0 : MOZ_ASSERT(!mLastChunk.IsNull());
325 0 : if (mLastChunk.CanCombineWithFollowing(chunk) || chunk.IsNull()) {
326 0 : TRACK_LOG(LogLevel::Verbose,
327 : ("[VideoTrackEncoder]: Got dupe or null chunk."));
328 : // This is the same frame as before (or null). We extend the last chunk
329 : // with its duration.
330 0 : mLastChunk.mDuration += chunk.mDuration;
331 :
332 0 : if (mLastChunk.mDuration < mTrackRate) {
333 0 : TRACK_LOG(LogLevel::Verbose,
334 : ("[VideoTrackEncoder]: Ignoring dupe/null chunk of duration %" PRId64,
335 : chunk.mDuration));
336 0 : continue;
337 : }
338 :
339 0 : TRACK_LOG(LogLevel::Verbose,
340 : ("[VideoTrackEncoder]: Chunk >1 second. duration=%" PRId64 ", "
341 : "trackRate=%" PRId32, mLastChunk.mDuration, mTrackRate));
342 :
343 : // If we have gotten dupes for over a second, we force send one
344 : // to the encoder to make sure there is some output.
345 0 : chunk.mTimeStamp = mLastChunk.mTimeStamp + TimeDuration::FromSeconds(1);
346 :
347 : // chunk's duration has already been accounted for.
348 0 : chunk.mDuration = 0;
349 :
350 0 : if (chunk.IsNull()) {
351 : // Ensure that we don't pass null to the encoder by making mLastChunk
352 : // null later on.
353 0 : chunk.mFrame = mLastChunk.mFrame;
354 : }
355 : }
356 :
357 0 : if (mStartOffset.IsNull()) {
358 0 : mStartOffset = mLastChunk.mTimeStamp;
359 : }
360 :
361 0 : TimeDuration relativeTime = chunk.mTimeStamp - mStartOffset;
362 0 : RefPtr<layers::Image> lastImage = mLastChunk.mFrame.GetImage();
363 0 : TRACK_LOG(LogLevel::Verbose,
364 : ("[VideoTrackEncoder]: Appending video frame %p, at pos %.5fs",
365 : lastImage.get(), relativeTime.ToSeconds()));
366 : CheckedInt64 totalDuration =
367 0 : UsecsToFrames(relativeTime.ToMicroseconds(), mTrackRate);
368 0 : if (!totalDuration.isValid()) {
369 0 : NS_ERROR("Duration overflow");
370 0 : return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
371 : }
372 :
373 0 : CheckedInt64 duration = totalDuration - mEncodedTicks;
374 0 : if (!duration.isValid()) {
375 0 : NS_ERROR("Duration overflow");
376 0 : return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
377 : }
378 :
379 0 : if (duration.isValid()) {
380 0 : if (duration.value() <= 0) {
381 : // The timestamp for mLastChunk is newer than for chunk.
382 : // This means the durations reported from MediaStreamGraph for
383 : // mLastChunk were larger than the timestamp diff - and durations were
384 : // used to trigger the 1-second frame above. This could happen due to
385 : // drift or underruns in the graph.
386 0 : TRACK_LOG(LogLevel::Warning,
387 : ("[VideoTrackEncoder]: Underrun detected. Diff=%" PRId64,
388 : duration.value()));
389 0 : chunk.mTimeStamp = mLastChunk.mTimeStamp;
390 : } else {
391 0 : mEncodedTicks += duration.value();
392 0 : mRawSegment.AppendFrame(lastImage.forget(),
393 : duration.value(),
394 0 : mLastChunk.mFrame.GetIntrinsicSize(),
395 : PRINCIPAL_HANDLE_NONE,
396 0 : mLastChunk.mFrame.GetForceBlack(),
397 0 : mLastChunk.mTimeStamp);
398 : }
399 : }
400 :
401 0 : mLastChunk = chunk;
402 : }
403 :
404 0 : if (mRawSegment.GetDuration() > 0) {
405 0 : mReentrantMonitor.NotifyAll();
406 : }
407 :
408 0 : return NS_OK;
409 : }
410 :
411 : void
412 0 : VideoTrackEncoder::NotifyEndOfStream()
413 : {
414 0 : ReentrantMonitorAutoEnter mon(mReentrantMonitor);
415 :
416 : // If source video track is muted till the end of encoding, initialize the
417 : // encoder with default frame width, frame height, and track rate.
418 0 : if (!mCanceled && !mInitialized) {
419 : Init(DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT,
420 0 : DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT);
421 : }
422 :
423 0 : if (mEndOfStream) {
424 : // We have already been notified.
425 0 : return;
426 : }
427 :
428 0 : mEndOfStream = true;
429 0 : TRACK_LOG(LogLevel::Info, ("[VideoTrackEncoder]: Reached end of stream"));
430 :
431 0 : if (!mLastChunk.IsNull() && mLastChunk.mDuration > 0) {
432 0 : RefPtr<layers::Image> lastImage = mLastChunk.mFrame.GetImage();
433 0 : TRACK_LOG(LogLevel::Debug,
434 : ("[VideoTrackEncoder]: Appending last video frame %p, "
435 : "duration=%.5f", lastImage.get(),
436 : FramesToTimeUnit(mLastChunk.mDuration, mTrackRate).ToSeconds()));
437 0 : mRawSegment.AppendFrame(lastImage.forget(),
438 : mLastChunk.mDuration,
439 0 : mLastChunk.mFrame.GetIntrinsicSize(),
440 : PRINCIPAL_HANDLE_NONE,
441 0 : mLastChunk.mFrame.GetForceBlack(),
442 0 : mLastChunk.mTimeStamp);
443 : }
444 0 : mReentrantMonitor.NotifyAll();
445 : }
446 :
447 : size_t
448 0 : VideoTrackEncoder::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
449 : {
450 0 : return mRawSegment.SizeOfExcludingThis(aMallocSizeOf);
451 : }
452 :
453 : } // namespace mozilla
|
