Line data Source code
1 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 : /* This Source Code Form is subject to the terms of the Mozilla Public
3 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
5 : #include "MediaEncoder.h"
6 : #include "MediaDecoder.h"
7 : #include "nsIPrincipal.h"
8 : #include "nsMimeTypes.h"
9 : #include "TimeUnits.h"
10 : #include "mozilla/Logging.h"
11 : #include "mozilla/Preferences.h"
12 : #include "mozilla/StaticPtr.h"
13 : #include "mozilla/gfx/Point.h" // IntSize
14 :
15 : #include"GeckoProfiler.h"
16 : #include "OggWriter.h"
17 : #include "OpusTrackEncoder.h"
18 :
19 : #ifdef MOZ_WEBM_ENCODER
20 : #include "VP8TrackEncoder.h"
21 : #include "WebMWriter.h"
22 : #endif
23 :
24 : #ifdef LOG
25 : #undef LOG
26 : #endif
27 :
28 : mozilla::LazyLogModule gMediaEncoderLog("MediaEncoder");
29 : #define LOG(type, msg) MOZ_LOG(gMediaEncoderLog, type, msg)
30 :
31 : namespace mozilla {
32 :
33 : void
34 0 : MediaStreamVideoRecorderSink::SetCurrentFrames(const VideoSegment& aSegment)
35 : {
36 0 : MOZ_ASSERT(mVideoEncoder);
37 : // If we're suspended (paused) we don't forward frames
38 0 : if (!mSuspended) {
39 0 : mVideoEncoder->SetCurrentFrames(aSegment);
40 : }
41 0 : }
42 :
43 : void
44 0 : MediaEncoder::Suspend()
45 : {
46 0 : MOZ_ASSERT(NS_IsMainThread());
47 0 : mLastPauseStartTime = TimeStamp::Now();
48 0 : mSuspended = true;
49 0 : mVideoSink->Suspend();
50 0 : }
51 :
52 : void
53 0 : MediaEncoder::Resume()
54 : {
55 0 : MOZ_ASSERT(NS_IsMainThread());
56 0 : if (!mSuspended) {
57 0 : return;
58 : }
59 : media::TimeUnit timeSpentPaused =
60 : media::TimeUnit::FromTimeDuration(
61 0 : TimeStamp::Now() - mLastPauseStartTime);
62 0 : MOZ_ASSERT(timeSpentPaused.ToMicroseconds() >= 0);
63 0 : MOZ_RELEASE_ASSERT(timeSpentPaused.IsValid());
64 0 : mMicrosecondsSpentPaused += timeSpentPaused.ToMicroseconds();;
65 0 : mSuspended = false;
66 0 : mVideoSink->Resume();
67 : }
68 :
69 : void
70 0 : MediaEncoder::SetDirectConnect(bool aConnected)
71 : {
72 0 : mDirectConnected = aConnected;
73 0 : }
74 :
75 : void
76 0 : MediaEncoder::NotifyRealtimeData(MediaStreamGraph* aGraph,
77 : TrackID aID,
78 : StreamTime aTrackOffset,
79 : uint32_t aTrackEvents,
80 : const MediaSegment& aRealtimeMedia)
81 : {
82 0 : if (mSuspended) {
83 0 : return;
84 : }
85 : // Process the incoming raw track data from MediaStreamGraph, called on the
86 : // thread of MediaStreamGraph.
87 0 : if (mAudioEncoder && aRealtimeMedia.GetType() == MediaSegment::AUDIO) {
88 0 : mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID,
89 : aTrackOffset, aTrackEvents,
90 0 : aRealtimeMedia);
91 0 : } else if (mVideoEncoder &&
92 0 : aRealtimeMedia.GetType() == MediaSegment::VIDEO &&
93 : aTrackEvents != TrackEventCommand::TRACK_EVENT_NONE) {
94 0 : mVideoEncoder->NotifyQueuedTrackChanges(aGraph, aID,
95 : aTrackOffset, aTrackEvents,
96 0 : aRealtimeMedia);
97 : }
98 : }
99 :
100 : void
101 0 : MediaEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
102 : TrackID aID,
103 : StreamTime aTrackOffset,
104 : TrackEventCommand aTrackEvents,
105 : const MediaSegment& aQueuedMedia,
106 : MediaStream* aInputStream,
107 : TrackID aInputTrackID)
108 : {
109 0 : if (!mDirectConnected) {
110 0 : NotifyRealtimeData(aGraph, aID, aTrackOffset, aTrackEvents, aQueuedMedia);
111 : } else {
112 0 : if (aTrackEvents != TrackEventCommand::TRACK_EVENT_NONE) {
113 : // forward events (TRACK_EVENT_ENDED) but not the media
114 0 : if (aQueuedMedia.GetType() == MediaSegment::VIDEO) {
115 0 : VideoSegment segment;
116 0 : NotifyRealtimeData(aGraph, aID, aTrackOffset, aTrackEvents, segment);
117 : } else {
118 0 : AudioSegment segment;
119 0 : NotifyRealtimeData(aGraph, aID, aTrackOffset, aTrackEvents, segment);
120 : }
121 : }
122 : }
123 0 : }
124 :
125 : void
126 0 : MediaEncoder::NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
127 : StreamTime aTrackOffset,
128 : const AudioSegment& aQueuedMedia,
129 : MediaStream* aInputStream,
130 : TrackID aInputTrackID)
131 : {
132 0 : if (!mDirectConnected) {
133 0 : NotifyRealtimeData(aGraph, aID, aTrackOffset, 0, aQueuedMedia);
134 : }
135 0 : }
136 :
137 : void
138 0 : MediaEncoder::NotifyEvent(MediaStreamGraph* aGraph,
139 : MediaStreamGraphEvent event)
140 : {
141 : // In case that MediaEncoder does not receive a TRACK_EVENT_ENDED event.
142 0 : LOG(LogLevel::Debug, ("NotifyRemoved in [MediaEncoder]."));
143 0 : if (mAudioEncoder) {
144 0 : mAudioEncoder->NotifyEvent(aGraph, event);
145 : }
146 0 : if (mVideoEncoder) {
147 0 : mVideoEncoder->NotifyEvent(aGraph, event);
148 : }
149 0 : }
150 :
151 : /* static */
152 : already_AddRefed<MediaEncoder>
153 0 : MediaEncoder::CreateEncoder(const nsAString& aMIMEType, uint32_t aAudioBitrate,
154 : uint32_t aVideoBitrate, uint32_t aBitrate,
155 : uint8_t aTrackTypes,
156 : TrackRate aTrackRate)
157 : {
158 0 : AUTO_PROFILER_LABEL("MediaEncoder::CreateEncoder", OTHER);
159 :
160 0 : nsAutoPtr<ContainerWriter> writer;
161 0 : nsAutoPtr<AudioTrackEncoder> audioEncoder;
162 0 : nsAutoPtr<VideoTrackEncoder> videoEncoder;
163 0 : RefPtr<MediaEncoder> encoder;
164 0 : nsString mimeType;
165 0 : if (!aTrackTypes) {
166 0 : LOG(LogLevel::Error, ("NO TrackTypes!!!"));
167 0 : return nullptr;
168 : }
169 : #ifdef MOZ_WEBM_ENCODER
170 0 : else if (MediaEncoder::IsWebMEncoderEnabled() &&
171 0 : (aMIMEType.EqualsLiteral(VIDEO_WEBM) ||
172 0 : (aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK))) {
173 0 : if (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK
174 0 : && MediaDecoder::IsOpusEnabled()) {
175 0 : audioEncoder = new OpusTrackEncoder();
176 0 : NS_ENSURE_TRUE(audioEncoder, nullptr);
177 : }
178 0 : videoEncoder = new VP8TrackEncoder(aTrackRate);
179 0 : writer = new WebMWriter(aTrackTypes);
180 0 : NS_ENSURE_TRUE(writer, nullptr);
181 0 : NS_ENSURE_TRUE(videoEncoder, nullptr);
182 0 : mimeType = NS_LITERAL_STRING(VIDEO_WEBM);
183 : }
184 : #endif //MOZ_WEBM_ENCODER
185 0 : else if (MediaDecoder::IsOggEnabled() && MediaDecoder::IsOpusEnabled() &&
186 0 : (aMIMEType.EqualsLiteral(AUDIO_OGG) ||
187 0 : (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK))) {
188 0 : writer = new OggWriter();
189 0 : audioEncoder = new OpusTrackEncoder();
190 0 : NS_ENSURE_TRUE(writer, nullptr);
191 0 : NS_ENSURE_TRUE(audioEncoder, nullptr);
192 0 : mimeType = NS_LITERAL_STRING(AUDIO_OGG);
193 : }
194 : else {
195 0 : LOG(LogLevel::Error, ("Can not find any encoder to record this media stream"));
196 0 : return nullptr;
197 : }
198 0 : LOG(LogLevel::Debug, ("Create encoder result:a[%d] v[%d] w[%d] mimeType = %s.",
199 : audioEncoder != nullptr, videoEncoder != nullptr,
200 : writer != nullptr, NS_ConvertUTF16toUTF8(mimeType).get()));
201 0 : if (videoEncoder && aVideoBitrate != 0) {
202 0 : videoEncoder->SetBitrate(aVideoBitrate);
203 : }
204 0 : if (audioEncoder && aAudioBitrate != 0) {
205 0 : audioEncoder->SetBitrate(aAudioBitrate);
206 : }
207 0 : encoder = new MediaEncoder(writer.forget(), audioEncoder.forget(),
208 0 : videoEncoder.forget(), mimeType, aAudioBitrate,
209 0 : aVideoBitrate, aBitrate);
210 0 : return encoder.forget();
211 : }
212 :
213 : /**
214 : * GetEncodedData() runs as a state machine, starting with mState set to
215 : * GET_METADDATA, the procedure should be as follow:
216 : *
217 : * While non-stop
218 : * If mState is GET_METADDATA
219 : * Get the meta data from audio/video encoder
220 : * If a meta data is generated
221 : * Get meta data from audio/video encoder
222 : * Set mState to ENCODE_TRACK
223 : * Return the final container data
224 : *
225 : * If mState is ENCODE_TRACK
226 : * Get encoded track data from audio/video encoder
227 : * If a packet of track data is generated
228 : * Insert encoded track data into the container stream of writer
229 : * If the final container data is copied to aOutput
230 : * Return the copy of final container data
231 : * If this is the last packet of input stream
232 : * Set mState to ENCODE_DONE
233 : *
234 : * If mState is ENCODE_DONE or ENCODE_ERROR
235 : * Stop the loop
236 : */
237 : void
238 0 : MediaEncoder::GetEncodedData(nsTArray<nsTArray<uint8_t> >* aOutputBufs,
239 : nsAString& aMIMEType)
240 : {
241 0 : MOZ_ASSERT(!NS_IsMainThread());
242 :
243 0 : aMIMEType = mMIMEType;
244 0 : AUTO_PROFILER_LABEL("MediaEncoder::GetEncodedData", OTHER);
245 :
246 0 : bool reloop = true;
247 0 : while (reloop) {
248 0 : switch (mState) {
249 : case ENCODE_METADDATA: {
250 0 : LOG(LogLevel::Debug, ("ENCODE_METADDATA TimeStamp = %f", GetEncodeTimeStamp()));
251 0 : nsresult rv = CopyMetadataToMuxer(mAudioEncoder.get());
252 0 : if (NS_FAILED(rv)) {
253 0 : LOG(LogLevel::Error, ("Error! Fail to Set Audio Metadata"));
254 0 : break;
255 : }
256 0 : rv = CopyMetadataToMuxer(mVideoEncoder.get());
257 0 : if (NS_FAILED(rv)) {
258 0 : LOG(LogLevel::Error, ("Error! Fail to Set Video Metadata"));
259 0 : break;
260 : }
261 :
262 0 : rv = mWriter->GetContainerData(aOutputBufs,
263 0 : ContainerWriter::GET_HEADER);
264 0 : if (aOutputBufs != nullptr) {
265 0 : mSizeOfBuffer = aOutputBufs->ShallowSizeOfExcludingThis(MallocSizeOf);
266 : }
267 0 : if (NS_FAILED(rv)) {
268 0 : LOG(LogLevel::Error,("Error! writer fail to generate header!"));
269 0 : mState = ENCODE_ERROR;
270 0 : break;
271 : }
272 0 : LOG(LogLevel::Debug, ("Finish ENCODE_METADDATA TimeStamp = %f", GetEncodeTimeStamp()));
273 0 : mState = ENCODE_TRACK;
274 0 : break;
275 : }
276 :
277 : case ENCODE_TRACK: {
278 0 : LOG(LogLevel::Debug, ("ENCODE_TRACK TimeStamp = %f", GetEncodeTimeStamp()));
279 0 : EncodedFrameContainer encodedData;
280 0 : nsresult rv = NS_OK;
281 : // We're most likely to actually wait for a video frame, so do that first to minimize
282 : // capture offset/lipsync issues
283 0 : rv = WriteEncodedDataToMuxer(mVideoEncoder.get());
284 0 : if (NS_FAILED(rv)) {
285 0 : LOG(LogLevel::Error, ("Fail to write video encoder data to muxer"));
286 0 : break;
287 : }
288 0 : rv = WriteEncodedDataToMuxer(mAudioEncoder.get());
289 0 : if (NS_FAILED(rv)) {
290 0 : LOG(LogLevel::Error, ("Error! Fail to write audio encoder data to muxer"));
291 0 : break;
292 : }
293 0 : LOG(LogLevel::Debug, ("Audio encoded TimeStamp = %f", GetEncodeTimeStamp()));
294 0 : LOG(LogLevel::Debug, ("Video encoded TimeStamp = %f", GetEncodeTimeStamp()));
295 : // In audio only or video only case, let unavailable track's flag to be true.
296 0 : bool isAudioCompleted = (mAudioEncoder && mAudioEncoder->IsEncodingComplete()) || !mAudioEncoder;
297 0 : bool isVideoCompleted = (mVideoEncoder && mVideoEncoder->IsEncodingComplete()) || !mVideoEncoder;
298 0 : rv = mWriter->GetContainerData(aOutputBufs,
299 0 : isAudioCompleted && isVideoCompleted ?
300 0 : ContainerWriter::FLUSH_NEEDED : 0);
301 0 : if (aOutputBufs != nullptr) {
302 0 : mSizeOfBuffer = aOutputBufs->ShallowSizeOfExcludingThis(MallocSizeOf);
303 : }
304 0 : if (NS_SUCCEEDED(rv)) {
305 : // Successfully get the copy of final container data from writer.
306 0 : reloop = false;
307 : }
308 0 : mState = (mWriter->IsWritingComplete()) ? ENCODE_DONE : ENCODE_TRACK;
309 0 : LOG(LogLevel::Debug, ("END ENCODE_TRACK TimeStamp = %f "
310 : "mState = %d aComplete %d vComplete %d",
311 : GetEncodeTimeStamp(), mState, isAudioCompleted, isVideoCompleted));
312 0 : break;
313 : }
314 :
315 : case ENCODE_DONE:
316 : case ENCODE_ERROR:
317 0 : LOG(LogLevel::Debug, ("MediaEncoder has been shutdown."));
318 0 : mSizeOfBuffer = 0;
319 0 : mShutdown = true;
320 0 : reloop = false;
321 0 : break;
322 : default:
323 0 : MOZ_CRASH("Invalid encode state");
324 : }
325 : }
326 0 : }
327 :
328 : nsresult
329 0 : MediaEncoder::WriteEncodedDataToMuxer(TrackEncoder *aTrackEncoder)
330 : {
331 0 : if (aTrackEncoder == nullptr) {
332 0 : return NS_OK;
333 : }
334 0 : if (aTrackEncoder->IsEncodingComplete()) {
335 0 : return NS_OK;
336 : }
337 :
338 0 : AUTO_PROFILER_LABEL("MediaEncoder::WriteEncodedDataToMuxer", OTHER);
339 :
340 0 : EncodedFrameContainer encodedVideoData;
341 0 : nsresult rv = aTrackEncoder->GetEncodedTrack(encodedVideoData);
342 0 : if (NS_FAILED(rv)) {
343 : // Encoding might be canceled.
344 0 : LOG(LogLevel::Error, ("Error! Fail to get encoded data from video encoder."));
345 0 : mState = ENCODE_ERROR;
346 0 : return rv;
347 : }
348 :
349 : // Update timestamps to accommodate pauses
350 : const nsTArray<RefPtr<EncodedFrame> >& encodedFrames =
351 0 : encodedVideoData.GetEncodedFrames();
352 : // Take a copy of the atomic so we don't continually access it
353 0 : uint64_t microsecondsSpentPaused = mMicrosecondsSpentPaused;
354 0 : for (size_t i = 0; i < encodedFrames.Length(); ++i) {
355 0 : RefPtr<EncodedFrame> frame = encodedFrames[i];
356 0 : if (frame->GetTimeStamp() > microsecondsSpentPaused &&
357 0 : frame->GetTimeStamp() - microsecondsSpentPaused > mLastMuxedTimestamp) {
358 : // Use the adjusted timestamp if it's after the last timestamp
359 0 : frame->SetTimeStamp(frame->GetTimeStamp() - microsecondsSpentPaused);
360 : } else {
361 : // If not, we force the last time stamp. We do this so the frames are
362 : // still around and in order in case the codec needs to reference them.
363 : // Dropping them here may result in artifacts in playback.
364 0 : frame->SetTimeStamp(mLastMuxedTimestamp);
365 : }
366 0 : MOZ_ASSERT(mLastMuxedTimestamp <= frame->GetTimeStamp(),
367 : "Our frames should be ordered by this point!");
368 0 : mLastMuxedTimestamp = frame->GetTimeStamp();
369 : }
370 :
371 0 : rv = mWriter->WriteEncodedTrack(encodedVideoData,
372 0 : aTrackEncoder->IsEncodingComplete() ?
373 0 : ContainerWriter::END_OF_STREAM : 0);
374 0 : if (NS_FAILED(rv)) {
375 0 : LOG(LogLevel::Error, ("Error! Fail to write encoded video track to the media container."));
376 0 : mState = ENCODE_ERROR;
377 : }
378 0 : return rv;
379 : }
380 :
381 : nsresult
382 0 : MediaEncoder::CopyMetadataToMuxer(TrackEncoder *aTrackEncoder)
383 : {
384 0 : if (aTrackEncoder == nullptr) {
385 0 : return NS_OK;
386 : }
387 :
388 0 : AUTO_PROFILER_LABEL("MediaEncoder::CopyMetadataToMuxer", OTHER);
389 :
390 0 : RefPtr<TrackMetadataBase> meta = aTrackEncoder->GetMetadata();
391 0 : if (meta == nullptr) {
392 0 : LOG(LogLevel::Error, ("Error! metadata = null"));
393 0 : mState = ENCODE_ERROR;
394 0 : return NS_ERROR_ABORT;
395 : }
396 :
397 0 : nsresult rv = mWriter->SetMetadata(meta);
398 0 : if (NS_FAILED(rv)) {
399 0 : LOG(LogLevel::Error, ("Error! SetMetadata fail"));
400 0 : mState = ENCODE_ERROR;
401 : }
402 0 : return rv;
403 : }
404 :
405 : #ifdef MOZ_WEBM_ENCODER
406 : bool
407 0 : MediaEncoder::IsWebMEncoderEnabled()
408 : {
409 0 : return Preferences::GetBool("media.encoder.webm.enabled");
410 : }
411 : #endif
412 :
413 : /*
414 : * SizeOfExcludingThis measures memory being used by the Media Encoder.
415 : * Currently it measures the size of the Encoder buffer and memory occupied
416 : * by mAudioEncoder and mVideoEncoder.
417 : */
418 : size_t
419 0 : MediaEncoder::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
420 : {
421 0 : size_t amount = 0;
422 0 : if (mState == ENCODE_TRACK) {
423 0 : amount = mSizeOfBuffer +
424 0 : (mAudioEncoder != nullptr ? mAudioEncoder->SizeOfExcludingThis(aMallocSizeOf) : 0) +
425 0 : (mVideoEncoder != nullptr ? mVideoEncoder->SizeOfExcludingThis(aMallocSizeOf) : 0);
426 : }
427 0 : return amount;
428 : }
429 :
430 : } // namespace mozilla
|
