Line data Source code
1 : /* This Source Code Form is subject to the terms of the Mozilla Public
2 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
4 :
5 : #include "MediaEngineCameraVideoSource.h"
6 :
7 : #include "mozilla/IntegerPrintfMacros.h"
8 : #include "mozilla/SizePrintfMacros.h"
9 :
10 : #include <limits>
11 :
12 : namespace mozilla {
13 :
14 : using namespace mozilla::gfx;
15 : using namespace mozilla::dom;
16 :
17 : extern LogModule* GetMediaManagerLog();
18 : #define LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg)
19 : #define LOGFRAME(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Verbose, msg)
20 :
21 : // guts for appending data to the MSG track
22 0 : bool MediaEngineCameraVideoSource::AppendToTrack(SourceMediaStream* aSource,
23 : layers::Image* aImage,
24 : TrackID aID,
25 : StreamTime delta,
26 : const PrincipalHandle& aPrincipalHandle)
27 : {
28 0 : MOZ_ASSERT(aSource);
29 :
30 0 : VideoSegment segment;
31 0 : RefPtr<layers::Image> image = aImage;
32 0 : IntSize size(image ? mWidth : 0, image ? mHeight : 0);
33 0 : segment.AppendFrame(image.forget(), delta, size, aPrincipalHandle);
34 :
35 : // This is safe from any thread, and is safe if the track is Finished
36 : // or Destroyed.
37 : // This can fail if either a) we haven't added the track yet, or b)
38 : // we've removed or finished the track.
39 0 : return aSource->AppendToTrack(aID, &(segment));
40 : }
41 :
42 : // Sub-classes (B2G or desktop) should overload one of both of these two methods
43 : // to provide capabilities
44 : size_t
45 0 : MediaEngineCameraVideoSource::NumCapabilities() const
46 : {
47 0 : return mHardcodedCapabilities.Length();
48 : }
49 :
50 : void
51 0 : MediaEngineCameraVideoSource::GetCapability(size_t aIndex,
52 : webrtc::CaptureCapability& aOut) const
53 : {
54 0 : MOZ_ASSERT(aIndex < mHardcodedCapabilities.Length());
55 0 : aOut = mHardcodedCapabilities.SafeElementAt(aIndex, webrtc::CaptureCapability());
56 0 : }
57 :
58 : uint32_t
59 0 : MediaEngineCameraVideoSource::GetFitnessDistance(
60 : const webrtc::CaptureCapability& aCandidate,
61 : const NormalizedConstraintSet &aConstraints,
62 : const nsString& aDeviceId) const
63 : {
64 : // Treat width|height|frameRate == 0 on capability as "can do any".
65 : // This allows for orthogonal capabilities that are not in discrete steps.
66 :
67 : uint64_t distance =
68 0 : uint64_t(FitnessDistance(aDeviceId, aConstraints.mDeviceId)) +
69 0 : uint64_t(FitnessDistance(mFacingMode, aConstraints.mFacingMode)) +
70 0 : uint64_t(aCandidate.width? FitnessDistance(int32_t(aCandidate.width),
71 0 : aConstraints.mWidth) : 0) +
72 0 : uint64_t(aCandidate.height? FitnessDistance(int32_t(aCandidate.height),
73 0 : aConstraints.mHeight) : 0) +
74 0 : uint64_t(aCandidate.maxFPS? FitnessDistance(double(aCandidate.maxFPS),
75 0 : aConstraints.mFrameRate) : 0);
76 0 : return uint32_t(std::min(distance, uint64_t(UINT32_MAX)));
77 : }
78 :
79 : // Find best capability by removing inferiors. May leave >1 of equal distance
80 :
81 : /* static */ void
82 0 : MediaEngineCameraVideoSource::TrimLessFitCandidates(CapabilitySet& set) {
83 0 : uint32_t best = UINT32_MAX;
84 0 : for (auto& candidate : set) {
85 0 : if (best > candidate.mDistance) {
86 0 : best = candidate.mDistance;
87 : }
88 : }
89 0 : for (size_t i = 0; i < set.Length();) {
90 0 : if (set[i].mDistance > best) {
91 0 : set.RemoveElementAt(i);
92 : } else {
93 0 : ++i;
94 : }
95 : }
96 0 : MOZ_ASSERT(set.Length());
97 0 : }
98 :
99 : // GetBestFitnessDistance returns the best distance the capture device can offer
100 : // as a whole, given an accumulated number of ConstraintSets.
101 : // Ideal values are considered in the first ConstraintSet only.
102 : // Plain values are treated as Ideal in the first ConstraintSet.
103 : // Plain values are treated as Exact in subsequent ConstraintSets.
104 : // Infinity = UINT32_MAX e.g. device cannot satisfy accumulated ConstraintSets.
105 : // A finite result may be used to calculate this device's ranking as a choice.
106 :
107 : uint32_t
108 0 : MediaEngineCameraVideoSource::GetBestFitnessDistance(
109 : const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
110 : const nsString& aDeviceId) const
111 : {
112 0 : size_t num = NumCapabilities();
113 :
114 0 : CapabilitySet candidateSet;
115 0 : for (size_t i = 0; i < num; i++) {
116 0 : candidateSet.AppendElement(i);
117 : }
118 :
119 0 : bool first = true;
120 0 : for (const NormalizedConstraintSet* ns : aConstraintSets) {
121 0 : for (size_t i = 0; i < candidateSet.Length(); ) {
122 0 : auto& candidate = candidateSet[i];
123 0 : webrtc::CaptureCapability cap;
124 0 : GetCapability(candidate.mIndex, cap);
125 0 : uint32_t distance = GetFitnessDistance(cap, *ns, aDeviceId);
126 0 : if (distance == UINT32_MAX) {
127 0 : candidateSet.RemoveElementAt(i);
128 : } else {
129 0 : ++i;
130 0 : if (first) {
131 0 : candidate.mDistance = distance;
132 : }
133 : }
134 : }
135 0 : first = false;
136 : }
137 0 : if (!candidateSet.Length()) {
138 0 : return UINT32_MAX;
139 : }
140 0 : TrimLessFitCandidates(candidateSet);
141 0 : return candidateSet[0].mDistance;
142 : }
143 :
144 : void
145 0 : MediaEngineCameraVideoSource::LogConstraints(
146 : const NormalizedConstraintSet& aConstraints)
147 : {
148 0 : auto& c = aConstraints;
149 0 : if (c.mWidth.mIdeal.isSome()) {
150 0 : LOG(("Constraints: width: { min: %d, max: %d, ideal: %d }",
151 : c.mWidth.mMin, c.mWidth.mMax,
152 : c.mWidth.mIdeal.valueOr(0)));
153 : } else {
154 0 : LOG(("Constraints: width: { min: %d, max: %d }",
155 : c.mWidth.mMin, c.mWidth.mMax));
156 : }
157 0 : if (c.mHeight.mIdeal.isSome()) {
158 0 : LOG((" height: { min: %d, max: %d, ideal: %d }",
159 : c.mHeight.mMin, c.mHeight.mMax,
160 : c.mHeight.mIdeal.valueOr(0)));
161 : } else {
162 0 : LOG((" height: { min: %d, max: %d }",
163 : c.mHeight.mMin, c.mHeight.mMax));
164 : }
165 0 : if (c.mFrameRate.mIdeal.isSome()) {
166 0 : LOG((" frameRate: { min: %f, max: %f, ideal: %f }",
167 : c.mFrameRate.mMin, c.mFrameRate.mMax,
168 : c.mFrameRate.mIdeal.valueOr(0)));
169 : } else {
170 0 : LOG((" frameRate: { min: %f, max: %f }",
171 : c.mFrameRate.mMin, c.mFrameRate.mMax));
172 : }
173 0 : }
174 :
175 : void
176 0 : MediaEngineCameraVideoSource::LogCapability(const char* aHeader,
177 : const webrtc::CaptureCapability &aCapability, uint32_t aDistance)
178 : {
179 : // RawVideoType and VideoCodecType media/webrtc/trunk/webrtc/common_types.h
180 : static const char* const types[] = {
181 : "I420",
182 : "YV12",
183 : "YUY2",
184 : "UYVY",
185 : "IYUV",
186 : "ARGB",
187 : "RGB24",
188 : "RGB565",
189 : "ARGB4444",
190 : "ARGB1555",
191 : "MJPEG",
192 : "NV12",
193 : "NV21",
194 : "BGRA",
195 : "Unknown type"
196 : };
197 :
198 : static const char* const codec[] = {
199 : "VP8",
200 : "VP9",
201 : "H264",
202 : "I420",
203 : "RED",
204 : "ULPFEC",
205 : "Generic codec",
206 : "Unknown codec"
207 : };
208 :
209 0 : LOG(("%s: %4u x %4u x %2u maxFps, %s, %s. Distance = %" PRIu32,
210 : aHeader, aCapability.width, aCapability.height, aCapability.maxFPS,
211 : types[std::min(std::max(uint32_t(0), uint32_t(aCapability.rawType)),
212 : uint32_t(sizeof(types) / sizeof(*types) - 1))],
213 : codec[std::min(std::max(uint32_t(0), uint32_t(aCapability.codecType)),
214 : uint32_t(sizeof(codec) / sizeof(*codec) - 1))],
215 : aDistance));
216 0 : }
217 :
218 : bool
219 0 : MediaEngineCameraVideoSource::ChooseCapability(
220 : const NormalizedConstraints &aConstraints,
221 : const MediaEnginePrefs &aPrefs,
222 : const nsString& aDeviceId)
223 : {
224 0 : if (MOZ_LOG_TEST(GetMediaManagerLog(), LogLevel::Debug)) {
225 0 : LOG(("ChooseCapability: prefs: %dx%d @%d-%dfps",
226 : aPrefs.GetWidth(), aPrefs.GetHeight(),
227 : aPrefs.mFPS, aPrefs.mMinFPS));
228 0 : LogConstraints(aConstraints);
229 0 : if (aConstraints.mAdvanced.size()) {
230 0 : LOG(("Advanced array[%" PRIuSIZE "]:", aConstraints.mAdvanced.size()));
231 0 : for (auto& advanced : aConstraints.mAdvanced) {
232 0 : LogConstraints(advanced);
233 : }
234 : }
235 : }
236 :
237 0 : size_t num = NumCapabilities();
238 :
239 0 : CapabilitySet candidateSet;
240 0 : for (size_t i = 0; i < num; i++) {
241 0 : candidateSet.AppendElement(i);
242 : }
243 :
244 : // First, filter capabilities by required constraints (min, max, exact).
245 :
246 0 : for (size_t i = 0; i < candidateSet.Length();) {
247 0 : auto& candidate = candidateSet[i];
248 0 : webrtc::CaptureCapability cap;
249 0 : GetCapability(candidate.mIndex, cap);
250 0 : candidate.mDistance = GetFitnessDistance(cap, aConstraints, aDeviceId);
251 0 : LogCapability("Capability", cap, candidate.mDistance);
252 0 : if (candidate.mDistance == UINT32_MAX) {
253 0 : candidateSet.RemoveElementAt(i);
254 : } else {
255 0 : ++i;
256 : }
257 : }
258 :
259 0 : if (!candidateSet.Length()) {
260 0 : LOG(("failed to find capability match from %" PRIuSIZE " choices",num));
261 0 : return false;
262 : }
263 :
264 : // Filter further with all advanced constraints (that don't overconstrain).
265 :
266 0 : for (const auto &cs : aConstraints.mAdvanced) {
267 0 : CapabilitySet rejects;
268 0 : for (size_t i = 0; i < candidateSet.Length();) {
269 0 : auto& candidate = candidateSet[i];
270 0 : webrtc::CaptureCapability cap;
271 0 : GetCapability(candidate.mIndex, cap);
272 0 : if (GetFitnessDistance(cap, cs, aDeviceId) == UINT32_MAX) {
273 0 : rejects.AppendElement(candidate);
274 0 : candidateSet.RemoveElementAt(i);
275 : } else {
276 0 : ++i;
277 : }
278 : }
279 0 : if (!candidateSet.Length()) {
280 0 : candidateSet.AppendElements(Move(rejects));
281 : }
282 : }
283 0 : MOZ_ASSERT(candidateSet.Length(),
284 : "advanced constraints filtering step can't reduce candidates to zero");
285 :
286 : // Remaining algorithm is up to the UA.
287 :
288 0 : TrimLessFitCandidates(candidateSet);
289 :
290 : // Any remaining multiples all have the same distance. A common case of this
291 : // occurs when no ideal is specified. Lean toward defaults.
292 0 : uint32_t sameDistance = candidateSet[0].mDistance;
293 : {
294 0 : MediaTrackConstraintSet prefs;
295 0 : prefs.mWidth.SetAsLong() = aPrefs.GetWidth();
296 0 : prefs.mHeight.SetAsLong() = aPrefs.GetHeight();
297 0 : prefs.mFrameRate.SetAsDouble() = aPrefs.mFPS;
298 0 : NormalizedConstraintSet normPrefs(prefs, false);
299 :
300 0 : for (auto& candidate : candidateSet) {
301 0 : webrtc::CaptureCapability cap;
302 0 : GetCapability(candidate.mIndex, cap);
303 0 : candidate.mDistance = GetFitnessDistance(cap, normPrefs, aDeviceId);
304 : }
305 0 : TrimLessFitCandidates(candidateSet);
306 : }
307 :
308 : // Any remaining multiples all have the same distance, but may vary on
309 : // format. Some formats are more desirable for certain use like WebRTC.
310 : // E.g. I420 over RGB24 can remove a needless format conversion.
311 :
312 0 : bool found = false;
313 0 : for (auto& candidate : candidateSet) {
314 0 : webrtc::CaptureCapability cap;
315 0 : GetCapability(candidate.mIndex, cap);
316 0 : if (cap.rawType == webrtc::RawVideoType::kVideoI420 ||
317 0 : cap.rawType == webrtc::RawVideoType::kVideoYUY2 ||
318 0 : cap.rawType == webrtc::RawVideoType::kVideoYV12) {
319 0 : mCapability = cap;
320 0 : found = true;
321 0 : break;
322 : }
323 : }
324 0 : if (!found) {
325 0 : GetCapability(candidateSet[0].mIndex, mCapability);
326 : }
327 :
328 0 : LogCapability("Chosen capability", mCapability, sameDistance);
329 0 : return true;
330 : }
331 :
332 : void
333 0 : MediaEngineCameraVideoSource::SetName(nsString aName)
334 : {
335 0 : mDeviceName = aName;
336 0 : bool hasFacingMode = false;
337 0 : VideoFacingModeEnum facingMode = VideoFacingModeEnum::User;
338 :
339 : // Set facing mode based on device name.
340 : #if defined(ANDROID) && !defined(MOZ_WIDGET_GONK)
341 : // Names are generated. Example: "Camera 0, Facing back, Orientation 90"
342 : //
343 : // See media/webrtc/trunk/webrtc/modules/video_capture/android/java/src/org/
344 : // webrtc/videoengine/VideoCaptureDeviceInfoAndroid.java
345 :
346 : if (aName.Find(NS_LITERAL_STRING("Facing back")) != kNotFound) {
347 : hasFacingMode = true;
348 : facingMode = VideoFacingModeEnum::Environment;
349 : } else if (aName.Find(NS_LITERAL_STRING("Facing front")) != kNotFound) {
350 : hasFacingMode = true;
351 : facingMode = VideoFacingModeEnum::User;
352 : }
353 : #endif // ANDROID
354 : #ifdef XP_MACOSX
355 : // Kludge to test user-facing cameras on OSX.
356 : if (aName.Find(NS_LITERAL_STRING("Face")) != -1) {
357 : hasFacingMode = true;
358 : facingMode = VideoFacingModeEnum::User;
359 : }
360 : #endif
361 : #ifdef XP_WIN
362 : // The cameras' name of Surface book are "Microsoft Camera Front" and
363 : // "Microsoft Camera Rear" respectively.
364 :
365 : if (aName.Find(NS_LITERAL_STRING("Front")) != kNotFound) {
366 : hasFacingMode = true;
367 : facingMode = VideoFacingModeEnum::User;
368 : } else if (aName.Find(NS_LITERAL_STRING("Rear")) != kNotFound) {
369 : hasFacingMode = true;
370 : facingMode = VideoFacingModeEnum::Environment;
371 : }
372 : #endif // WINDOWS
373 0 : if (hasFacingMode) {
374 0 : mFacingMode.Assign(NS_ConvertUTF8toUTF16(
375 0 : VideoFacingModeEnumValues::strings[uint32_t(facingMode)].value));
376 : } else {
377 0 : mFacingMode.Truncate();
378 : }
379 0 : }
380 :
381 : void
382 0 : MediaEngineCameraVideoSource::GetName(nsAString& aName) const
383 : {
384 0 : aName = mDeviceName;
385 0 : }
386 :
387 : void
388 0 : MediaEngineCameraVideoSource::SetUUID(const char* aUUID)
389 : {
390 0 : mUniqueId.Assign(aUUID);
391 0 : }
392 :
393 : void
394 0 : MediaEngineCameraVideoSource::GetUUID(nsACString& aUUID) const
395 : {
396 0 : aUUID = mUniqueId;
397 0 : }
398 :
399 : const nsCString&
400 0 : MediaEngineCameraVideoSource::GetUUID() const
401 : {
402 0 : return mUniqueId;
403 : }
404 :
405 : void
406 0 : MediaEngineCameraVideoSource::SetDirectListeners(bool aHasDirectListeners)
407 : {
408 0 : LOG((__FUNCTION__));
409 0 : mHasDirectListeners = aHasDirectListeners;
410 0 : }
411 :
412 : } // namespace mozilla
|
