1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/media_stream_video_source.h"
11 #include "base/logging.h"
12 #include "base/strings/string_number_conversions.h"
13 #include "base/trace_event/trace_event.h"
14 #include "content/child/child_process.h"
15 #include "content/renderer/media/media_stream_constraints_util.h"
16 #include "content/renderer/media/media_stream_video_track.h"
17 #include "content/renderer/media/video_track_adapter.h"
21 // Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b
22 const char MediaStreamVideoSource::kMinAspectRatio
[] = "minAspectRatio";
23 const char MediaStreamVideoSource::kMaxAspectRatio
[] = "maxAspectRatio";
24 const char MediaStreamVideoSource::kMaxWidth
[] = "maxWidth";
25 const char MediaStreamVideoSource::kMinWidth
[] = "minWidth";
26 const char MediaStreamVideoSource::kMaxHeight
[] = "maxHeight";
27 const char MediaStreamVideoSource::kMinHeight
[] = "minHeight";
28 const char MediaStreamVideoSource::kMaxFrameRate
[] = "maxFrameRate";
29 const char MediaStreamVideoSource::kMinFrameRate
[] = "minFrameRate";
31 const char* kSupportedConstraints
[] = {
32 MediaStreamVideoSource::kMaxAspectRatio
,
33 MediaStreamVideoSource::kMinAspectRatio
,
34 MediaStreamVideoSource::kMaxWidth
,
35 MediaStreamVideoSource::kMinWidth
,
36 MediaStreamVideoSource::kMaxHeight
,
37 MediaStreamVideoSource::kMinHeight
,
38 MediaStreamVideoSource::kMaxFrameRate
,
39 MediaStreamVideoSource::kMinFrameRate
,
44 // Google-specific key prefix. Constraints with this prefix are ignored if they
46 const char kGooglePrefix
[] = "goog";
48 // Returns true if |constraint| has mandatory constraints.
49 bool HasMandatoryConstraints(const blink::WebMediaConstraints
& constraints
) {
50 blink::WebVector
<blink::WebMediaConstraint
> mandatory_constraints
;
51 constraints
.getMandatoryConstraints(mandatory_constraints
);
52 return !mandatory_constraints
.isEmpty();
55 // Retrieve the desired max width and height from |constraints|. If not set,
56 // the |desired_width| and |desired_height| are set to
57 // std::numeric_limits<int>::max();
58 // If either max width or height is set as a mandatory constraint, the optional
59 // constraints are not checked.
60 void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints
& constraints
,
61 int* desired_width
, int* desired_height
) {
62 *desired_width
= std::numeric_limits
<int>::max();
63 *desired_height
= std::numeric_limits
<int>::max();
65 bool mandatory
= GetMandatoryConstraintValueAsInteger(
67 MediaStreamVideoSource::kMaxWidth
,
69 mandatory
|= GetMandatoryConstraintValueAsInteger(
71 MediaStreamVideoSource::kMaxHeight
,
76 GetOptionalConstraintValueAsInteger(constraints
,
77 MediaStreamVideoSource::kMaxWidth
,
79 GetOptionalConstraintValueAsInteger(constraints
,
80 MediaStreamVideoSource::kMaxHeight
,
84 // Retrieve the desired max and min aspect ratio from |constraints|. If not set,
85 // the |min_aspect_ratio| is set to 0 and |max_aspect_ratio| is set to
86 // std::numeric_limits<double>::max();
87 // If either min or max aspect ratio is set as a mandatory constraint, the
88 // optional constraints are not checked.
89 void GetDesiredMinAndMaxAspectRatio(
90 const blink::WebMediaConstraints
& constraints
,
91 double* min_aspect_ratio
,
92 double* max_aspect_ratio
) {
93 *min_aspect_ratio
= 0;
94 *max_aspect_ratio
= std::numeric_limits
<double>::max();
96 bool mandatory
= GetMandatoryConstraintValueAsDouble(
98 MediaStreamVideoSource::kMinAspectRatio
,
100 mandatory
|= GetMandatoryConstraintValueAsDouble(
102 MediaStreamVideoSource::kMaxAspectRatio
,
107 GetOptionalConstraintValueAsDouble(
109 MediaStreamVideoSource::kMinAspectRatio
,
111 GetOptionalConstraintValueAsDouble(
113 MediaStreamVideoSource::kMaxAspectRatio
,
117 // Returns true if |constraint| is fulfilled. |format| can be changed by a
118 // constraint, e.g. the frame rate can be changed by setting maxFrameRate.
119 bool UpdateFormatForConstraint(
120 const blink::WebMediaConstraint
& constraint
,
122 media::VideoCaptureFormat
* format
) {
123 DCHECK(format
!= NULL
);
125 if (!format
->IsValid())
128 std::string constraint_name
= constraint
.m_name
.utf8();
129 std::string constraint_value
= constraint
.m_value
.utf8();
131 if (constraint_name
.find(kGooglePrefix
) == 0) {
132 // These are actually options, not constraints, so they can be satisfied
133 // regardless of the format.
137 if (constraint_name
== MediaStreamSource::kSourceId
) {
138 // This is a constraint that doesn't affect the format.
142 // Ignore Chrome specific Tab capture constraints.
143 if (constraint_name
== kMediaStreamSource
||
144 constraint_name
== kMediaStreamSourceId
)
147 if (constraint_name
== MediaStreamVideoSource::kMinAspectRatio
||
148 constraint_name
== MediaStreamVideoSource::kMaxAspectRatio
) {
149 // These constraints are handled by cropping if the camera outputs the wrong
152 return base::StringToDouble(constraint_value
, &value
);
156 if (!base::StringToDouble(constraint_value
, &value
)) {
157 DLOG(WARNING
) << "Can't parse MediaStream constraint. Name:"
158 << constraint_name
<< " Value:" << constraint_value
;
162 if (constraint_name
== MediaStreamVideoSource::kMinWidth
) {
163 return (value
<= format
->frame_size
.width());
164 } else if (constraint_name
== MediaStreamVideoSource::kMaxWidth
) {
166 } else if (constraint_name
== MediaStreamVideoSource::kMinHeight
) {
167 return (value
<= format
->frame_size
.height());
168 } else if (constraint_name
== MediaStreamVideoSource::kMaxHeight
) {
170 } else if (constraint_name
== MediaStreamVideoSource::kMinFrameRate
) {
171 return (value
> 0.0) && (value
<= format
->frame_rate
);
172 } else if (constraint_name
== MediaStreamVideoSource::kMaxFrameRate
) {
174 // The frame rate is set by constraint.
175 // Don't allow 0 as frame rate if it is a mandatory constraint.
176 // Set the frame rate to 1 if it is not mandatory.
184 (format
->frame_rate
> value
) ? value
: format
->frame_rate
;
187 LOG(WARNING
) << "Found unknown MediaStream constraint. Name:"
188 << constraint_name
<< " Value:" << constraint_value
;
193 // Removes media::VideoCaptureFormats from |formats| that don't meet
195 void FilterFormatsByConstraint(
196 const blink::WebMediaConstraint
& constraint
,
198 media::VideoCaptureFormats
* formats
) {
199 DVLOG(3) << "FilterFormatsByConstraint("
200 << "{ constraint.m_name = " << constraint
.m_name
.utf8()
201 << " constraint.m_value = " << constraint
.m_value
.utf8()
202 << " mandatory = " << mandatory
<< "})";
203 media::VideoCaptureFormats::iterator format_it
= formats
->begin();
204 while (format_it
!= formats
->end()) {
205 // Modify the format_it to fulfill the constraint if possible.
206 // Delete it otherwise.
207 if (!UpdateFormatForConstraint(constraint
, mandatory
, &(*format_it
)))
208 format_it
= formats
->erase(format_it
);
214 // Returns the media::VideoCaptureFormats that matches |constraints|.
215 media::VideoCaptureFormats
FilterFormats(
216 const blink::WebMediaConstraints
& constraints
,
217 const media::VideoCaptureFormats
& supported_formats
,
218 blink::WebString
* unsatisfied_constraint
) {
219 if (constraints
.isNull())
220 return supported_formats
;
222 double max_aspect_ratio
;
223 double min_aspect_ratio
;
224 GetDesiredMinAndMaxAspectRatio(constraints
,
228 if (min_aspect_ratio
> max_aspect_ratio
|| max_aspect_ratio
< 0.05f
) {
229 DLOG(WARNING
) << "Wrong requested aspect ratio.";
230 return media::VideoCaptureFormats();
234 GetMandatoryConstraintValueAsInteger(constraints
,
235 MediaStreamVideoSource::kMinWidth
,
238 GetMandatoryConstraintValueAsInteger(constraints
,
239 MediaStreamVideoSource::kMinHeight
,
243 GetDesiredMaxWidthAndHeight(constraints
, &max_width
, &max_height
);
245 if (min_width
> max_width
|| min_height
> max_height
)
246 return media::VideoCaptureFormats();
248 double min_frame_rate
= 0.0f
;
249 double max_frame_rate
= 0.0f
;
250 if (GetConstraintValueAsDouble(constraints
,
251 MediaStreamVideoSource::kMaxFrameRate
,
253 GetConstraintValueAsDouble(constraints
,
254 MediaStreamVideoSource::kMinFrameRate
,
256 if (min_frame_rate
> max_frame_rate
) {
257 DLOG(WARNING
) << "Wrong requested frame rate.";
258 return media::VideoCaptureFormats();
262 blink::WebVector
<blink::WebMediaConstraint
> mandatory
;
263 blink::WebVector
<blink::WebMediaConstraint
> optional
;
264 constraints
.getMandatoryConstraints(mandatory
);
265 constraints
.getOptionalConstraints(optional
);
266 media::VideoCaptureFormats candidates
= supported_formats
;
267 for (size_t i
= 0; i
< mandatory
.size(); ++i
) {
268 FilterFormatsByConstraint(mandatory
[i
], true, &candidates
);
269 if (candidates
.empty()) {
270 *unsatisfied_constraint
= mandatory
[i
].m_name
;
275 if (candidates
.empty())
278 // Ok - all mandatory checked and we still have candidates.
279 // Let's try filtering using the optional constraints. The optional
280 // constraints must be filtered in the order they occur in |optional|.
281 // But if a constraint produce zero candidates, the constraint is ignored and
282 // the next constraint is tested.
283 // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#idl-def-Constraints
284 for (size_t i
= 0; i
< optional
.size(); ++i
) {
285 media::VideoCaptureFormats current_candidates
= candidates
;
286 FilterFormatsByConstraint(optional
[i
], false, ¤t_candidates
);
287 if (!current_candidates
.empty())
288 candidates
= current_candidates
;
291 // We have done as good as we can to filter the supported resolutions.
295 const media::VideoCaptureFormat
& GetBestFormatBasedOnArea(
296 const media::VideoCaptureFormats
& formats
,
298 media::VideoCaptureFormats::const_iterator it
= formats
.begin();
299 media::VideoCaptureFormats::const_iterator best_it
= formats
.begin();
300 int best_diff
= std::numeric_limits
<int>::max();
301 for (; it
!= formats
.end(); ++it
) {
302 const int diff
= abs(area
- it
->frame_size
.GetArea());
303 if (diff
< best_diff
) {
311 // Find the format that best matches the default video size.
312 // This algorithm is chosen since a resolution must be picked even if no
313 // constraints are provided. We don't just select the maximum supported
314 // resolution since higher resolutions cost more in terms of complexity and
315 // many cameras have lower frame rate and have more noise in the image at
316 // their maximum supported resolution.
317 void GetBestCaptureFormat(
318 const media::VideoCaptureFormats
& formats
,
319 const blink::WebMediaConstraints
& constraints
,
320 media::VideoCaptureFormat
* capture_format
) {
321 DCHECK(!formats
.empty());
325 GetDesiredMaxWidthAndHeight(constraints
, &max_width
, &max_height
);
327 *capture_format
= GetBestFormatBasedOnArea(
330 static_cast<int>(MediaStreamVideoSource::kDefaultWidth
)) *
332 static_cast<int>(MediaStreamVideoSource::kDefaultHeight
)));
335 } // anonymous namespace
338 MediaStreamVideoSource
* MediaStreamVideoSource::GetVideoSource(
339 const blink::WebMediaStreamSource
& source
) {
340 return static_cast<MediaStreamVideoSource
*>(source
.extraData());
344 bool MediaStreamVideoSource::IsConstraintSupported(const std::string
& name
) {
345 for (const char* constraint
: kSupportedConstraints
) {
346 if (constraint
== name
)
352 MediaStreamVideoSource::MediaStreamVideoSource()
354 track_adapter_(new VideoTrackAdapter(
355 ChildProcess::current()->io_message_loop_proxy())),
356 weak_factory_(this) {
359 MediaStreamVideoSource::~MediaStreamVideoSource() {
360 DCHECK(CalledOnValidThread());
363 void MediaStreamVideoSource::AddTrack(
364 MediaStreamVideoTrack
* track
,
365 const VideoCaptureDeliverFrameCB
& frame_callback
,
366 const blink::WebMediaConstraints
& constraints
,
367 const ConstraintsCallback
& callback
) {
368 DCHECK(CalledOnValidThread());
369 DCHECK(!constraints
.isNull());
370 DCHECK(std::find(tracks_
.begin(), tracks_
.end(),
371 track
) == tracks_
.end());
372 tracks_
.push_back(track
);
374 requested_constraints_
.push_back(
375 RequestedConstraints(track
, frame_callback
, constraints
, callback
));
379 // Tab capture and Screen capture needs the maximum requested height
380 // and width to decide on the resolution.
381 int max_requested_width
= 0;
382 GetMandatoryConstraintValueAsInteger(constraints
, kMaxWidth
,
383 &max_requested_width
);
385 int max_requested_height
= 0;
386 GetMandatoryConstraintValueAsInteger(constraints
, kMaxHeight
,
387 &max_requested_height
);
389 double max_requested_frame_rate
;
390 if (!GetConstraintValueAsDouble(constraints
, kMaxFrameRate
,
391 &max_requested_frame_rate
)) {
392 max_requested_frame_rate
= kDefaultFrameRate
;
395 state_
= RETRIEVING_CAPABILITIES
;
396 GetCurrentSupportedFormats(
398 max_requested_height
,
399 max_requested_frame_rate
,
400 base::Bind(&MediaStreamVideoSource::OnSupportedFormats
,
401 weak_factory_
.GetWeakPtr()));
406 case RETRIEVING_CAPABILITIES
: {
407 // The |callback| will be triggered once the source has started or
408 // the capabilities have been retrieved.
413 // Currently, reconfiguring the source is not supported.
419 void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack
* video_track
) {
420 DCHECK(CalledOnValidThread());
421 std::vector
<MediaStreamVideoTrack
*>::iterator it
=
422 std::find(tracks_
.begin(), tracks_
.end(), video_track
);
423 DCHECK(it
!= tracks_
.end());
426 // Check if |video_track| is waiting for applying new constraints and remove
427 // the request in that case.
428 for (std::vector
<RequestedConstraints
>::iterator it
=
429 requested_constraints_
.begin();
430 it
!= requested_constraints_
.end(); ++it
) {
431 if (it
->track
== video_track
) {
432 requested_constraints_
.erase(it
);
436 // Call |frame_adapter_->RemoveTrack| here even if adding the track has
437 // failed and |frame_adapter_->AddCallback| has not been called.
438 track_adapter_
->RemoveTrack(video_track
);
444 const scoped_refptr
<base::MessageLoopProxy
>&
445 MediaStreamVideoSource::io_message_loop() const {
446 DCHECK(CalledOnValidThread());
447 return track_adapter_
->io_message_loop();
450 void MediaStreamVideoSource::DoStopSource() {
451 DCHECK(CalledOnValidThread());
452 DVLOG(3) << "DoStopSource()";
455 track_adapter_
->StopFrameMonitoring();
458 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded
);
461 void MediaStreamVideoSource::OnSupportedFormats(
462 const media::VideoCaptureFormats
& formats
) {
463 DCHECK(CalledOnValidThread());
464 DCHECK_EQ(RETRIEVING_CAPABILITIES
, state_
);
466 supported_formats_
= formats
;
467 if (!FindBestFormatWithConstraints(supported_formats_
,
469 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded
);
470 // This object can be deleted after calling FinalizeAddTrack. See comment
471 // in the header file.
477 DVLOG(3) << "Starting the capturer with " << current_format_
.ToString();
481 base::Bind(&VideoTrackAdapter::DeliverFrameOnIO
, track_adapter_
));
484 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
485 const media::VideoCaptureFormats
& formats
,
486 media::VideoCaptureFormat
* best_format
) {
487 DCHECK(CalledOnValidThread());
488 // Find the first constraints that we can fulfill.
489 for (const auto& request
: requested_constraints_
) {
490 const blink::WebMediaConstraints
& requested_constraints
=
493 // If the source doesn't support capability enumeration it is still ok if
494 // no mandatory constraints have been specified. That just means that
495 // we will start with whatever format is native to the source.
496 if (formats
.empty() && !HasMandatoryConstraints(requested_constraints
)) {
497 *best_format
= media::VideoCaptureFormat();
500 blink::WebString unsatisfied_constraint
;
501 media::VideoCaptureFormats filtered_formats
=
502 FilterFormats(requested_constraints
, formats
, &unsatisfied_constraint
);
503 if (filtered_formats
.size() > 0) {
504 // A request with constraints that can be fulfilled.
505 GetBestCaptureFormat(filtered_formats
,
506 requested_constraints
,
514 void MediaStreamVideoSource::OnStartDone(MediaStreamRequestResult result
) {
515 DCHECK(CalledOnValidThread());
516 DVLOG(3) << "OnStartDone({result =" << result
<< "})";
517 if (result
== MEDIA_DEVICE_OK
) {
518 DCHECK_EQ(STARTING
, state_
);
520 SetReadyState(blink::WebMediaStreamSource::ReadyStateLive
);
522 track_adapter_
->StartFrameMonitoring(
523 current_format_
.frame_rate
,
524 base::Bind(&MediaStreamVideoSource::SetMutedState
,
525 weak_factory_
.GetWeakPtr()));
531 // This object can be deleted after calling FinalizeAddTrack. See comment in
536 void MediaStreamVideoSource::FinalizeAddTrack() {
537 DCHECK(CalledOnValidThread());
538 media::VideoCaptureFormats formats
;
539 formats
.push_back(current_format_
);
541 std::vector
<RequestedConstraints
> callbacks
;
542 callbacks
.swap(requested_constraints_
);
543 for (const auto& request
: callbacks
) {
544 MediaStreamRequestResult result
= MEDIA_DEVICE_OK
;
545 blink::WebString unsatisfied_constraint
;
547 if (HasMandatoryConstraints(request
.constraints
) &&
548 FilterFormats(request
.constraints
, formats
,
549 &unsatisfied_constraint
).empty()) {
550 result
= MEDIA_DEVICE_CONSTRAINT_NOT_SATISFIED
;
553 if (state_
!= STARTED
&& result
== MEDIA_DEVICE_OK
)
554 result
= MEDIA_DEVICE_TRACK_START_FAILURE
;
556 if (result
== MEDIA_DEVICE_OK
) {
559 GetDesiredMaxWidthAndHeight(request
.constraints
, &max_width
, &max_height
);
560 double max_aspect_ratio
;
561 double min_aspect_ratio
;
562 GetDesiredMinAndMaxAspectRatio(request
.constraints
,
565 double max_frame_rate
= 0.0f
;
566 GetConstraintValueAsDouble(request
.constraints
,
567 kMaxFrameRate
, &max_frame_rate
);
569 track_adapter_
->AddTrack(request
.track
, request
.frame_callback
,
570 max_width
, max_height
,
571 min_aspect_ratio
, max_aspect_ratio
,
575 DVLOG(3) << "FinalizeAddTrack() result " << result
;
577 if (!request
.callback
.is_null()) {
578 request
.callback
.Run(this, result
, unsatisfied_constraint
);
583 void MediaStreamVideoSource::SetReadyState(
584 blink::WebMediaStreamSource::ReadyState state
) {
585 DVLOG(3) << "MediaStreamVideoSource::SetReadyState state " << state
;
586 DCHECK(CalledOnValidThread());
587 if (!owner().isNull())
588 owner().setReadyState(state
);
589 for (const auto& track
: tracks_
)
590 track
->OnReadyStateChanged(state
);
593 void MediaStreamVideoSource::SetMutedState(bool muted_state
) {
594 DVLOG(3) << "MediaStreamVideoSource::SetMutedState state=" << muted_state
;
595 DCHECK(CalledOnValidThread());
596 if (!owner().isNull()) {
597 owner().setReadyState(muted_state
598 ? blink::WebMediaStreamSource::ReadyStateMuted
599 : blink::WebMediaStreamSource::ReadyStateLive
);
603 MediaStreamVideoSource::RequestedConstraints::RequestedConstraints(
604 MediaStreamVideoTrack
* track
,
605 const VideoCaptureDeliverFrameCB
& frame_callback
,
606 const blink::WebMediaConstraints
& constraints
,
607 const ConstraintsCallback
& callback
)
609 frame_callback(frame_callback
),
610 constraints(constraints
),
614 MediaStreamVideoSource::RequestedConstraints::~RequestedConstraints() {
617 } // namespace content