1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/media_stream_video_source.h"
11 #include "base/debug/trace_event.h"
12 #include "base/logging.h"
13 #include "base/strings/string_number_conversions.h"
14 #include "content/child/child_process.h"
15 #include "content/renderer/media/media_stream_constraints_util.h"
16 #include "content/renderer/media/media_stream_video_track.h"
17 #include "content/renderer/media/video_track_adapter.h"
18 #include "media/base/bind_to_current_loop.h"
22 // Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b
23 const char MediaStreamVideoSource::kMinAspectRatio
[] = "minAspectRatio";
24 const char MediaStreamVideoSource::kMaxAspectRatio
[] = "maxAspectRatio";
25 const char MediaStreamVideoSource::kMaxWidth
[] = "maxWidth";
26 const char MediaStreamVideoSource::kMinWidth
[] = "minWidth";
27 const char MediaStreamVideoSource::kMaxHeight
[] = "maxHeight";
28 const char MediaStreamVideoSource::kMinHeight
[] = "minHeight";
29 const char MediaStreamVideoSource::kMaxFrameRate
[] = "maxFrameRate";
30 const char MediaStreamVideoSource::kMinFrameRate
[] = "minFrameRate";
32 const char* kSupportedConstraints
[] = {
33 MediaStreamVideoSource::kMaxAspectRatio
,
34 MediaStreamVideoSource::kMinAspectRatio
,
35 MediaStreamVideoSource::kMaxWidth
,
36 MediaStreamVideoSource::kMinWidth
,
37 MediaStreamVideoSource::kMaxHeight
,
38 MediaStreamVideoSource::kMinHeight
,
39 MediaStreamVideoSource::kMaxFrameRate
,
40 MediaStreamVideoSource::kMinFrameRate
,
43 const int MediaStreamVideoSource::kDefaultWidth
= 640;
44 const int MediaStreamVideoSource::kDefaultHeight
= 480;
45 const int MediaStreamVideoSource::kDefaultFrameRate
= 30;
49 // Google-specific key prefix. Constraints with this prefix are ignored if they
51 const char kGooglePrefix
[] = "goog";
53 // Returns true if |constraint| has mandatory constraints.
54 bool HasMandatoryConstraints(const blink::WebMediaConstraints
& constraints
) {
55 blink::WebVector
<blink::WebMediaConstraint
> mandatory_constraints
;
56 constraints
.getMandatoryConstraints(mandatory_constraints
);
57 return !mandatory_constraints
.isEmpty();
60 // Retrieve the desired max width and height from |constraints|. If not set,
61 // the |desired_width| and |desired_height| are set to
62 // std::numeric_limits<int>::max();
63 // If either max width or height is set as a mandatory constraint, the optional
64 // constraints are not checked.
65 void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints
& constraints
,
66 int* desired_width
, int* desired_height
) {
67 *desired_width
= std::numeric_limits
<int>::max();
68 *desired_height
= std::numeric_limits
<int>::max();
70 bool mandatory
= GetMandatoryConstraintValueAsInteger(
72 MediaStreamVideoSource::kMaxWidth
,
74 mandatory
|= GetMandatoryConstraintValueAsInteger(
76 MediaStreamVideoSource::kMaxHeight
,
81 GetOptionalConstraintValueAsInteger(constraints
,
82 MediaStreamVideoSource::kMaxWidth
,
84 GetOptionalConstraintValueAsInteger(constraints
,
85 MediaStreamVideoSource::kMaxHeight
,
89 // Retrieve the desired max and min aspect ratio from |constraints|. If not set,
90 // the |min_aspect_ratio| is set to 0 and |max_aspect_ratio| is set to
91 // std::numeric_limits<double>::max();
92 // If either min or max aspect ratio is set as a mandatory constraint, the
93 // optional constraints are not checked.
94 void GetDesiredMinAndMaxAspectRatio(
95 const blink::WebMediaConstraints
& constraints
,
96 double* min_aspect_ratio
,
97 double* max_aspect_ratio
) {
98 *min_aspect_ratio
= 0;
99 *max_aspect_ratio
= std::numeric_limits
<double>::max();
101 bool mandatory
= GetMandatoryConstraintValueAsDouble(
103 MediaStreamVideoSource::kMinAspectRatio
,
105 mandatory
|= GetMandatoryConstraintValueAsDouble(
107 MediaStreamVideoSource::kMaxAspectRatio
,
112 GetOptionalConstraintValueAsDouble(
114 MediaStreamVideoSource::kMinAspectRatio
,
116 GetOptionalConstraintValueAsDouble(
118 MediaStreamVideoSource::kMaxAspectRatio
,
122 // Returns true if |constraint| is fulfilled. |format| can be changed by a
123 // constraint, e.g. the frame rate can be changed by setting maxFrameRate.
124 bool UpdateFormatForConstraint(
125 const blink::WebMediaConstraint
& constraint
,
127 media::VideoCaptureFormat
* format
) {
128 DCHECK(format
!= NULL
);
130 if (!format
->IsValid())
133 std::string constraint_name
= constraint
.m_name
.utf8();
134 std::string constraint_value
= constraint
.m_value
.utf8();
136 if (constraint_name
.find(kGooglePrefix
) == 0) {
137 // These are actually options, not constraints, so they can be satisfied
138 // regardless of the format.
142 if (constraint_name
== MediaStreamSource::kSourceId
) {
143 // This is a constraint that doesn't affect the format.
147 // Ignore Chrome specific Tab capture constraints.
148 if (constraint_name
== kMediaStreamSource
||
149 constraint_name
== kMediaStreamSourceId
)
152 if (constraint_name
== MediaStreamVideoSource::kMinAspectRatio
||
153 constraint_name
== MediaStreamVideoSource::kMaxAspectRatio
) {
154 // These constraints are handled by cropping if the camera outputs the wrong
157 return base::StringToDouble(constraint_value
, &value
);
161 if (!base::StringToDouble(constraint_value
, &value
)) {
162 DLOG(WARNING
) << "Can't parse MediaStream constraint. Name:"
163 << constraint_name
<< " Value:" << constraint_value
;
167 if (constraint_name
== MediaStreamVideoSource::kMinWidth
) {
168 return (value
<= format
->frame_size
.width());
169 } else if (constraint_name
== MediaStreamVideoSource::kMaxWidth
) {
171 } else if (constraint_name
== MediaStreamVideoSource::kMinHeight
) {
172 return (value
<= format
->frame_size
.height());
173 } else if (constraint_name
== MediaStreamVideoSource::kMaxHeight
) {
175 } else if (constraint_name
== MediaStreamVideoSource::kMinFrameRate
) {
176 return (value
> 0.0) && (value
<= format
->frame_rate
);
177 } else if (constraint_name
== MediaStreamVideoSource::kMaxFrameRate
) {
179 // The frame rate is set by constraint.
180 // Don't allow 0 as frame rate if it is a mandatory constraint.
181 // Set the frame rate to 1 if it is not mandatory.
189 (format
->frame_rate
> value
) ? value
: format
->frame_rate
;
192 LOG(WARNING
) << "Found unknown MediaStream constraint. Name:"
193 << constraint_name
<< " Value:" << constraint_value
;
198 // Removes media::VideoCaptureFormats from |formats| that don't meet
200 void FilterFormatsByConstraint(
201 const blink::WebMediaConstraint
& constraint
,
203 media::VideoCaptureFormats
* formats
) {
204 DVLOG(3) << "FilterFormatsByConstraint("
205 << "{ constraint.m_name = " << constraint
.m_name
.utf8()
206 << " constraint.m_value = " << constraint
.m_value
.utf8()
207 << " mandatory = " << mandatory
<< "})";
208 media::VideoCaptureFormats::iterator format_it
= formats
->begin();
209 while (format_it
!= formats
->end()) {
210 // Modify the format_it to fulfill the constraint if possible.
211 // Delete it otherwise.
212 if (!UpdateFormatForConstraint(constraint
, mandatory
, &(*format_it
))) {
213 format_it
= formats
->erase(format_it
);
220 // Returns the media::VideoCaptureFormats that matches |constraints|.
221 media::VideoCaptureFormats
FilterFormats(
222 const blink::WebMediaConstraints
& constraints
,
223 const media::VideoCaptureFormats
& supported_formats
,
224 blink::WebString
* unsatisfied_constraint
) {
225 if (constraints
.isNull()) {
226 return supported_formats
;
229 double max_aspect_ratio
;
230 double min_aspect_ratio
;
231 GetDesiredMinAndMaxAspectRatio(constraints
,
235 if (min_aspect_ratio
> max_aspect_ratio
|| max_aspect_ratio
< 0.05f
) {
236 DLOG(WARNING
) << "Wrong requested aspect ratio.";
237 return media::VideoCaptureFormats();
241 GetMandatoryConstraintValueAsInteger(constraints
,
242 MediaStreamVideoSource::kMinWidth
,
245 GetMandatoryConstraintValueAsInteger(constraints
,
246 MediaStreamVideoSource::kMinHeight
,
250 GetDesiredMaxWidthAndHeight(constraints
, &max_width
, &max_height
);
252 if (min_width
> max_width
|| min_height
> max_height
)
253 return media::VideoCaptureFormats();
255 double min_frame_rate
= 0.0f
;
256 double max_frame_rate
= 0.0f
;
257 if (GetConstraintValueAsDouble(constraints
,
258 MediaStreamVideoSource::kMaxFrameRate
,
260 GetConstraintValueAsDouble(constraints
,
261 MediaStreamVideoSource::kMinFrameRate
,
263 if (min_frame_rate
> max_frame_rate
) {
264 DLOG(WARNING
) << "Wrong requested frame rate.";
265 return media::VideoCaptureFormats();
269 blink::WebVector
<blink::WebMediaConstraint
> mandatory
;
270 blink::WebVector
<blink::WebMediaConstraint
> optional
;
271 constraints
.getMandatoryConstraints(mandatory
);
272 constraints
.getOptionalConstraints(optional
);
273 media::VideoCaptureFormats candidates
= supported_formats
;
274 for (size_t i
= 0; i
< mandatory
.size(); ++i
) {
275 FilterFormatsByConstraint(mandatory
[i
], true, &candidates
);
276 if (candidates
.empty()) {
277 *unsatisfied_constraint
= mandatory
[i
].m_name
;
282 if (candidates
.empty())
285 // Ok - all mandatory checked and we still have candidates.
286 // Let's try filtering using the optional constraints. The optional
287 // constraints must be filtered in the order they occur in |optional|.
288 // But if a constraint produce zero candidates, the constraint is ignored and
289 // the next constraint is tested.
290 // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#idl-def-Constraints
291 for (size_t i
= 0; i
< optional
.size(); ++i
) {
292 media::VideoCaptureFormats current_candidates
= candidates
;
293 FilterFormatsByConstraint(optional
[i
], false, ¤t_candidates
);
294 if (!current_candidates
.empty()) {
295 candidates
= current_candidates
;
299 // We have done as good as we can to filter the supported resolutions.
303 const media::VideoCaptureFormat
& GetBestFormatBasedOnArea(
304 const media::VideoCaptureFormats
& formats
,
306 media::VideoCaptureFormats::const_iterator it
= formats
.begin();
307 media::VideoCaptureFormats::const_iterator best_it
= formats
.begin();
308 int best_diff
= std::numeric_limits
<int>::max();
309 for (; it
!= formats
.end(); ++it
) {
310 int diff
= abs(area
- it
->frame_size
.width() * it
->frame_size
.height());
311 if (diff
< best_diff
) {
319 // Find the format that best matches the default video size.
320 // This algorithm is chosen since a resolution must be picked even if no
321 // constraints are provided. We don't just select the maximum supported
322 // resolution since higher resolutions cost more in terms of complexity and
323 // many cameras have lower frame rate and have more noise in the image at
324 // their maximum supported resolution.
325 void GetBestCaptureFormat(
326 const media::VideoCaptureFormats
& formats
,
327 const blink::WebMediaConstraints
& constraints
,
328 media::VideoCaptureFormat
* capture_format
) {
329 DCHECK(!formats
.empty());
333 GetDesiredMaxWidthAndHeight(constraints
, &max_width
, &max_height
);
335 *capture_format
= GetBestFormatBasedOnArea(
337 std::min(max_width
, MediaStreamVideoSource::kDefaultWidth
) *
338 std::min(max_height
, MediaStreamVideoSource::kDefaultHeight
));
341 } // anonymous namespace
344 MediaStreamVideoSource
* MediaStreamVideoSource::GetVideoSource(
345 const blink::WebMediaStreamSource
& source
) {
346 return static_cast<MediaStreamVideoSource
*>(source
.extraData());
350 bool MediaStreamVideoSource::IsConstraintSupported(const std::string
& name
) {
351 for (size_t i
= 0; i
< arraysize(kSupportedConstraints
); ++i
) {
352 if (kSupportedConstraints
[i
] == name
)
358 MediaStreamVideoSource::MediaStreamVideoSource()
361 track_adapter_(new VideoTrackAdapter(
362 ChildProcess::current()->io_message_loop_proxy())),
363 weak_factory_(this) {
366 MediaStreamVideoSource::~MediaStreamVideoSource() {
367 DCHECK(CalledOnValidThread());
370 void MediaStreamVideoSource::AddTrack(
371 MediaStreamVideoTrack
* track
,
372 const VideoCaptureDeliverFrameCB
& frame_callback
,
373 const blink::WebMediaConstraints
& constraints
,
374 const ConstraintsCallback
& callback
) {
375 DCHECK(CalledOnValidThread());
376 DCHECK(!constraints
.isNull());
377 DCHECK(std::find(tracks_
.begin(), tracks_
.end(),
378 track
) == tracks_
.end());
379 tracks_
.push_back(track
);
381 requested_constraints_
.push_back(
382 RequestedConstraints(track
, frame_callback
, constraints
, callback
));
386 // Tab capture and Screen capture needs the maximum requested height
387 // and width to decide on the resolution.
388 int max_requested_width
= 0;
389 GetMandatoryConstraintValueAsInteger(constraints
, kMaxWidth
,
390 &max_requested_width
);
392 int max_requested_height
= 0;
393 GetMandatoryConstraintValueAsInteger(constraints
, kMaxHeight
,
394 &max_requested_height
);
396 double max_requested_frame_rate
;
397 if (!GetConstraintValueAsDouble(constraints
, kMaxFrameRate
,
398 &max_requested_frame_rate
)) {
399 max_requested_frame_rate
= kDefaultFrameRate
;
402 state_
= RETRIEVING_CAPABILITIES
;
403 GetCurrentSupportedFormats(
405 max_requested_height
,
406 max_requested_frame_rate
,
407 base::Bind(&MediaStreamVideoSource::OnSupportedFormats
,
408 weak_factory_
.GetWeakPtr()));
413 case RETRIEVING_CAPABILITIES
: {
414 // The |callback| will be triggered once the source has started or
415 // the capabilities have been retrieved.
420 // Currently, reconfiguring the source is not supported.
426 void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack
* video_track
) {
427 DCHECK(CalledOnValidThread());
428 std::vector
<MediaStreamVideoTrack
*>::iterator it
=
429 std::find(tracks_
.begin(), tracks_
.end(), video_track
);
430 DCHECK(it
!= tracks_
.end());
433 // Check if |video_track| is waiting for applying new constraints and remove
434 // the request in that case.
435 for (std::vector
<RequestedConstraints
>::iterator it
=
436 requested_constraints_
.begin();
437 it
!= requested_constraints_
.end(); ++it
) {
438 if (it
->track
== video_track
) {
439 requested_constraints_
.erase(it
);
443 // Call |frame_adapter_->RemoveTrack| here even if adding the track has
444 // failed and |frame_adapter_->AddCallback| has not been called.
445 track_adapter_
->RemoveTrack(video_track
);
451 const scoped_refptr
<base::MessageLoopProxy
>&
452 MediaStreamVideoSource::io_message_loop() const {
453 DCHECK(CalledOnValidThread());
454 return track_adapter_
->io_message_loop();
457 void MediaStreamVideoSource::DoStopSource() {
458 DCHECK(CalledOnValidThread());
459 DVLOG(3) << "DoStopSource()";
464 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded
);
467 void MediaStreamVideoSource::OnSupportedFormats(
468 const media::VideoCaptureFormats
& formats
) {
469 DCHECK(CalledOnValidThread());
470 DCHECK_EQ(RETRIEVING_CAPABILITIES
, state_
);
472 supported_formats_
= formats
;
473 if (!FindBestFormatWithConstraints(supported_formats_
,
475 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded
);
476 // This object can be deleted after calling FinalizeAddTrack. See comment
477 // in the header file.
483 DVLOG(3) << "Starting the capturer with"
484 << " width = " << current_format_
.frame_size
.width()
485 << " height = " << current_format_
.frame_size
.height()
486 << " frame rate = " << current_format_
.frame_rate
487 << " pixel format = "
488 << media::VideoCaptureFormat::PixelFormatToString(
489 current_format_
.pixel_format
);
491 media::VideoCaptureParams params
;
492 params
.requested_format
= current_format_
;
495 base::Bind(&VideoTrackAdapter::DeliverFrameOnIO
, track_adapter_
));
498 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
499 const media::VideoCaptureFormats
& formats
,
500 media::VideoCaptureFormat
* best_format
) {
501 DCHECK(CalledOnValidThread());
502 // Find the first constraints that we can fulfill.
503 for (std::vector
<RequestedConstraints
>::iterator request_it
=
504 requested_constraints_
.begin();
505 request_it
!= requested_constraints_
.end(); ++request_it
) {
506 const blink::WebMediaConstraints
& requested_constraints
=
507 request_it
->constraints
;
509 // If the source doesn't support capability enumeration it is still ok if
510 // no mandatory constraints have been specified. That just means that
511 // we will start with whatever format is native to the source.
512 if (formats
.empty() && !HasMandatoryConstraints(requested_constraints
)) {
513 *best_format
= media::VideoCaptureFormat();
516 blink::WebString unsatisfied_constraint
;
517 media::VideoCaptureFormats filtered_formats
=
518 FilterFormats(requested_constraints
, formats
, &unsatisfied_constraint
);
519 if (filtered_formats
.size() > 0) {
520 // A request with constraints that can be fulfilled.
521 GetBestCaptureFormat(filtered_formats
,
522 requested_constraints
,
530 void MediaStreamVideoSource::OnStartDone(MediaStreamRequestResult result
) {
531 DCHECK(CalledOnValidThread());
532 DVLOG(3) << "OnStartDone({result =" << result
<< "})";
533 if (result
== MEDIA_DEVICE_OK
) {
534 DCHECK_EQ(STARTING
, state_
);
536 SetReadyState(blink::WebMediaStreamSource::ReadyStateLive
);
541 // This object can be deleted after calling FinalizeAddTrack. See comment in
546 void MediaStreamVideoSource::FinalizeAddTrack() {
547 DCHECK(CalledOnValidThread());
548 media::VideoCaptureFormats formats
;
549 formats
.push_back(current_format_
);
551 std::vector
<RequestedConstraints
> callbacks
;
552 callbacks
.swap(requested_constraints_
);
553 for (std::vector
<RequestedConstraints
>::iterator it
= callbacks
.begin();
554 it
!= callbacks
.end(); ++it
) {
555 MediaStreamRequestResult result
= MEDIA_DEVICE_OK
;
556 blink::WebString unsatisfied_constraint
;
558 if (HasMandatoryConstraints(it
->constraints
) &&
559 FilterFormats(it
->constraints
, formats
,
560 &unsatisfied_constraint
).empty())
561 result
= MEDIA_DEVICE_CONSTRAINT_NOT_SATISFIED
;
563 if (state_
!= STARTED
&& result
== MEDIA_DEVICE_OK
)
564 result
= MEDIA_DEVICE_TRACK_START_FAILURE
;
566 if (result
== MEDIA_DEVICE_OK
) {
569 GetDesiredMaxWidthAndHeight(it
->constraints
, &max_width
, &max_height
);
570 double max_aspect_ratio
;
571 double min_aspect_ratio
;
572 GetDesiredMinAndMaxAspectRatio(it
->constraints
,
575 double max_frame_rate
= 0.0f
;
576 GetConstraintValueAsDouble(it
->constraints
,
577 kMaxFrameRate
, &max_frame_rate
);
579 VideoTrackAdapter::OnMutedCallback on_mute_callback
=
580 media::BindToCurrentLoop(base::Bind(
581 &MediaStreamVideoSource::SetMutedState
,
582 weak_factory_
.GetWeakPtr()));
583 track_adapter_
->AddTrack(it
->track
, it
->frame_callback
,
584 max_width
, max_height
,
585 min_aspect_ratio
, max_aspect_ratio
,
586 max_frame_rate
, current_format_
.frame_rate
,
590 DVLOG(3) << "FinalizeAddTrack() result " << result
;
592 if (!it
->callback
.is_null()) {
593 it
->callback
.Run(this, result
, unsatisfied_constraint
);
598 void MediaStreamVideoSource::SetReadyState(
599 blink::WebMediaStreamSource::ReadyState state
) {
600 DVLOG(3) << "MediaStreamVideoSource::SetReadyState state " << state
;
601 DCHECK(CalledOnValidThread());
602 if (!owner().isNull())
603 owner().setReadyState(state
);
604 for (std::vector
<MediaStreamVideoTrack
*>::iterator it
= tracks_
.begin();
605 it
!= tracks_
.end(); ++it
) {
606 (*it
)->OnReadyStateChanged(state
);
610 void MediaStreamVideoSource::SetMutedState(bool muted_state
) {
611 DVLOG(3) << "MediaStreamVideoSource::SetMutedState state=" << muted_state
;
612 DCHECK(CalledOnValidThread());
613 if (muted_state
!= muted_state_
) {
614 muted_state_
= muted_state
;
615 if (!owner().isNull()) {
616 owner().setReadyState(muted_state_
617 ? blink::WebMediaStreamSource::ReadyStateMuted
618 : blink::WebMediaStreamSource::ReadyStateLive
);
621 // WebMediaStreamSource doesn't have a muted state, the tracks do.
622 for (std::vector
<MediaStreamVideoTrack
*>::iterator it
= tracks_
.begin();
623 it
!= tracks_
.end(); ++it
) {
624 (*it
)->SetMutedState(muted_state
);
628 MediaStreamVideoSource::RequestedConstraints::RequestedConstraints(
629 MediaStreamVideoTrack
* track
,
630 const VideoCaptureDeliverFrameCB
& frame_callback
,
631 const blink::WebMediaConstraints
& constraints
,
632 const ConstraintsCallback
& callback
)
634 frame_callback(frame_callback
),
635 constraints(constraints
),
639 MediaStreamVideoSource::RequestedConstraints::~RequestedConstraints() {
642 } // namespace content