1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/media_stream_video_source.h"
11 #include "base/logging.h"
12 #include "base/strings/string_number_conversions.h"
13 #include "base/trace_event/trace_event.h"
14 #include "content/child/child_process.h"
15 #include "content/renderer/media/media_stream_constraints_util.h"
16 #include "content/renderer/media/media_stream_video_track.h"
17 #include "content/renderer/media/video_track_adapter.h"
21 // Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b
22 const char MediaStreamVideoSource::kMinAspectRatio
[] = "minAspectRatio";
23 const char MediaStreamVideoSource::kMaxAspectRatio
[] = "maxAspectRatio";
24 const char MediaStreamVideoSource::kMaxWidth
[] = "maxWidth";
25 const char MediaStreamVideoSource::kMinWidth
[] = "minWidth";
26 const char MediaStreamVideoSource::kMaxHeight
[] = "maxHeight";
27 const char MediaStreamVideoSource::kMinHeight
[] = "minHeight";
28 const char MediaStreamVideoSource::kMaxFrameRate
[] = "maxFrameRate";
29 const char MediaStreamVideoSource::kMinFrameRate
[] = "minFrameRate";
31 // TODO(mcasas): Find a way to guarantee all constraints are added to the array.
32 const char* kSupportedConstraints
[] = {
33 MediaStreamVideoSource::kMaxAspectRatio
,
34 MediaStreamVideoSource::kMinAspectRatio
,
35 MediaStreamVideoSource::kMaxWidth
,
36 MediaStreamVideoSource::kMinWidth
,
37 MediaStreamVideoSource::kMaxHeight
,
38 MediaStreamVideoSource::kMinHeight
,
39 MediaStreamVideoSource::kMaxFrameRate
,
40 MediaStreamVideoSource::kMinFrameRate
,
45 // Google-specific key prefix. Constraints with this prefix are ignored if they
47 const char kGooglePrefix
[] = "goog";
49 // Returns true if |constraint| has mandatory constraints.
50 bool HasMandatoryConstraints(const blink::WebMediaConstraints
& constraints
) {
51 blink::WebVector
<blink::WebMediaConstraint
> mandatory_constraints
;
52 constraints
.getMandatoryConstraints(mandatory_constraints
);
53 return !mandatory_constraints
.isEmpty();
56 // Retrieve the desired max width and height from |constraints|. If not set,
57 // the |desired_width| and |desired_height| are set to
58 // std::numeric_limits<int>::max();
59 // If either max width or height is set as a mandatory constraint, the optional
60 // constraints are not checked.
61 void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints
& constraints
,
62 int* desired_width
, int* desired_height
) {
63 *desired_width
= std::numeric_limits
<int>::max();
64 *desired_height
= std::numeric_limits
<int>::max();
66 bool mandatory
= GetMandatoryConstraintValueAsInteger(
68 MediaStreamVideoSource::kMaxWidth
,
70 mandatory
|= GetMandatoryConstraintValueAsInteger(
72 MediaStreamVideoSource::kMaxHeight
,
77 GetOptionalConstraintValueAsInteger(constraints
,
78 MediaStreamVideoSource::kMaxWidth
,
80 GetOptionalConstraintValueAsInteger(constraints
,
81 MediaStreamVideoSource::kMaxHeight
,
85 // Retrieve the desired max and min aspect ratio from |constraints|. If not set,
86 // the |min_aspect_ratio| is set to 0 and |max_aspect_ratio| is set to
87 // std::numeric_limits<double>::max();
88 // If either min or max aspect ratio is set as a mandatory constraint, the
89 // optional constraints are not checked.
90 void GetDesiredMinAndMaxAspectRatio(
91 const blink::WebMediaConstraints
& constraints
,
92 double* min_aspect_ratio
,
93 double* max_aspect_ratio
) {
94 *min_aspect_ratio
= 0;
95 *max_aspect_ratio
= std::numeric_limits
<double>::max();
97 bool mandatory
= GetMandatoryConstraintValueAsDouble(
99 MediaStreamVideoSource::kMinAspectRatio
,
101 mandatory
|= GetMandatoryConstraintValueAsDouble(
103 MediaStreamVideoSource::kMaxAspectRatio
,
108 GetOptionalConstraintValueAsDouble(constraints
,
109 MediaStreamVideoSource::kMinAspectRatio
,
111 GetOptionalConstraintValueAsDouble(constraints
,
112 MediaStreamVideoSource::kMaxAspectRatio
,
116 // Returns true if |constraint| is fulfilled. |format| can be changed by a
117 // constraint, e.g. the frame rate can be changed by setting maxFrameRate.
118 bool UpdateFormatForConstraint(const blink::WebMediaConstraint
& constraint
,
120 media::VideoCaptureFormat
* format
) {
121 DCHECK(format
!= NULL
);
123 if (!format
->IsValid())
126 const std::string constraint_name
= constraint
.m_name
.utf8();
127 const std::string constraint_value
= constraint
.m_value
.utf8();
129 if (constraint_name
.find(kGooglePrefix
) == 0) {
130 // These are actually options, not constraints, so they can be satisfied
131 // regardless of the format.
135 if (constraint_name
== MediaStreamSource::kSourceId
) {
136 // This is a constraint that doesn't affect the format.
140 // Ignore Chrome specific Tab capture constraints.
141 if (constraint_name
== kMediaStreamSource
||
142 constraint_name
== kMediaStreamSourceId
)
145 if (constraint_name
== MediaStreamVideoSource::kMinAspectRatio
||
146 constraint_name
== MediaStreamVideoSource::kMaxAspectRatio
) {
147 // These constraints are handled by cropping if the camera outputs the wrong
150 return base::StringToDouble(constraint_value
, &value
);
154 if (!base::StringToDouble(constraint_value
, &value
)) {
155 DLOG(WARNING
) << "Can't parse MediaStream constraint. Name:"
156 << constraint_name
<< " Value:" << constraint_value
;
160 if (constraint_name
== MediaStreamVideoSource::kMinWidth
) {
161 return (value
<= format
->frame_size
.width());
162 } else if (constraint_name
== MediaStreamVideoSource::kMaxWidth
) {
164 } else if (constraint_name
== MediaStreamVideoSource::kMinHeight
) {
165 return (value
<= format
->frame_size
.height());
166 } else if (constraint_name
== MediaStreamVideoSource::kMaxHeight
) {
168 } else if (constraint_name
== MediaStreamVideoSource::kMinFrameRate
) {
169 return (value
> 0.0) && (value
<= format
->frame_rate
);
170 } else if (constraint_name
== MediaStreamVideoSource::kMaxFrameRate
) {
172 // The frame rate is set by constraint.
173 // Don't allow 0 as frame rate if it is a mandatory constraint.
174 // Set the frame rate to 1 if it is not mandatory.
182 (format
->frame_rate
> value
) ? value
: format
->frame_rate
;
185 LOG(WARNING
) << "Found unknown MediaStream constraint. Name:"
186 << constraint_name
<< " Value:" << constraint_value
;
191 // Removes media::VideoCaptureFormats from |formats| that don't meet
193 void FilterFormatsByConstraint(const blink::WebMediaConstraint
& constraint
,
195 media::VideoCaptureFormats
* formats
) {
196 DVLOG(3) << "FilterFormatsByConstraint("
197 << "{ constraint.m_name = " << constraint
.m_name
.utf8()
198 << " constraint.m_value = " << constraint
.m_value
.utf8()
199 << " mandatory = " << mandatory
<< "})";
200 media::VideoCaptureFormats::iterator format_it
= formats
->begin();
201 while (format_it
!= formats
->end()) {
202 // Modify the format_it to fulfill the constraint if possible.
203 // Delete it otherwise.
204 if (!UpdateFormatForConstraint(constraint
, mandatory
, &(*format_it
)))
205 format_it
= formats
->erase(format_it
);
211 // Returns the media::VideoCaptureFormats that matches |constraints|.
212 media::VideoCaptureFormats
FilterFormats(
213 const blink::WebMediaConstraints
& constraints
,
214 const media::VideoCaptureFormats
& supported_formats
,
215 blink::WebString
* unsatisfied_constraint
) {
216 if (constraints
.isNull())
217 return supported_formats
;
219 double max_aspect_ratio
;
220 double min_aspect_ratio
;
221 GetDesiredMinAndMaxAspectRatio(constraints
,
225 if (min_aspect_ratio
> max_aspect_ratio
|| max_aspect_ratio
< 0.05f
) {
226 DLOG(WARNING
) << "Wrong requested aspect ratio.";
227 return media::VideoCaptureFormats();
231 GetMandatoryConstraintValueAsInteger(constraints
,
232 MediaStreamVideoSource::kMinWidth
,
235 GetMandatoryConstraintValueAsInteger(constraints
,
236 MediaStreamVideoSource::kMinHeight
,
240 GetDesiredMaxWidthAndHeight(constraints
, &max_width
, &max_height
);
242 if (min_width
> max_width
|| min_height
> max_height
)
243 return media::VideoCaptureFormats();
245 double min_frame_rate
= 0.0f
;
246 double max_frame_rate
= 0.0f
;
247 if (GetConstraintValueAsDouble(constraints
,
248 MediaStreamVideoSource::kMaxFrameRate
,
250 GetConstraintValueAsDouble(constraints
,
251 MediaStreamVideoSource::kMinFrameRate
,
253 if (min_frame_rate
> max_frame_rate
) {
254 DLOG(WARNING
) << "Wrong requested frame rate.";
255 return media::VideoCaptureFormats();
259 blink::WebVector
<blink::WebMediaConstraint
> mandatory
;
260 blink::WebVector
<blink::WebMediaConstraint
> optional
;
261 constraints
.getMandatoryConstraints(mandatory
);
262 constraints
.getOptionalConstraints(optional
);
263 media::VideoCaptureFormats candidates
= supported_formats
;
264 for (const auto& constraint
: mandatory
) {
265 FilterFormatsByConstraint(constraint
, true, &candidates
);
266 if (candidates
.empty()) {
267 *unsatisfied_constraint
= constraint
.m_name
;
272 if (candidates
.empty())
275 // Ok - all mandatory checked and we still have candidates.
276 // Let's try filtering using the optional constraints. The optional
277 // constraints must be filtered in the order they occur in |optional|.
278 // But if a constraint produce zero candidates, the constraint is ignored and
279 // the next constraint is tested.
280 // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#idl-def-Constraints
281 for (const auto& constraint
: optional
) {
282 media::VideoCaptureFormats current_candidates
= candidates
;
283 FilterFormatsByConstraint(constraint
, false, ¤t_candidates
);
284 if (!current_candidates
.empty())
285 candidates
= current_candidates
;
288 // We have done as good as we can to filter the supported resolutions.
292 media::VideoCaptureFormat
GetBestFormatBasedOnArea(
293 const media::VideoCaptureFormats
& formats
,
295 DCHECK(!formats
.empty());
296 const media::VideoCaptureFormat
* best_format
= nullptr;
297 int best_diff
= std::numeric_limits
<int>::max();
298 for (const auto& format
: formats
) {
299 const int diff
= abs(area
- format
.frame_size
.GetArea());
300 if (diff
< best_diff
) {
302 best_format
= &format
;
308 // Find the format that best matches the default video size.
309 // This algorithm is chosen since a resolution must be picked even if no
310 // constraints are provided. We don't just select the maximum supported
311 // resolution since higher resolutions cost more in terms of complexity and
312 // many cameras have lower frame rate and have more noise in the image at
313 // their maximum supported resolution.
314 media::VideoCaptureFormat
GetBestCaptureFormat(
315 const media::VideoCaptureFormats
& formats
,
316 const blink::WebMediaConstraints
& constraints
) {
317 DCHECK(!formats
.empty());
321 GetDesiredMaxWidthAndHeight(constraints
, &max_width
, &max_height
);
324 static_cast<int>(MediaStreamVideoSource::kDefaultWidth
)) *
326 static_cast<int>(MediaStreamVideoSource::kDefaultHeight
));
328 return GetBestFormatBasedOnArea(formats
, area
);
331 } // anonymous namespace
334 MediaStreamVideoSource
* MediaStreamVideoSource::GetVideoSource(
335 const blink::WebMediaStreamSource
& source
) {
336 return static_cast<MediaStreamVideoSource
*>(source
.extraData());
340 bool MediaStreamVideoSource::IsConstraintSupported(const std::string
& name
) {
341 return std::find(kSupportedConstraints
,
342 kSupportedConstraints
+ arraysize(kSupportedConstraints
),
344 kSupportedConstraints
+ arraysize(kSupportedConstraints
);
347 MediaStreamVideoSource::MediaStreamVideoSource()
350 new VideoTrackAdapter(ChildProcess::current()->io_task_runner())),
351 weak_factory_(this) {
354 MediaStreamVideoSource::~MediaStreamVideoSource() {
355 DCHECK(CalledOnValidThread());
358 void MediaStreamVideoSource::AddTrack(
359 MediaStreamVideoTrack
* track
,
360 const VideoCaptureDeliverFrameCB
& frame_callback
,
361 const blink::WebMediaConstraints
& constraints
,
362 const ConstraintsCallback
& callback
) {
363 DCHECK(CalledOnValidThread());
364 DCHECK(!constraints
.isNull());
365 DCHECK(std::find(tracks_
.begin(), tracks_
.end(), track
) == tracks_
.end());
366 tracks_
.push_back(track
);
368 track_descriptors_
.push_back(
369 TrackDescriptor(track
, frame_callback
, constraints
, callback
));
373 // Tab capture and Screen capture needs the maximum requested height
374 // and width to decide on the resolution.
375 int max_requested_width
= 0;
376 GetMandatoryConstraintValueAsInteger(constraints
, kMaxWidth
,
377 &max_requested_width
);
379 int max_requested_height
= 0;
380 GetMandatoryConstraintValueAsInteger(constraints
, kMaxHeight
,
381 &max_requested_height
);
383 double max_requested_frame_rate
= kDefaultFrameRate
;
384 GetConstraintValueAsDouble(constraints
, kMaxFrameRate
,
385 &max_requested_frame_rate
);
387 state_
= RETRIEVING_CAPABILITIES
;
388 GetCurrentSupportedFormats(
390 max_requested_height
,
391 max_requested_frame_rate
,
392 base::Bind(&MediaStreamVideoSource::OnSupportedFormats
,
393 weak_factory_
.GetWeakPtr()));
398 case RETRIEVING_CAPABILITIES
: {
399 // The |callback| will be triggered once the source has started or
400 // the capabilities have been retrieved.
405 // Currently, reconfiguring the source is not supported.
411 void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack
* video_track
) {
412 DCHECK(CalledOnValidThread());
413 std::vector
<MediaStreamVideoTrack
*>::iterator it
=
414 std::find(tracks_
.begin(), tracks_
.end(), video_track
);
415 DCHECK(it
!= tracks_
.end());
418 for (std::vector
<TrackDescriptor
>::iterator it
= track_descriptors_
.begin();
419 it
!= track_descriptors_
.end(); ++it
) {
420 if (it
->track
== video_track
) {
421 track_descriptors_
.erase(it
);
426 // Call |frame_adapter_->RemoveTrack| here even if adding the track has
427 // failed and |frame_adapter_->AddCallback| has not been called.
428 track_adapter_
->RemoveTrack(video_track
);
434 base::SingleThreadTaskRunner
* MediaStreamVideoSource::io_task_runner() const {
435 DCHECK(CalledOnValidThread());
436 return track_adapter_
->io_task_runner();
439 void MediaStreamVideoSource::DoStopSource() {
440 DCHECK(CalledOnValidThread());
441 DVLOG(3) << "DoStopSource()";
444 track_adapter_
->StopFrameMonitoring();
447 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded
);
450 void MediaStreamVideoSource::OnSupportedFormats(
451 const media::VideoCaptureFormats
& formats
) {
452 DCHECK(CalledOnValidThread());
453 DCHECK_EQ(RETRIEVING_CAPABILITIES
, state_
);
455 supported_formats_
= formats
;
456 blink::WebMediaConstraints fulfilled_constraints
;
457 if (!FindBestFormatWithConstraints(supported_formats_
,
459 &fulfilled_constraints
)) {
460 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded
);
461 // This object can be deleted after calling FinalizeAddTrack. See comment
462 // in the header file.
468 DVLOG(3) << "Starting the capturer with "
469 << media::VideoCaptureFormat::ToString(current_format_
);
473 fulfilled_constraints
,
474 base::Bind(&VideoTrackAdapter::DeliverFrameOnIO
, track_adapter_
));
477 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
478 const media::VideoCaptureFormats
& formats
,
479 media::VideoCaptureFormat
* best_format
,
480 blink::WebMediaConstraints
* fulfilled_constraints
) {
481 DCHECK(CalledOnValidThread());
482 // Find the first constraints that we can fulfill.
483 for (const auto& track
: track_descriptors_
) {
484 const blink::WebMediaConstraints
& track_constraints
= track
.constraints
;
486 // If the source doesn't support capability enumeration it is still ok if
487 // no mandatory constraints have been specified. That just means that
488 // we will start with whatever format is native to the source.
489 if (formats
.empty() && !HasMandatoryConstraints(track_constraints
)) {
490 *fulfilled_constraints
= track_constraints
;
491 *best_format
= media::VideoCaptureFormat();
494 blink::WebString unsatisfied_constraint
;
495 const media::VideoCaptureFormats filtered_formats
=
496 FilterFormats(track_constraints
, formats
, &unsatisfied_constraint
);
497 if (filtered_formats
.empty())
500 // A request with constraints that can be fulfilled.
501 *fulfilled_constraints
= track_constraints
;
502 *best_format
= GetBestCaptureFormat(filtered_formats
, track_constraints
);
508 void MediaStreamVideoSource::OnStartDone(MediaStreamRequestResult result
) {
509 DCHECK(CalledOnValidThread());
510 DVLOG(3) << "OnStartDone({result =" << result
<< "})";
511 if (result
== MEDIA_DEVICE_OK
) {
512 DCHECK_EQ(STARTING
, state_
);
514 SetReadyState(blink::WebMediaStreamSource::ReadyStateLive
);
516 track_adapter_
->StartFrameMonitoring(
517 current_format_
.frame_rate
,
518 base::Bind(&MediaStreamVideoSource::SetMutedState
,
519 weak_factory_
.GetWeakPtr()));
525 // This object can be deleted after calling FinalizeAddTrack. See comment in
530 void MediaStreamVideoSource::FinalizeAddTrack() {
531 DCHECK(CalledOnValidThread());
532 const media::VideoCaptureFormats
formats(1, current_format_
);
534 std::vector
<TrackDescriptor
> track_descriptors
;
535 track_descriptors
.swap(track_descriptors_
);
536 for (const auto& track
: track_descriptors
) {
537 MediaStreamRequestResult result
= MEDIA_DEVICE_OK
;
538 blink::WebString unsatisfied_constraint
;
540 if (HasMandatoryConstraints(track
.constraints
) &&
541 FilterFormats(track
.constraints
, formats
, &unsatisfied_constraint
)
543 result
= MEDIA_DEVICE_CONSTRAINT_NOT_SATISFIED
;
546 if (state_
!= STARTED
&& result
== MEDIA_DEVICE_OK
)
547 result
= MEDIA_DEVICE_TRACK_START_FAILURE
;
549 if (result
== MEDIA_DEVICE_OK
) {
552 GetDesiredMaxWidthAndHeight(track
.constraints
, &max_width
, &max_height
);
553 double max_aspect_ratio
;
554 double min_aspect_ratio
;
555 GetDesiredMinAndMaxAspectRatio(track
.constraints
,
558 double max_frame_rate
= 0.0f
;
559 GetConstraintValueAsDouble(track
.constraints
,
560 kMaxFrameRate
, &max_frame_rate
);
562 track_adapter_
->AddTrack(track
.track
, track
.frame_callback
, max_width
,
563 max_height
, min_aspect_ratio
, max_aspect_ratio
,
567 DVLOG(3) << "FinalizeAddTrack() result " << result
;
569 if (!track
.callback
.is_null())
570 track
.callback
.Run(this, result
, unsatisfied_constraint
);
574 void MediaStreamVideoSource::SetReadyState(
575 blink::WebMediaStreamSource::ReadyState state
) {
576 DVLOG(3) << "MediaStreamVideoSource::SetReadyState state " << state
;
577 DCHECK(CalledOnValidThread());
578 if (!owner().isNull())
579 owner().setReadyState(state
);
580 for (const auto& track
: tracks_
)
581 track
->OnReadyStateChanged(state
);
584 void MediaStreamVideoSource::SetMutedState(bool muted_state
) {
585 DVLOG(3) << "MediaStreamVideoSource::SetMutedState state=" << muted_state
;
586 DCHECK(CalledOnValidThread());
587 if (!owner().isNull()) {
588 owner().setReadyState(muted_state
589 ? blink::WebMediaStreamSource::ReadyStateMuted
590 : blink::WebMediaStreamSource::ReadyStateLive
);
594 MediaStreamVideoSource::TrackDescriptor::TrackDescriptor(
595 MediaStreamVideoTrack
* track
,
596 const VideoCaptureDeliverFrameCB
& frame_callback
,
597 const blink::WebMediaConstraints
& constraints
,
598 const ConstraintsCallback
& callback
)
600 frame_callback(frame_callback
),
601 constraints(constraints
),
605 MediaStreamVideoSource::TrackDescriptor::~TrackDescriptor() {
608 } // namespace content