1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/media_stream_video_source.h"
11 #include "base/logging.h"
12 #include "base/strings/string_number_conversions.h"
13 #include "base/trace_event/trace_event.h"
14 #include "content/child/child_process.h"
15 #include "content/renderer/media/media_stream_constraints_util.h"
16 #include "content/renderer/media/media_stream_video_track.h"
17 #include "content/renderer/media/video_track_adapter.h"
21 // Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b
22 const char MediaStreamVideoSource::kMinAspectRatio
[] = "minAspectRatio";
23 const char MediaStreamVideoSource::kMaxAspectRatio
[] = "maxAspectRatio";
24 const char MediaStreamVideoSource::kMaxWidth
[] = "maxWidth";
25 const char MediaStreamVideoSource::kMinWidth
[] = "minWidth";
26 const char MediaStreamVideoSource::kMaxHeight
[] = "maxHeight";
27 const char MediaStreamVideoSource::kMinHeight
[] = "minHeight";
28 const char MediaStreamVideoSource::kMaxFrameRate
[] = "maxFrameRate";
29 const char MediaStreamVideoSource::kMinFrameRate
[] = "minFrameRate";
31 const char* kSupportedConstraints
[] = {
32 MediaStreamVideoSource::kMaxAspectRatio
,
33 MediaStreamVideoSource::kMinAspectRatio
,
34 MediaStreamVideoSource::kMaxWidth
,
35 MediaStreamVideoSource::kMinWidth
,
36 MediaStreamVideoSource::kMaxHeight
,
37 MediaStreamVideoSource::kMinHeight
,
38 MediaStreamVideoSource::kMaxFrameRate
,
39 MediaStreamVideoSource::kMinFrameRate
,
44 // Google-specific key prefix. Constraints with this prefix are ignored if they
46 const char kGooglePrefix
[] = "goog";
48 // Returns true if |constraint| has mandatory constraints.
49 bool HasMandatoryConstraints(const blink::WebMediaConstraints
& constraints
) {
50 blink::WebVector
<blink::WebMediaConstraint
> mandatory_constraints
;
51 constraints
.getMandatoryConstraints(mandatory_constraints
);
52 return !mandatory_constraints
.isEmpty();
55 // Retrieve the desired max width and height from |constraints|. If not set,
56 // the |desired_width| and |desired_height| are set to
57 // std::numeric_limits<int>::max();
58 // If either max width or height is set as a mandatory constraint, the optional
59 // constraints are not checked.
60 void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints
& constraints
,
61 int* desired_width
, int* desired_height
) {
62 *desired_width
= std::numeric_limits
<int>::max();
63 *desired_height
= std::numeric_limits
<int>::max();
65 bool mandatory
= GetMandatoryConstraintValueAsInteger(
67 MediaStreamVideoSource::kMaxWidth
,
69 mandatory
|= GetMandatoryConstraintValueAsInteger(
71 MediaStreamVideoSource::kMaxHeight
,
76 GetOptionalConstraintValueAsInteger(constraints
,
77 MediaStreamVideoSource::kMaxWidth
,
79 GetOptionalConstraintValueAsInteger(constraints
,
80 MediaStreamVideoSource::kMaxHeight
,
84 // Retrieve the desired max and min aspect ratio from |constraints|. If not set,
85 // the |min_aspect_ratio| is set to 0 and |max_aspect_ratio| is set to
86 // std::numeric_limits<double>::max();
87 // If either min or max aspect ratio is set as a mandatory constraint, the
88 // optional constraints are not checked.
89 void GetDesiredMinAndMaxAspectRatio(
90 const blink::WebMediaConstraints
& constraints
,
91 double* min_aspect_ratio
,
92 double* max_aspect_ratio
) {
93 *min_aspect_ratio
= 0;
94 *max_aspect_ratio
= std::numeric_limits
<double>::max();
96 bool mandatory
= GetMandatoryConstraintValueAsDouble(
98 MediaStreamVideoSource::kMinAspectRatio
,
100 mandatory
|= GetMandatoryConstraintValueAsDouble(
102 MediaStreamVideoSource::kMaxAspectRatio
,
107 GetOptionalConstraintValueAsDouble(
109 MediaStreamVideoSource::kMinAspectRatio
,
111 GetOptionalConstraintValueAsDouble(
113 MediaStreamVideoSource::kMaxAspectRatio
,
117 // Returns true if |constraint| is fulfilled. |format| can be changed by a
118 // constraint, e.g. the frame rate can be changed by setting maxFrameRate.
119 bool UpdateFormatForConstraint(
120 const blink::WebMediaConstraint
& constraint
,
122 media::VideoCaptureFormat
* format
) {
123 DCHECK(format
!= NULL
);
125 if (!format
->IsValid())
128 std::string constraint_name
= constraint
.m_name
.utf8();
129 std::string constraint_value
= constraint
.m_value
.utf8();
131 if (constraint_name
.find(kGooglePrefix
) == 0) {
132 // These are actually options, not constraints, so they can be satisfied
133 // regardless of the format.
137 if (constraint_name
== MediaStreamSource::kSourceId
) {
138 // This is a constraint that doesn't affect the format.
142 // Ignore Chrome specific Tab capture constraints.
143 if (constraint_name
== kMediaStreamSource
||
144 constraint_name
== kMediaStreamSourceId
)
147 if (constraint_name
== MediaStreamVideoSource::kMinAspectRatio
||
148 constraint_name
== MediaStreamVideoSource::kMaxAspectRatio
) {
149 // These constraints are handled by cropping if the camera outputs the wrong
152 return base::StringToDouble(constraint_value
, &value
);
156 if (!base::StringToDouble(constraint_value
, &value
)) {
157 DLOG(WARNING
) << "Can't parse MediaStream constraint. Name:"
158 << constraint_name
<< " Value:" << constraint_value
;
162 if (constraint_name
== MediaStreamVideoSource::kMinWidth
) {
163 return (value
<= format
->frame_size
.width());
164 } else if (constraint_name
== MediaStreamVideoSource::kMaxWidth
) {
166 } else if (constraint_name
== MediaStreamVideoSource::kMinHeight
) {
167 return (value
<= format
->frame_size
.height());
168 } else if (constraint_name
== MediaStreamVideoSource::kMaxHeight
) {
170 } else if (constraint_name
== MediaStreamVideoSource::kMinFrameRate
) {
171 return (value
> 0.0) && (value
<= format
->frame_rate
);
172 } else if (constraint_name
== MediaStreamVideoSource::kMaxFrameRate
) {
174 // The frame rate is set by constraint.
175 // Don't allow 0 as frame rate if it is a mandatory constraint.
176 // Set the frame rate to 1 if it is not mandatory.
184 (format
->frame_rate
> value
) ? value
: format
->frame_rate
;
187 LOG(WARNING
) << "Found unknown MediaStream constraint. Name:"
188 << constraint_name
<< " Value:" << constraint_value
;
193 // Removes media::VideoCaptureFormats from |formats| that don't meet
195 void FilterFormatsByConstraint(
196 const blink::WebMediaConstraint
& constraint
,
198 media::VideoCaptureFormats
* formats
) {
199 DVLOG(3) << "FilterFormatsByConstraint("
200 << "{ constraint.m_name = " << constraint
.m_name
.utf8()
201 << " constraint.m_value = " << constraint
.m_value
.utf8()
202 << " mandatory = " << mandatory
<< "})";
203 media::VideoCaptureFormats::iterator format_it
= formats
->begin();
204 while (format_it
!= formats
->end()) {
205 // Modify the format_it to fulfill the constraint if possible.
206 // Delete it otherwise.
207 if (!UpdateFormatForConstraint(constraint
, mandatory
, &(*format_it
)))
208 format_it
= formats
->erase(format_it
);
214 // Returns the media::VideoCaptureFormats that matches |constraints|.
215 media::VideoCaptureFormats
FilterFormats(
216 const blink::WebMediaConstraints
& constraints
,
217 const media::VideoCaptureFormats
& supported_formats
,
218 blink::WebString
* unsatisfied_constraint
) {
219 if (constraints
.isNull())
220 return supported_formats
;
222 double max_aspect_ratio
;
223 double min_aspect_ratio
;
224 GetDesiredMinAndMaxAspectRatio(constraints
,
228 if (min_aspect_ratio
> max_aspect_ratio
|| max_aspect_ratio
< 0.05f
) {
229 DLOG(WARNING
) << "Wrong requested aspect ratio.";
230 return media::VideoCaptureFormats();
234 GetMandatoryConstraintValueAsInteger(constraints
,
235 MediaStreamVideoSource::kMinWidth
,
238 GetMandatoryConstraintValueAsInteger(constraints
,
239 MediaStreamVideoSource::kMinHeight
,
243 GetDesiredMaxWidthAndHeight(constraints
, &max_width
, &max_height
);
245 if (min_width
> max_width
|| min_height
> max_height
)
246 return media::VideoCaptureFormats();
248 double min_frame_rate
= 0.0f
;
249 double max_frame_rate
= 0.0f
;
250 if (GetConstraintValueAsDouble(constraints
,
251 MediaStreamVideoSource::kMaxFrameRate
,
253 GetConstraintValueAsDouble(constraints
,
254 MediaStreamVideoSource::kMinFrameRate
,
256 if (min_frame_rate
> max_frame_rate
) {
257 DLOG(WARNING
) << "Wrong requested frame rate.";
258 return media::VideoCaptureFormats();
262 blink::WebVector
<blink::WebMediaConstraint
> mandatory
;
263 blink::WebVector
<blink::WebMediaConstraint
> optional
;
264 constraints
.getMandatoryConstraints(mandatory
);
265 constraints
.getOptionalConstraints(optional
);
266 media::VideoCaptureFormats candidates
= supported_formats
;
267 for (size_t i
= 0; i
< mandatory
.size(); ++i
) {
268 FilterFormatsByConstraint(mandatory
[i
], true, &candidates
);
269 if (candidates
.empty()) {
270 *unsatisfied_constraint
= mandatory
[i
].m_name
;
275 if (candidates
.empty())
278 // Ok - all mandatory checked and we still have candidates.
279 // Let's try filtering using the optional constraints. The optional
280 // constraints must be filtered in the order they occur in |optional|.
281 // But if a constraint produce zero candidates, the constraint is ignored and
282 // the next constraint is tested.
283 // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#idl-def-Constraints
284 for (size_t i
= 0; i
< optional
.size(); ++i
) {
285 media::VideoCaptureFormats current_candidates
= candidates
;
286 FilterFormatsByConstraint(optional
[i
], false, ¤t_candidates
);
287 if (!current_candidates
.empty())
288 candidates
= current_candidates
;
291 // We have done as good as we can to filter the supported resolutions.
295 const media::VideoCaptureFormat
& GetBestFormatBasedOnArea(
296 const media::VideoCaptureFormats
& formats
,
298 media::VideoCaptureFormats::const_iterator it
= formats
.begin();
299 media::VideoCaptureFormats::const_iterator best_it
= formats
.begin();
300 int best_diff
= std::numeric_limits
<int>::max();
301 for (; it
!= formats
.end(); ++it
) {
302 const int diff
= abs(area
- it
->frame_size
.GetArea());
303 if (diff
< best_diff
) {
311 // Find the format that best matches the default video size.
312 // This algorithm is chosen since a resolution must be picked even if no
313 // constraints are provided. We don't just select the maximum supported
314 // resolution since higher resolutions cost more in terms of complexity and
315 // many cameras have lower frame rate and have more noise in the image at
316 // their maximum supported resolution.
317 void GetBestCaptureFormat(
318 const media::VideoCaptureFormats
& formats
,
319 const blink::WebMediaConstraints
& constraints
,
320 media::VideoCaptureFormat
* capture_format
) {
321 DCHECK(!formats
.empty());
325 GetDesiredMaxWidthAndHeight(constraints
, &max_width
, &max_height
);
327 *capture_format
= GetBestFormatBasedOnArea(
330 static_cast<int>(MediaStreamVideoSource::kDefaultWidth
)) *
332 static_cast<int>(MediaStreamVideoSource::kDefaultHeight
)));
335 } // anonymous namespace
338 MediaStreamVideoSource
* MediaStreamVideoSource::GetVideoSource(
339 const blink::WebMediaStreamSource
& source
) {
340 return static_cast<MediaStreamVideoSource
*>(source
.extraData());
344 bool MediaStreamVideoSource::IsConstraintSupported(const std::string
& name
) {
345 for (const char* constraint
: kSupportedConstraints
) {
346 if (constraint
== name
)
352 MediaStreamVideoSource::MediaStreamVideoSource()
355 new VideoTrackAdapter(ChildProcess::current()->io_task_runner())),
356 weak_factory_(this) {
359 MediaStreamVideoSource::~MediaStreamVideoSource() {
360 DCHECK(CalledOnValidThread());
363 void MediaStreamVideoSource::AddTrack(
364 MediaStreamVideoTrack
* track
,
365 const VideoCaptureDeliverFrameCB
& frame_callback
,
366 const blink::WebMediaConstraints
& constraints
,
367 const ConstraintsCallback
& callback
) {
368 DCHECK(CalledOnValidThread());
369 DCHECK(!constraints
.isNull());
370 DCHECK(std::find(tracks_
.begin(), tracks_
.end(),
371 track
) == tracks_
.end());
372 tracks_
.push_back(track
);
374 requested_constraints_
.push_back(
375 RequestedConstraints(track
, frame_callback
, constraints
, callback
));
379 // Tab capture and Screen capture needs the maximum requested height
380 // and width to decide on the resolution.
381 int max_requested_width
= 0;
382 GetMandatoryConstraintValueAsInteger(constraints
, kMaxWidth
,
383 &max_requested_width
);
385 int max_requested_height
= 0;
386 GetMandatoryConstraintValueAsInteger(constraints
, kMaxHeight
,
387 &max_requested_height
);
389 double max_requested_frame_rate
;
390 if (!GetConstraintValueAsDouble(constraints
, kMaxFrameRate
,
391 &max_requested_frame_rate
)) {
392 max_requested_frame_rate
= kDefaultFrameRate
;
395 state_
= RETRIEVING_CAPABILITIES
;
396 GetCurrentSupportedFormats(
398 max_requested_height
,
399 max_requested_frame_rate
,
400 base::Bind(&MediaStreamVideoSource::OnSupportedFormats
,
401 weak_factory_
.GetWeakPtr()));
406 case RETRIEVING_CAPABILITIES
: {
407 // The |callback| will be triggered once the source has started or
408 // the capabilities have been retrieved.
413 // Currently, reconfiguring the source is not supported.
419 void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack
* video_track
) {
420 DCHECK(CalledOnValidThread());
421 std::vector
<MediaStreamVideoTrack
*>::iterator it
=
422 std::find(tracks_
.begin(), tracks_
.end(), video_track
);
423 DCHECK(it
!= tracks_
.end());
426 // Check if |video_track| is waiting for applying new constraints and remove
427 // the request in that case.
428 for (std::vector
<RequestedConstraints
>::iterator it
=
429 requested_constraints_
.begin();
430 it
!= requested_constraints_
.end(); ++it
) {
431 if (it
->track
== video_track
) {
432 requested_constraints_
.erase(it
);
436 // Call |frame_adapter_->RemoveTrack| here even if adding the track has
437 // failed and |frame_adapter_->AddCallback| has not been called.
438 track_adapter_
->RemoveTrack(video_track
);
444 base::SingleThreadTaskRunner
* MediaStreamVideoSource::io_task_runner() const {
445 DCHECK(CalledOnValidThread());
446 return track_adapter_
->io_task_runner();
449 void MediaStreamVideoSource::DoStopSource() {
450 DCHECK(CalledOnValidThread());
451 DVLOG(3) << "DoStopSource()";
454 track_adapter_
->StopFrameMonitoring();
457 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded
);
460 void MediaStreamVideoSource::OnSupportedFormats(
461 const media::VideoCaptureFormats
& formats
) {
462 DCHECK(CalledOnValidThread());
463 DCHECK_EQ(RETRIEVING_CAPABILITIES
, state_
);
465 supported_formats_
= formats
;
466 blink::WebMediaConstraints fulfilled_constraints
;
467 if (!FindBestFormatWithConstraints(supported_formats_
,
469 &fulfilled_constraints
)) {
470 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded
);
471 // This object can be deleted after calling FinalizeAddTrack. See comment
472 // in the header file.
478 DVLOG(3) << "Starting the capturer with "
479 << media::VideoCaptureFormat::ToString(current_format_
);
483 fulfilled_constraints
,
484 base::Bind(&VideoTrackAdapter::DeliverFrameOnIO
, track_adapter_
));
487 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
488 const media::VideoCaptureFormats
& formats
,
489 media::VideoCaptureFormat
* best_format
,
490 blink::WebMediaConstraints
* fulfilled_constraints
) {
491 DCHECK(CalledOnValidThread());
492 // Find the first constraints that we can fulfill.
493 for (const auto& request
: requested_constraints_
) {
494 const blink::WebMediaConstraints
& requested_constraints
=
497 // If the source doesn't support capability enumeration it is still ok if
498 // no mandatory constraints have been specified. That just means that
499 // we will start with whatever format is native to the source.
500 if (formats
.empty() && !HasMandatoryConstraints(requested_constraints
)) {
501 *fulfilled_constraints
= requested_constraints
;
502 *best_format
= media::VideoCaptureFormat();
505 blink::WebString unsatisfied_constraint
;
506 media::VideoCaptureFormats filtered_formats
=
507 FilterFormats(requested_constraints
, formats
, &unsatisfied_constraint
);
508 if (filtered_formats
.size() > 0) {
509 // A request with constraints that can be fulfilled.
510 *fulfilled_constraints
= requested_constraints
;
511 GetBestCaptureFormat(filtered_formats
,
512 requested_constraints
,
520 void MediaStreamVideoSource::OnStartDone(MediaStreamRequestResult result
) {
521 DCHECK(CalledOnValidThread());
522 DVLOG(3) << "OnStartDone({result =" << result
<< "})";
523 if (result
== MEDIA_DEVICE_OK
) {
524 DCHECK_EQ(STARTING
, state_
);
526 SetReadyState(blink::WebMediaStreamSource::ReadyStateLive
);
528 track_adapter_
->StartFrameMonitoring(
529 current_format_
.frame_rate
,
530 base::Bind(&MediaStreamVideoSource::SetMutedState
,
531 weak_factory_
.GetWeakPtr()));
537 // This object can be deleted after calling FinalizeAddTrack. See comment in
542 void MediaStreamVideoSource::FinalizeAddTrack() {
543 DCHECK(CalledOnValidThread());
544 media::VideoCaptureFormats formats
;
545 formats
.push_back(current_format_
);
547 std::vector
<RequestedConstraints
> callbacks
;
548 callbacks
.swap(requested_constraints_
);
549 for (const auto& request
: callbacks
) {
550 MediaStreamRequestResult result
= MEDIA_DEVICE_OK
;
551 blink::WebString unsatisfied_constraint
;
553 if (HasMandatoryConstraints(request
.constraints
) &&
554 FilterFormats(request
.constraints
, formats
,
555 &unsatisfied_constraint
).empty()) {
556 result
= MEDIA_DEVICE_CONSTRAINT_NOT_SATISFIED
;
559 if (state_
!= STARTED
&& result
== MEDIA_DEVICE_OK
)
560 result
= MEDIA_DEVICE_TRACK_START_FAILURE
;
562 if (result
== MEDIA_DEVICE_OK
) {
565 GetDesiredMaxWidthAndHeight(request
.constraints
, &max_width
, &max_height
);
566 double max_aspect_ratio
;
567 double min_aspect_ratio
;
568 GetDesiredMinAndMaxAspectRatio(request
.constraints
,
571 double max_frame_rate
= 0.0f
;
572 GetConstraintValueAsDouble(request
.constraints
,
573 kMaxFrameRate
, &max_frame_rate
);
575 track_adapter_
->AddTrack(request
.track
, request
.frame_callback
,
576 max_width
, max_height
,
577 min_aspect_ratio
, max_aspect_ratio
,
581 DVLOG(3) << "FinalizeAddTrack() result " << result
;
583 if (!request
.callback
.is_null()) {
584 request
.callback
.Run(this, result
, unsatisfied_constraint
);
589 void MediaStreamVideoSource::SetReadyState(
590 blink::WebMediaStreamSource::ReadyState state
) {
591 DVLOG(3) << "MediaStreamVideoSource::SetReadyState state " << state
;
592 DCHECK(CalledOnValidThread());
593 if (!owner().isNull())
594 owner().setReadyState(state
);
595 for (const auto& track
: tracks_
)
596 track
->OnReadyStateChanged(state
);
599 void MediaStreamVideoSource::SetMutedState(bool muted_state
) {
600 DVLOG(3) << "MediaStreamVideoSource::SetMutedState state=" << muted_state
;
601 DCHECK(CalledOnValidThread());
602 if (!owner().isNull()) {
603 owner().setReadyState(muted_state
604 ? blink::WebMediaStreamSource::ReadyStateMuted
605 : blink::WebMediaStreamSource::ReadyStateLive
);
609 MediaStreamVideoSource::RequestedConstraints::RequestedConstraints(
610 MediaStreamVideoTrack
* track
,
611 const VideoCaptureDeliverFrameCB
& frame_callback
,
612 const blink::WebMediaConstraints
& constraints
,
613 const ConstraintsCallback
& callback
)
615 frame_callback(frame_callback
),
616 constraints(constraints
),
620 MediaStreamVideoSource::RequestedConstraints::~RequestedConstraints() {
623 } // namespace content