Roll ANGLE bc75f36:ef9d63e
[chromium-blink-merge.git] / content / renderer / media / media_stream_video_source.cc
blob0bd016bd92fa41faa0653c73800e072866253f14
1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/media_stream_video_source.h"
7 #include <algorithm>
8 #include <limits>
9 #include <string>
11 #include "base/debug/trace_event.h"
12 #include "base/logging.h"
13 #include "base/strings/string_number_conversions.h"
14 #include "content/child/child_process.h"
15 #include "content/renderer/media/media_stream_constraints_util.h"
16 #include "content/renderer/media/media_stream_video_track.h"
17 #include "content/renderer/media/video_track_adapter.h"
18 #include "media/base/bind_to_current_loop.h"
20 namespace content {
22 // Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b
23 const char MediaStreamVideoSource::kMinAspectRatio[] = "minAspectRatio";
24 const char MediaStreamVideoSource::kMaxAspectRatio[] = "maxAspectRatio";
25 const char MediaStreamVideoSource::kMaxWidth[] = "maxWidth";
26 const char MediaStreamVideoSource::kMinWidth[] = "minWidth";
27 const char MediaStreamVideoSource::kMaxHeight[] = "maxHeight";
28 const char MediaStreamVideoSource::kMinHeight[] = "minHeight";
29 const char MediaStreamVideoSource::kMaxFrameRate[] = "maxFrameRate";
30 const char MediaStreamVideoSource::kMinFrameRate[] = "minFrameRate";
32 const char* kSupportedConstraints[] = {
33 MediaStreamVideoSource::kMaxAspectRatio,
34 MediaStreamVideoSource::kMinAspectRatio,
35 MediaStreamVideoSource::kMaxWidth,
36 MediaStreamVideoSource::kMinWidth,
37 MediaStreamVideoSource::kMaxHeight,
38 MediaStreamVideoSource::kMinHeight,
39 MediaStreamVideoSource::kMaxFrameRate,
40 MediaStreamVideoSource::kMinFrameRate,
43 const int MediaStreamVideoSource::kDefaultWidth = 640;
44 const int MediaStreamVideoSource::kDefaultHeight = 480;
45 const int MediaStreamVideoSource::kDefaultFrameRate = 30;
47 namespace {
49 // Google-specific key prefix. Constraints with this prefix are ignored if they
50 // are unknown.
51 const char kGooglePrefix[] = "goog";
53 // Returns true if |constraint| has mandatory constraints.
54 bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
55 blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
56 constraints.getMandatoryConstraints(mandatory_constraints);
57 return !mandatory_constraints.isEmpty();
60 // Retrieve the desired max width and height from |constraints|. If not set,
61 // the |desired_width| and |desired_height| are set to
62 // std::numeric_limits<int>::max();
63 // If either max width or height is set as a mandatory constraint, the optional
64 // constraints are not checked.
65 void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
66 int* desired_width, int* desired_height) {
67 *desired_width = std::numeric_limits<int>::max();
68 *desired_height = std::numeric_limits<int>::max();
70 bool mandatory = GetMandatoryConstraintValueAsInteger(
71 constraints,
72 MediaStreamVideoSource::kMaxWidth,
73 desired_width);
74 mandatory |= GetMandatoryConstraintValueAsInteger(
75 constraints,
76 MediaStreamVideoSource::kMaxHeight,
77 desired_height);
78 if (mandatory)
79 return;
81 GetOptionalConstraintValueAsInteger(constraints,
82 MediaStreamVideoSource::kMaxWidth,
83 desired_width);
84 GetOptionalConstraintValueAsInteger(constraints,
85 MediaStreamVideoSource::kMaxHeight,
86 desired_height);
89 // Retrieve the desired max and min aspect ratio from |constraints|. If not set,
90 // the |min_aspect_ratio| is set to 0 and |max_aspect_ratio| is set to
91 // std::numeric_limits<double>::max();
92 // If either min or max aspect ratio is set as a mandatory constraint, the
93 // optional constraints are not checked.
94 void GetDesiredMinAndMaxAspectRatio(
95 const blink::WebMediaConstraints& constraints,
96 double* min_aspect_ratio,
97 double* max_aspect_ratio) {
98 *min_aspect_ratio = 0;
99 *max_aspect_ratio = std::numeric_limits<double>::max();
101 bool mandatory = GetMandatoryConstraintValueAsDouble(
102 constraints,
103 MediaStreamVideoSource::kMinAspectRatio,
104 min_aspect_ratio);
105 mandatory |= GetMandatoryConstraintValueAsDouble(
106 constraints,
107 MediaStreamVideoSource::kMaxAspectRatio,
108 max_aspect_ratio);
109 if (mandatory)
110 return;
112 GetOptionalConstraintValueAsDouble(
113 constraints,
114 MediaStreamVideoSource::kMinAspectRatio,
115 min_aspect_ratio);
116 GetOptionalConstraintValueAsDouble(
117 constraints,
118 MediaStreamVideoSource::kMaxAspectRatio,
119 max_aspect_ratio);
122 // Returns true if |constraint| is fulfilled. |format| can be changed by a
123 // constraint, e.g. the frame rate can be changed by setting maxFrameRate.
124 bool UpdateFormatForConstraint(
125 const blink::WebMediaConstraint& constraint,
126 bool mandatory,
127 media::VideoCaptureFormat* format) {
128 DCHECK(format != NULL);
130 if (!format->IsValid())
131 return false;
133 std::string constraint_name = constraint.m_name.utf8();
134 std::string constraint_value = constraint.m_value.utf8();
136 if (constraint_name.find(kGooglePrefix) == 0) {
137 // These are actually options, not constraints, so they can be satisfied
138 // regardless of the format.
139 return true;
142 if (constraint_name == MediaStreamSource::kSourceId) {
143 // This is a constraint that doesn't affect the format.
144 return true;
147 // Ignore Chrome specific Tab capture constraints.
148 if (constraint_name == kMediaStreamSource ||
149 constraint_name == kMediaStreamSourceId)
150 return true;
152 if (constraint_name == MediaStreamVideoSource::kMinAspectRatio ||
153 constraint_name == MediaStreamVideoSource::kMaxAspectRatio) {
154 // These constraints are handled by cropping if the camera outputs the wrong
155 // aspect ratio.
156 double value;
157 return base::StringToDouble(constraint_value, &value);
160 double value = 0.0;
161 if (!base::StringToDouble(constraint_value, &value)) {
162 DLOG(WARNING) << "Can't parse MediaStream constraint. Name:"
163 << constraint_name << " Value:" << constraint_value;
164 return false;
167 if (constraint_name == MediaStreamVideoSource::kMinWidth) {
168 return (value <= format->frame_size.width());
169 } else if (constraint_name == MediaStreamVideoSource::kMaxWidth) {
170 return value > 0.0;
171 } else if (constraint_name == MediaStreamVideoSource::kMinHeight) {
172 return (value <= format->frame_size.height());
173 } else if (constraint_name == MediaStreamVideoSource::kMaxHeight) {
174 return value > 0.0;
175 } else if (constraint_name == MediaStreamVideoSource::kMinFrameRate) {
176 return (value > 0.0) && (value <= format->frame_rate);
177 } else if (constraint_name == MediaStreamVideoSource::kMaxFrameRate) {
178 if (value <= 0.0) {
179 // The frame rate is set by constraint.
180 // Don't allow 0 as frame rate if it is a mandatory constraint.
181 // Set the frame rate to 1 if it is not mandatory.
182 if (mandatory) {
183 return false;
184 } else {
185 value = 1.0;
188 format->frame_rate =
189 (format->frame_rate > value) ? value : format->frame_rate;
190 return true;
191 } else {
192 LOG(WARNING) << "Found unknown MediaStream constraint. Name:"
193 << constraint_name << " Value:" << constraint_value;
194 return false;
198 // Removes media::VideoCaptureFormats from |formats| that don't meet
199 // |constraint|.
200 void FilterFormatsByConstraint(
201 const blink::WebMediaConstraint& constraint,
202 bool mandatory,
203 media::VideoCaptureFormats* formats) {
204 DVLOG(3) << "FilterFormatsByConstraint("
205 << "{ constraint.m_name = " << constraint.m_name.utf8()
206 << " constraint.m_value = " << constraint.m_value.utf8()
207 << " mandatory = " << mandatory << "})";
208 media::VideoCaptureFormats::iterator format_it = formats->begin();
209 while (format_it != formats->end()) {
210 // Modify the format_it to fulfill the constraint if possible.
211 // Delete it otherwise.
212 if (!UpdateFormatForConstraint(constraint, mandatory, &(*format_it))) {
213 format_it = formats->erase(format_it);
214 } else {
215 ++format_it;
220 // Returns the media::VideoCaptureFormats that matches |constraints|.
221 media::VideoCaptureFormats FilterFormats(
222 const blink::WebMediaConstraints& constraints,
223 const media::VideoCaptureFormats& supported_formats,
224 blink::WebString* unsatisfied_constraint) {
225 if (constraints.isNull()) {
226 return supported_formats;
229 double max_aspect_ratio;
230 double min_aspect_ratio;
231 GetDesiredMinAndMaxAspectRatio(constraints,
232 &min_aspect_ratio,
233 &max_aspect_ratio);
235 if (min_aspect_ratio > max_aspect_ratio || max_aspect_ratio < 0.05f) {
236 DLOG(WARNING) << "Wrong requested aspect ratio.";
237 return media::VideoCaptureFormats();
240 int min_width = 0;
241 GetMandatoryConstraintValueAsInteger(constraints,
242 MediaStreamVideoSource::kMinWidth,
243 &min_width);
244 int min_height = 0;
245 GetMandatoryConstraintValueAsInteger(constraints,
246 MediaStreamVideoSource::kMinHeight,
247 &min_height);
248 int max_width;
249 int max_height;
250 GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
252 if (min_width > max_width || min_height > max_height)
253 return media::VideoCaptureFormats();
255 double min_frame_rate = 0.0f;
256 double max_frame_rate = 0.0f;
257 if (GetConstraintValueAsDouble(constraints,
258 MediaStreamVideoSource::kMaxFrameRate,
259 &max_frame_rate) &&
260 GetConstraintValueAsDouble(constraints,
261 MediaStreamVideoSource::kMinFrameRate,
262 &min_frame_rate)) {
263 if (min_frame_rate > max_frame_rate) {
264 DLOG(WARNING) << "Wrong requested frame rate.";
265 return media::VideoCaptureFormats();
269 blink::WebVector<blink::WebMediaConstraint> mandatory;
270 blink::WebVector<blink::WebMediaConstraint> optional;
271 constraints.getMandatoryConstraints(mandatory);
272 constraints.getOptionalConstraints(optional);
273 media::VideoCaptureFormats candidates = supported_formats;
274 for (size_t i = 0; i < mandatory.size(); ++i) {
275 FilterFormatsByConstraint(mandatory[i], true, &candidates);
276 if (candidates.empty()) {
277 *unsatisfied_constraint = mandatory[i].m_name;
278 return candidates;
282 if (candidates.empty())
283 return candidates;
285 // Ok - all mandatory checked and we still have candidates.
286 // Let's try filtering using the optional constraints. The optional
287 // constraints must be filtered in the order they occur in |optional|.
288 // But if a constraint produce zero candidates, the constraint is ignored and
289 // the next constraint is tested.
290 // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#idl-def-Constraints
291 for (size_t i = 0; i < optional.size(); ++i) {
292 media::VideoCaptureFormats current_candidates = candidates;
293 FilterFormatsByConstraint(optional[i], false, &current_candidates);
294 if (!current_candidates.empty()) {
295 candidates = current_candidates;
299 // We have done as good as we can to filter the supported resolutions.
300 return candidates;
303 const media::VideoCaptureFormat& GetBestFormatBasedOnArea(
304 const media::VideoCaptureFormats& formats,
305 int area) {
306 media::VideoCaptureFormats::const_iterator it = formats.begin();
307 media::VideoCaptureFormats::const_iterator best_it = formats.begin();
308 int best_diff = std::numeric_limits<int>::max();
309 for (; it != formats.end(); ++it) {
310 int diff = abs(area - it->frame_size.width() * it->frame_size.height());
311 if (diff < best_diff) {
312 best_diff = diff;
313 best_it = it;
316 return *best_it;
319 // Find the format that best matches the default video size.
320 // This algorithm is chosen since a resolution must be picked even if no
321 // constraints are provided. We don't just select the maximum supported
322 // resolution since higher resolutions cost more in terms of complexity and
323 // many cameras have lower frame rate and have more noise in the image at
324 // their maximum supported resolution.
325 void GetBestCaptureFormat(
326 const media::VideoCaptureFormats& formats,
327 const blink::WebMediaConstraints& constraints,
328 media::VideoCaptureFormat* capture_format) {
329 DCHECK(!formats.empty());
331 int max_width;
332 int max_height;
333 GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
335 *capture_format = GetBestFormatBasedOnArea(
336 formats,
337 std::min(max_width, MediaStreamVideoSource::kDefaultWidth) *
338 std::min(max_height, MediaStreamVideoSource::kDefaultHeight));
341 } // anonymous namespace
343 // static
344 MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource(
345 const blink::WebMediaStreamSource& source) {
346 return static_cast<MediaStreamVideoSource*>(source.extraData());
349 // static
350 bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) {
351 for (size_t i = 0; i < arraysize(kSupportedConstraints); ++i) {
352 if (kSupportedConstraints[i] == name)
353 return true;
355 return false;
358 MediaStreamVideoSource::MediaStreamVideoSource()
359 : state_(NEW),
360 muted_state_(false),
361 track_adapter_(new VideoTrackAdapter(
362 ChildProcess::current()->io_message_loop_proxy())),
363 weak_factory_(this) {
366 MediaStreamVideoSource::~MediaStreamVideoSource() {
367 DCHECK(CalledOnValidThread());
370 void MediaStreamVideoSource::AddTrack(
371 MediaStreamVideoTrack* track,
372 const VideoCaptureDeliverFrameCB& frame_callback,
373 const blink::WebMediaConstraints& constraints,
374 const ConstraintsCallback& callback) {
375 DCHECK(CalledOnValidThread());
376 DCHECK(!constraints.isNull());
377 DCHECK(std::find(tracks_.begin(), tracks_.end(),
378 track) == tracks_.end());
379 tracks_.push_back(track);
381 requested_constraints_.push_back(
382 RequestedConstraints(track, frame_callback, constraints, callback));
384 switch (state_) {
385 case NEW: {
386 // Tab capture and Screen capture needs the maximum requested height
387 // and width to decide on the resolution.
388 int max_requested_width = 0;
389 GetMandatoryConstraintValueAsInteger(constraints, kMaxWidth,
390 &max_requested_width);
392 int max_requested_height = 0;
393 GetMandatoryConstraintValueAsInteger(constraints, kMaxHeight,
394 &max_requested_height);
396 double max_requested_frame_rate;
397 if (!GetConstraintValueAsDouble(constraints, kMaxFrameRate,
398 &max_requested_frame_rate)) {
399 max_requested_frame_rate = kDefaultFrameRate;
402 state_ = RETRIEVING_CAPABILITIES;
403 GetCurrentSupportedFormats(
404 max_requested_width,
405 max_requested_height,
406 max_requested_frame_rate,
407 base::Bind(&MediaStreamVideoSource::OnSupportedFormats,
408 weak_factory_.GetWeakPtr()));
410 break;
412 case STARTING:
413 case RETRIEVING_CAPABILITIES: {
414 // The |callback| will be triggered once the source has started or
415 // the capabilities have been retrieved.
416 break;
418 case ENDED:
419 case STARTED: {
420 // Currently, reconfiguring the source is not supported.
421 FinalizeAddTrack();
426 void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
427 DCHECK(CalledOnValidThread());
428 std::vector<MediaStreamVideoTrack*>::iterator it =
429 std::find(tracks_.begin(), tracks_.end(), video_track);
430 DCHECK(it != tracks_.end());
431 tracks_.erase(it);
433 // Check if |video_track| is waiting for applying new constraints and remove
434 // the request in that case.
435 for (std::vector<RequestedConstraints>::iterator it =
436 requested_constraints_.begin();
437 it != requested_constraints_.end(); ++it) {
438 if (it->track == video_track) {
439 requested_constraints_.erase(it);
440 break;
443 // Call |frame_adapter_->RemoveTrack| here even if adding the track has
444 // failed and |frame_adapter_->AddCallback| has not been called.
445 track_adapter_->RemoveTrack(video_track);
447 if (tracks_.empty())
448 StopSource();
451 const scoped_refptr<base::MessageLoopProxy>&
452 MediaStreamVideoSource::io_message_loop() const {
453 DCHECK(CalledOnValidThread());
454 return track_adapter_->io_message_loop();
457 void MediaStreamVideoSource::DoStopSource() {
458 DCHECK(CalledOnValidThread());
459 DVLOG(3) << "DoStopSource()";
460 if (state_ == ENDED)
461 return;
462 StopSourceImpl();
463 state_ = ENDED;
464 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
467 void MediaStreamVideoSource::OnSupportedFormats(
468 const media::VideoCaptureFormats& formats) {
469 DCHECK(CalledOnValidThread());
470 DCHECK_EQ(RETRIEVING_CAPABILITIES, state_);
472 supported_formats_ = formats;
473 if (!FindBestFormatWithConstraints(supported_formats_,
474 &current_format_)) {
475 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
476 // This object can be deleted after calling FinalizeAddTrack. See comment
477 // in the header file.
478 FinalizeAddTrack();
479 return;
482 state_ = STARTING;
483 DVLOG(3) << "Starting the capturer with"
484 << " width = " << current_format_.frame_size.width()
485 << " height = " << current_format_.frame_size.height()
486 << " frame rate = " << current_format_.frame_rate
487 << " pixel format = "
488 << media::VideoCaptureFormat::PixelFormatToString(
489 current_format_.pixel_format);
491 media::VideoCaptureParams params;
492 params.requested_format = current_format_;
493 StartSourceImpl(
494 params,
495 base::Bind(&VideoTrackAdapter::DeliverFrameOnIO, track_adapter_));
498 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
499 const media::VideoCaptureFormats& formats,
500 media::VideoCaptureFormat* best_format) {
501 DCHECK(CalledOnValidThread());
502 // Find the first constraints that we can fulfill.
503 for (std::vector<RequestedConstraints>::iterator request_it =
504 requested_constraints_.begin();
505 request_it != requested_constraints_.end(); ++request_it) {
506 const blink::WebMediaConstraints& requested_constraints =
507 request_it->constraints;
509 // If the source doesn't support capability enumeration it is still ok if
510 // no mandatory constraints have been specified. That just means that
511 // we will start with whatever format is native to the source.
512 if (formats.empty() && !HasMandatoryConstraints(requested_constraints)) {
513 *best_format = media::VideoCaptureFormat();
514 return true;
516 blink::WebString unsatisfied_constraint;
517 media::VideoCaptureFormats filtered_formats =
518 FilterFormats(requested_constraints, formats, &unsatisfied_constraint);
519 if (filtered_formats.size() > 0) {
520 // A request with constraints that can be fulfilled.
521 GetBestCaptureFormat(filtered_formats,
522 requested_constraints,
523 best_format);
524 return true;
527 return false;
530 void MediaStreamVideoSource::OnStartDone(MediaStreamRequestResult result) {
531 DCHECK(CalledOnValidThread());
532 DVLOG(3) << "OnStartDone({result =" << result << "})";
533 if (result == MEDIA_DEVICE_OK) {
534 DCHECK_EQ(STARTING, state_);
535 state_ = STARTED;
536 SetReadyState(blink::WebMediaStreamSource::ReadyStateLive);
537 } else {
538 StopSource();
541 // This object can be deleted after calling FinalizeAddTrack. See comment in
542 // the header file.
543 FinalizeAddTrack();
546 void MediaStreamVideoSource::FinalizeAddTrack() {
547 DCHECK(CalledOnValidThread());
548 media::VideoCaptureFormats formats;
549 formats.push_back(current_format_);
551 std::vector<RequestedConstraints> callbacks;
552 callbacks.swap(requested_constraints_);
553 for (std::vector<RequestedConstraints>::iterator it = callbacks.begin();
554 it != callbacks.end(); ++it) {
555 MediaStreamRequestResult result = MEDIA_DEVICE_OK;
556 blink::WebString unsatisfied_constraint;
558 if (HasMandatoryConstraints(it->constraints) &&
559 FilterFormats(it->constraints, formats,
560 &unsatisfied_constraint).empty())
561 result = MEDIA_DEVICE_CONSTRAINT_NOT_SATISFIED;
563 if (state_ != STARTED && result == MEDIA_DEVICE_OK)
564 result = MEDIA_DEVICE_TRACK_START_FAILURE;
566 if (result == MEDIA_DEVICE_OK) {
567 int max_width;
568 int max_height;
569 GetDesiredMaxWidthAndHeight(it->constraints, &max_width, &max_height);
570 double max_aspect_ratio;
571 double min_aspect_ratio;
572 GetDesiredMinAndMaxAspectRatio(it->constraints,
573 &min_aspect_ratio,
574 &max_aspect_ratio);
575 double max_frame_rate = 0.0f;
576 GetConstraintValueAsDouble(it->constraints,
577 kMaxFrameRate, &max_frame_rate);
579 VideoTrackAdapter::OnMutedCallback on_mute_callback =
580 media::BindToCurrentLoop(base::Bind(
581 &MediaStreamVideoSource::SetMutedState,
582 weak_factory_.GetWeakPtr()));
583 track_adapter_->AddTrack(it->track, it->frame_callback,
584 max_width, max_height,
585 min_aspect_ratio, max_aspect_ratio,
586 max_frame_rate, current_format_.frame_rate,
587 on_mute_callback);
590 DVLOG(3) << "FinalizeAddTrack() result " << result;
592 if (!it->callback.is_null()) {
593 it->callback.Run(this, result, unsatisfied_constraint);
598 void MediaStreamVideoSource::SetReadyState(
599 blink::WebMediaStreamSource::ReadyState state) {
600 DVLOG(3) << "MediaStreamVideoSource::SetReadyState state " << state;
601 DCHECK(CalledOnValidThread());
602 if (!owner().isNull())
603 owner().setReadyState(state);
604 for (std::vector<MediaStreamVideoTrack*>::iterator it = tracks_.begin();
605 it != tracks_.end(); ++it) {
606 (*it)->OnReadyStateChanged(state);
610 void MediaStreamVideoSource::SetMutedState(bool muted_state) {
611 DVLOG(3) << "MediaStreamVideoSource::SetMutedState state=" << muted_state;
612 DCHECK(CalledOnValidThread());
613 if (muted_state != muted_state_) {
614 muted_state_ = muted_state;
615 if (!owner().isNull()) {
616 owner().setReadyState(muted_state_
617 ? blink::WebMediaStreamSource::ReadyStateMuted
618 : blink::WebMediaStreamSource::ReadyStateLive);
621 // WebMediaStreamSource doesn't have a muted state, the tracks do.
622 for (std::vector<MediaStreamVideoTrack*>::iterator it = tracks_.begin();
623 it != tracks_.end(); ++it) {
624 (*it)->SetMutedState(muted_state);
628 MediaStreamVideoSource::RequestedConstraints::RequestedConstraints(
629 MediaStreamVideoTrack* track,
630 const VideoCaptureDeliverFrameCB& frame_callback,
631 const blink::WebMediaConstraints& constraints,
632 const ConstraintsCallback& callback)
633 : track(track),
634 frame_callback(frame_callback),
635 constraints(constraints),
636 callback(callback) {
639 MediaStreamVideoSource::RequestedConstraints::~RequestedConstraints() {
642 } // namespace content