Re-subimission of https://codereview.chromium.org/1041213003/
[chromium-blink-merge.git] / content / renderer / media / media_stream_video_source.cc
blob6e501215844803aacdacfdd5b53871dc331cc764
1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/media_stream_video_source.h"
7 #include <algorithm>
8 #include <limits>
9 #include <string>
11 #include "base/logging.h"
12 #include "base/strings/string_number_conversions.h"
13 #include "base/trace_event/trace_event.h"
14 #include "content/child/child_process.h"
15 #include "content/renderer/media/media_stream_constraints_util.h"
16 #include "content/renderer/media/media_stream_video_track.h"
17 #include "content/renderer/media/video_track_adapter.h"
19 namespace content {
21 // Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b
22 const char MediaStreamVideoSource::kMinAspectRatio[] = "minAspectRatio";
23 const char MediaStreamVideoSource::kMaxAspectRatio[] = "maxAspectRatio";
24 const char MediaStreamVideoSource::kMaxWidth[] = "maxWidth";
25 const char MediaStreamVideoSource::kMinWidth[] = "minWidth";
26 const char MediaStreamVideoSource::kMaxHeight[] = "maxHeight";
27 const char MediaStreamVideoSource::kMinHeight[] = "minHeight";
28 const char MediaStreamVideoSource::kMaxFrameRate[] = "maxFrameRate";
29 const char MediaStreamVideoSource::kMinFrameRate[] = "minFrameRate";
31 const char* kSupportedConstraints[] = {
32 MediaStreamVideoSource::kMaxAspectRatio,
33 MediaStreamVideoSource::kMinAspectRatio,
34 MediaStreamVideoSource::kMaxWidth,
35 MediaStreamVideoSource::kMinWidth,
36 MediaStreamVideoSource::kMaxHeight,
37 MediaStreamVideoSource::kMinHeight,
38 MediaStreamVideoSource::kMaxFrameRate,
39 MediaStreamVideoSource::kMinFrameRate,
42 namespace {
44 // Google-specific key prefix. Constraints with this prefix are ignored if they
45 // are unknown.
46 const char kGooglePrefix[] = "goog";
48 // Returns true if |constraint| has mandatory constraints.
49 bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
50 blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
51 constraints.getMandatoryConstraints(mandatory_constraints);
52 return !mandatory_constraints.isEmpty();
55 // Retrieve the desired max width and height from |constraints|. If not set,
56 // the |desired_width| and |desired_height| are set to
57 // std::numeric_limits<int>::max();
58 // If either max width or height is set as a mandatory constraint, the optional
59 // constraints are not checked.
60 void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
61 int* desired_width, int* desired_height) {
62 *desired_width = std::numeric_limits<int>::max();
63 *desired_height = std::numeric_limits<int>::max();
65 bool mandatory = GetMandatoryConstraintValueAsInteger(
66 constraints,
67 MediaStreamVideoSource::kMaxWidth,
68 desired_width);
69 mandatory |= GetMandatoryConstraintValueAsInteger(
70 constraints,
71 MediaStreamVideoSource::kMaxHeight,
72 desired_height);
73 if (mandatory)
74 return;
76 GetOptionalConstraintValueAsInteger(constraints,
77 MediaStreamVideoSource::kMaxWidth,
78 desired_width);
79 GetOptionalConstraintValueAsInteger(constraints,
80 MediaStreamVideoSource::kMaxHeight,
81 desired_height);
84 // Retrieve the desired max and min aspect ratio from |constraints|. If not set,
85 // the |min_aspect_ratio| is set to 0 and |max_aspect_ratio| is set to
86 // std::numeric_limits<double>::max();
87 // If either min or max aspect ratio is set as a mandatory constraint, the
88 // optional constraints are not checked.
89 void GetDesiredMinAndMaxAspectRatio(
90 const blink::WebMediaConstraints& constraints,
91 double* min_aspect_ratio,
92 double* max_aspect_ratio) {
93 *min_aspect_ratio = 0;
94 *max_aspect_ratio = std::numeric_limits<double>::max();
96 bool mandatory = GetMandatoryConstraintValueAsDouble(
97 constraints,
98 MediaStreamVideoSource::kMinAspectRatio,
99 min_aspect_ratio);
100 mandatory |= GetMandatoryConstraintValueAsDouble(
101 constraints,
102 MediaStreamVideoSource::kMaxAspectRatio,
103 max_aspect_ratio);
104 if (mandatory)
105 return;
107 GetOptionalConstraintValueAsDouble(
108 constraints,
109 MediaStreamVideoSource::kMinAspectRatio,
110 min_aspect_ratio);
111 GetOptionalConstraintValueAsDouble(
112 constraints,
113 MediaStreamVideoSource::kMaxAspectRatio,
114 max_aspect_ratio);
117 // Returns true if |constraint| is fulfilled. |format| can be changed by a
118 // constraint, e.g. the frame rate can be changed by setting maxFrameRate.
119 bool UpdateFormatForConstraint(
120 const blink::WebMediaConstraint& constraint,
121 bool mandatory,
122 media::VideoCaptureFormat* format) {
123 DCHECK(format != NULL);
125 if (!format->IsValid())
126 return false;
128 std::string constraint_name = constraint.m_name.utf8();
129 std::string constraint_value = constraint.m_value.utf8();
131 if (constraint_name.find(kGooglePrefix) == 0) {
132 // These are actually options, not constraints, so they can be satisfied
133 // regardless of the format.
134 return true;
137 if (constraint_name == MediaStreamSource::kSourceId) {
138 // This is a constraint that doesn't affect the format.
139 return true;
142 // Ignore Chrome specific Tab capture constraints.
143 if (constraint_name == kMediaStreamSource ||
144 constraint_name == kMediaStreamSourceId)
145 return true;
147 if (constraint_name == MediaStreamVideoSource::kMinAspectRatio ||
148 constraint_name == MediaStreamVideoSource::kMaxAspectRatio) {
149 // These constraints are handled by cropping if the camera outputs the wrong
150 // aspect ratio.
151 double value;
152 return base::StringToDouble(constraint_value, &value);
155 double value = 0.0;
156 if (!base::StringToDouble(constraint_value, &value)) {
157 DLOG(WARNING) << "Can't parse MediaStream constraint. Name:"
158 << constraint_name << " Value:" << constraint_value;
159 return false;
162 if (constraint_name == MediaStreamVideoSource::kMinWidth) {
163 return (value <= format->frame_size.width());
164 } else if (constraint_name == MediaStreamVideoSource::kMaxWidth) {
165 return value > 0.0;
166 } else if (constraint_name == MediaStreamVideoSource::kMinHeight) {
167 return (value <= format->frame_size.height());
168 } else if (constraint_name == MediaStreamVideoSource::kMaxHeight) {
169 return value > 0.0;
170 } else if (constraint_name == MediaStreamVideoSource::kMinFrameRate) {
171 return (value > 0.0) && (value <= format->frame_rate);
172 } else if (constraint_name == MediaStreamVideoSource::kMaxFrameRate) {
173 if (value <= 0.0) {
174 // The frame rate is set by constraint.
175 // Don't allow 0 as frame rate if it is a mandatory constraint.
176 // Set the frame rate to 1 if it is not mandatory.
177 if (mandatory) {
178 return false;
179 } else {
180 value = 1.0;
183 format->frame_rate =
184 (format->frame_rate > value) ? value : format->frame_rate;
185 return true;
186 } else {
187 LOG(WARNING) << "Found unknown MediaStream constraint. Name:"
188 << constraint_name << " Value:" << constraint_value;
189 return false;
193 // Removes media::VideoCaptureFormats from |formats| that don't meet
194 // |constraint|.
195 void FilterFormatsByConstraint(
196 const blink::WebMediaConstraint& constraint,
197 bool mandatory,
198 media::VideoCaptureFormats* formats) {
199 DVLOG(3) << "FilterFormatsByConstraint("
200 << "{ constraint.m_name = " << constraint.m_name.utf8()
201 << " constraint.m_value = " << constraint.m_value.utf8()
202 << " mandatory = " << mandatory << "})";
203 media::VideoCaptureFormats::iterator format_it = formats->begin();
204 while (format_it != formats->end()) {
205 // Modify the format_it to fulfill the constraint if possible.
206 // Delete it otherwise.
207 if (!UpdateFormatForConstraint(constraint, mandatory, &(*format_it)))
208 format_it = formats->erase(format_it);
209 else
210 ++format_it;
214 // Returns the media::VideoCaptureFormats that matches |constraints|.
215 media::VideoCaptureFormats FilterFormats(
216 const blink::WebMediaConstraints& constraints,
217 const media::VideoCaptureFormats& supported_formats,
218 blink::WebString* unsatisfied_constraint) {
219 if (constraints.isNull())
220 return supported_formats;
222 double max_aspect_ratio;
223 double min_aspect_ratio;
224 GetDesiredMinAndMaxAspectRatio(constraints,
225 &min_aspect_ratio,
226 &max_aspect_ratio);
228 if (min_aspect_ratio > max_aspect_ratio || max_aspect_ratio < 0.05f) {
229 DLOG(WARNING) << "Wrong requested aspect ratio.";
230 return media::VideoCaptureFormats();
233 int min_width = 0;
234 GetMandatoryConstraintValueAsInteger(constraints,
235 MediaStreamVideoSource::kMinWidth,
236 &min_width);
237 int min_height = 0;
238 GetMandatoryConstraintValueAsInteger(constraints,
239 MediaStreamVideoSource::kMinHeight,
240 &min_height);
241 int max_width;
242 int max_height;
243 GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
245 if (min_width > max_width || min_height > max_height)
246 return media::VideoCaptureFormats();
248 double min_frame_rate = 0.0f;
249 double max_frame_rate = 0.0f;
250 if (GetConstraintValueAsDouble(constraints,
251 MediaStreamVideoSource::kMaxFrameRate,
252 &max_frame_rate) &&
253 GetConstraintValueAsDouble(constraints,
254 MediaStreamVideoSource::kMinFrameRate,
255 &min_frame_rate)) {
256 if (min_frame_rate > max_frame_rate) {
257 DLOG(WARNING) << "Wrong requested frame rate.";
258 return media::VideoCaptureFormats();
262 blink::WebVector<blink::WebMediaConstraint> mandatory;
263 blink::WebVector<blink::WebMediaConstraint> optional;
264 constraints.getMandatoryConstraints(mandatory);
265 constraints.getOptionalConstraints(optional);
266 media::VideoCaptureFormats candidates = supported_formats;
267 for (size_t i = 0; i < mandatory.size(); ++i) {
268 FilterFormatsByConstraint(mandatory[i], true, &candidates);
269 if (candidates.empty()) {
270 *unsatisfied_constraint = mandatory[i].m_name;
271 return candidates;
275 if (candidates.empty())
276 return candidates;
278 // Ok - all mandatory checked and we still have candidates.
279 // Let's try filtering using the optional constraints. The optional
280 // constraints must be filtered in the order they occur in |optional|.
281 // But if a constraint produce zero candidates, the constraint is ignored and
282 // the next constraint is tested.
283 // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#idl-def-Constraints
284 for (size_t i = 0; i < optional.size(); ++i) {
285 media::VideoCaptureFormats current_candidates = candidates;
286 FilterFormatsByConstraint(optional[i], false, &current_candidates);
287 if (!current_candidates.empty())
288 candidates = current_candidates;
291 // We have done as good as we can to filter the supported resolutions.
292 return candidates;
295 const media::VideoCaptureFormat& GetBestFormatBasedOnArea(
296 const media::VideoCaptureFormats& formats,
297 int area) {
298 media::VideoCaptureFormats::const_iterator it = formats.begin();
299 media::VideoCaptureFormats::const_iterator best_it = formats.begin();
300 int best_diff = std::numeric_limits<int>::max();
301 for (; it != formats.end(); ++it) {
302 const int diff = abs(area - it->frame_size.GetArea());
303 if (diff < best_diff) {
304 best_diff = diff;
305 best_it = it;
308 return *best_it;
311 // Find the format that best matches the default video size.
312 // This algorithm is chosen since a resolution must be picked even if no
313 // constraints are provided. We don't just select the maximum supported
314 // resolution since higher resolutions cost more in terms of complexity and
315 // many cameras have lower frame rate and have more noise in the image at
316 // their maximum supported resolution.
317 void GetBestCaptureFormat(
318 const media::VideoCaptureFormats& formats,
319 const blink::WebMediaConstraints& constraints,
320 media::VideoCaptureFormat* capture_format) {
321 DCHECK(!formats.empty());
323 int max_width;
324 int max_height;
325 GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
327 *capture_format = GetBestFormatBasedOnArea(
328 formats,
329 std::min(max_width,
330 static_cast<int>(MediaStreamVideoSource::kDefaultWidth)) *
331 std::min(max_height,
332 static_cast<int>(MediaStreamVideoSource::kDefaultHeight)));
335 } // anonymous namespace
337 // static
338 MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource(
339 const blink::WebMediaStreamSource& source) {
340 return static_cast<MediaStreamVideoSource*>(source.extraData());
343 // static
344 bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) {
345 for (const char* constraint : kSupportedConstraints) {
346 if (constraint == name)
347 return true;
349 return false;
352 MediaStreamVideoSource::MediaStreamVideoSource()
353 : state_(NEW),
354 track_adapter_(new VideoTrackAdapter(
355 ChildProcess::current()->io_message_loop_proxy())),
356 weak_factory_(this) {
359 MediaStreamVideoSource::~MediaStreamVideoSource() {
360 DCHECK(CalledOnValidThread());
363 void MediaStreamVideoSource::AddTrack(
364 MediaStreamVideoTrack* track,
365 const VideoCaptureDeliverFrameCB& frame_callback,
366 const blink::WebMediaConstraints& constraints,
367 const ConstraintsCallback& callback) {
368 DCHECK(CalledOnValidThread());
369 DCHECK(!constraints.isNull());
370 DCHECK(std::find(tracks_.begin(), tracks_.end(),
371 track) == tracks_.end());
372 tracks_.push_back(track);
374 requested_constraints_.push_back(
375 RequestedConstraints(track, frame_callback, constraints, callback));
377 switch (state_) {
378 case NEW: {
379 // Tab capture and Screen capture needs the maximum requested height
380 // and width to decide on the resolution.
381 int max_requested_width = 0;
382 GetMandatoryConstraintValueAsInteger(constraints, kMaxWidth,
383 &max_requested_width);
385 int max_requested_height = 0;
386 GetMandatoryConstraintValueAsInteger(constraints, kMaxHeight,
387 &max_requested_height);
389 double max_requested_frame_rate;
390 if (!GetConstraintValueAsDouble(constraints, kMaxFrameRate,
391 &max_requested_frame_rate)) {
392 max_requested_frame_rate = kDefaultFrameRate;
395 state_ = RETRIEVING_CAPABILITIES;
396 GetCurrentSupportedFormats(
397 max_requested_width,
398 max_requested_height,
399 max_requested_frame_rate,
400 base::Bind(&MediaStreamVideoSource::OnSupportedFormats,
401 weak_factory_.GetWeakPtr()));
403 break;
405 case STARTING:
406 case RETRIEVING_CAPABILITIES: {
407 // The |callback| will be triggered once the source has started or
408 // the capabilities have been retrieved.
409 break;
411 case ENDED:
412 case STARTED: {
413 // Currently, reconfiguring the source is not supported.
414 FinalizeAddTrack();
419 void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
420 DCHECK(CalledOnValidThread());
421 std::vector<MediaStreamVideoTrack*>::iterator it =
422 std::find(tracks_.begin(), tracks_.end(), video_track);
423 DCHECK(it != tracks_.end());
424 tracks_.erase(it);
426 // Check if |video_track| is waiting for applying new constraints and remove
427 // the request in that case.
428 for (std::vector<RequestedConstraints>::iterator it =
429 requested_constraints_.begin();
430 it != requested_constraints_.end(); ++it) {
431 if (it->track == video_track) {
432 requested_constraints_.erase(it);
433 break;
436 // Call |frame_adapter_->RemoveTrack| here even if adding the track has
437 // failed and |frame_adapter_->AddCallback| has not been called.
438 track_adapter_->RemoveTrack(video_track);
440 if (tracks_.empty())
441 StopSource();
444 const scoped_refptr<base::MessageLoopProxy>&
445 MediaStreamVideoSource::io_message_loop() const {
446 DCHECK(CalledOnValidThread());
447 return track_adapter_->io_message_loop();
450 void MediaStreamVideoSource::DoStopSource() {
451 DCHECK(CalledOnValidThread());
452 DVLOG(3) << "DoStopSource()";
453 if (state_ == ENDED)
454 return;
455 track_adapter_->StopFrameMonitoring();
456 StopSourceImpl();
457 state_ = ENDED;
458 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
461 void MediaStreamVideoSource::OnSupportedFormats(
462 const media::VideoCaptureFormats& formats) {
463 DCHECK(CalledOnValidThread());
464 DCHECK_EQ(RETRIEVING_CAPABILITIES, state_);
466 supported_formats_ = formats;
467 if (!FindBestFormatWithConstraints(supported_formats_,
468 &current_format_)) {
469 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
470 // This object can be deleted after calling FinalizeAddTrack. See comment
471 // in the header file.
472 FinalizeAddTrack();
473 return;
476 state_ = STARTING;
477 DVLOG(3) << "Starting the capturer with " << current_format_.ToString();
479 StartSourceImpl(
480 current_format_,
481 base::Bind(&VideoTrackAdapter::DeliverFrameOnIO, track_adapter_));
484 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
485 const media::VideoCaptureFormats& formats,
486 media::VideoCaptureFormat* best_format) {
487 DCHECK(CalledOnValidThread());
488 // Find the first constraints that we can fulfill.
489 for (const auto& request : requested_constraints_) {
490 const blink::WebMediaConstraints& requested_constraints =
491 request.constraints;
493 // If the source doesn't support capability enumeration it is still ok if
494 // no mandatory constraints have been specified. That just means that
495 // we will start with whatever format is native to the source.
496 if (formats.empty() && !HasMandatoryConstraints(requested_constraints)) {
497 *best_format = media::VideoCaptureFormat();
498 return true;
500 blink::WebString unsatisfied_constraint;
501 media::VideoCaptureFormats filtered_formats =
502 FilterFormats(requested_constraints, formats, &unsatisfied_constraint);
503 if (filtered_formats.size() > 0) {
504 // A request with constraints that can be fulfilled.
505 GetBestCaptureFormat(filtered_formats,
506 requested_constraints,
507 best_format);
508 return true;
511 return false;
514 void MediaStreamVideoSource::OnStartDone(MediaStreamRequestResult result) {
515 DCHECK(CalledOnValidThread());
516 DVLOG(3) << "OnStartDone({result =" << result << "})";
517 if (result == MEDIA_DEVICE_OK) {
518 DCHECK_EQ(STARTING, state_);
519 state_ = STARTED;
520 SetReadyState(blink::WebMediaStreamSource::ReadyStateLive);
522 track_adapter_->StartFrameMonitoring(
523 current_format_.frame_rate,
524 base::Bind(&MediaStreamVideoSource::SetMutedState,
525 weak_factory_.GetWeakPtr()));
527 } else {
528 StopSource();
531 // This object can be deleted after calling FinalizeAddTrack. See comment in
532 // the header file.
533 FinalizeAddTrack();
536 void MediaStreamVideoSource::FinalizeAddTrack() {
537 DCHECK(CalledOnValidThread());
538 media::VideoCaptureFormats formats;
539 formats.push_back(current_format_);
541 std::vector<RequestedConstraints> callbacks;
542 callbacks.swap(requested_constraints_);
543 for (const auto& request : callbacks) {
544 MediaStreamRequestResult result = MEDIA_DEVICE_OK;
545 blink::WebString unsatisfied_constraint;
547 if (HasMandatoryConstraints(request.constraints) &&
548 FilterFormats(request.constraints, formats,
549 &unsatisfied_constraint).empty()) {
550 result = MEDIA_DEVICE_CONSTRAINT_NOT_SATISFIED;
553 if (state_ != STARTED && result == MEDIA_DEVICE_OK)
554 result = MEDIA_DEVICE_TRACK_START_FAILURE;
556 if (result == MEDIA_DEVICE_OK) {
557 int max_width;
558 int max_height;
559 GetDesiredMaxWidthAndHeight(request.constraints, &max_width, &max_height);
560 double max_aspect_ratio;
561 double min_aspect_ratio;
562 GetDesiredMinAndMaxAspectRatio(request.constraints,
563 &min_aspect_ratio,
564 &max_aspect_ratio);
565 double max_frame_rate = 0.0f;
566 GetConstraintValueAsDouble(request.constraints,
567 kMaxFrameRate, &max_frame_rate);
569 track_adapter_->AddTrack(request.track, request.frame_callback,
570 max_width, max_height,
571 min_aspect_ratio, max_aspect_ratio,
572 max_frame_rate);
575 DVLOG(3) << "FinalizeAddTrack() result " << result;
577 if (!request.callback.is_null()) {
578 request.callback.Run(this, result, unsatisfied_constraint);
583 void MediaStreamVideoSource::SetReadyState(
584 blink::WebMediaStreamSource::ReadyState state) {
585 DVLOG(3) << "MediaStreamVideoSource::SetReadyState state " << state;
586 DCHECK(CalledOnValidThread());
587 if (!owner().isNull())
588 owner().setReadyState(state);
589 for (const auto& track : tracks_)
590 track->OnReadyStateChanged(state);
593 void MediaStreamVideoSource::SetMutedState(bool muted_state) {
594 DVLOG(3) << "MediaStreamVideoSource::SetMutedState state=" << muted_state;
595 DCHECK(CalledOnValidThread());
596 if (!owner().isNull()) {
597 owner().setReadyState(muted_state
598 ? blink::WebMediaStreamSource::ReadyStateMuted
599 : blink::WebMediaStreamSource::ReadyStateLive);
603 MediaStreamVideoSource::RequestedConstraints::RequestedConstraints(
604 MediaStreamVideoTrack* track,
605 const VideoCaptureDeliverFrameCB& frame_callback,
606 const blink::WebMediaConstraints& constraints,
607 const ConstraintsCallback& callback)
608 : track(track),
609 frame_callback(frame_callback),
610 constraints(constraints),
611 callback(callback) {
614 MediaStreamVideoSource::RequestedConstraints::~RequestedConstraints() {
617 } // namespace content