Upstreaming browser/ui/uikit_ui_util from iOS.
[chromium-blink-merge.git] / content / renderer / media / media_stream_video_source.cc
blob8493fe9856c43ea3c6ccc31fadbe0e588c7c826a
1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/media_stream_video_source.h"
7 #include <algorithm>
8 #include <limits>
9 #include <string>
11 #include "base/logging.h"
12 #include "base/strings/string_number_conversions.h"
13 #include "base/trace_event/trace_event.h"
14 #include "content/child/child_process.h"
15 #include "content/renderer/media/media_stream_constraints_util.h"
16 #include "content/renderer/media/media_stream_video_track.h"
17 #include "content/renderer/media/video_track_adapter.h"
19 namespace content {
21 // Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b
22 const char MediaStreamVideoSource::kMinAspectRatio[] = "minAspectRatio";
23 const char MediaStreamVideoSource::kMaxAspectRatio[] = "maxAspectRatio";
24 const char MediaStreamVideoSource::kMaxWidth[] = "maxWidth";
25 const char MediaStreamVideoSource::kMinWidth[] = "minWidth";
26 const char MediaStreamVideoSource::kMaxHeight[] = "maxHeight";
27 const char MediaStreamVideoSource::kMinHeight[] = "minHeight";
28 const char MediaStreamVideoSource::kMaxFrameRate[] = "maxFrameRate";
29 const char MediaStreamVideoSource::kMinFrameRate[] = "minFrameRate";
31 const char* kSupportedConstraints[] = {
32 MediaStreamVideoSource::kMaxAspectRatio,
33 MediaStreamVideoSource::kMinAspectRatio,
34 MediaStreamVideoSource::kMaxWidth,
35 MediaStreamVideoSource::kMinWidth,
36 MediaStreamVideoSource::kMaxHeight,
37 MediaStreamVideoSource::kMinHeight,
38 MediaStreamVideoSource::kMaxFrameRate,
39 MediaStreamVideoSource::kMinFrameRate,
42 namespace {
44 // Google-specific key prefix. Constraints with this prefix are ignored if they
45 // are unknown.
46 const char kGooglePrefix[] = "goog";
48 // Returns true if |constraint| has mandatory constraints.
49 bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
50 blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
51 constraints.getMandatoryConstraints(mandatory_constraints);
52 return !mandatory_constraints.isEmpty();
55 // Retrieve the desired max width and height from |constraints|. If not set,
56 // the |desired_width| and |desired_height| are set to
57 // std::numeric_limits<int>::max();
58 // If either max width or height is set as a mandatory constraint, the optional
59 // constraints are not checked.
60 void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
61 int* desired_width, int* desired_height) {
62 *desired_width = std::numeric_limits<int>::max();
63 *desired_height = std::numeric_limits<int>::max();
65 bool mandatory = GetMandatoryConstraintValueAsInteger(
66 constraints,
67 MediaStreamVideoSource::kMaxWidth,
68 desired_width);
69 mandatory |= GetMandatoryConstraintValueAsInteger(
70 constraints,
71 MediaStreamVideoSource::kMaxHeight,
72 desired_height);
73 if (mandatory)
74 return;
76 GetOptionalConstraintValueAsInteger(constraints,
77 MediaStreamVideoSource::kMaxWidth,
78 desired_width);
79 GetOptionalConstraintValueAsInteger(constraints,
80 MediaStreamVideoSource::kMaxHeight,
81 desired_height);
84 // Retrieve the desired max and min aspect ratio from |constraints|. If not set,
85 // the |min_aspect_ratio| is set to 0 and |max_aspect_ratio| is set to
86 // std::numeric_limits<double>::max();
87 // If either min or max aspect ratio is set as a mandatory constraint, the
88 // optional constraints are not checked.
89 void GetDesiredMinAndMaxAspectRatio(
90 const blink::WebMediaConstraints& constraints,
91 double* min_aspect_ratio,
92 double* max_aspect_ratio) {
93 *min_aspect_ratio = 0;
94 *max_aspect_ratio = std::numeric_limits<double>::max();
96 bool mandatory = GetMandatoryConstraintValueAsDouble(
97 constraints,
98 MediaStreamVideoSource::kMinAspectRatio,
99 min_aspect_ratio);
100 mandatory |= GetMandatoryConstraintValueAsDouble(
101 constraints,
102 MediaStreamVideoSource::kMaxAspectRatio,
103 max_aspect_ratio);
104 if (mandatory)
105 return;
107 GetOptionalConstraintValueAsDouble(
108 constraints,
109 MediaStreamVideoSource::kMinAspectRatio,
110 min_aspect_ratio);
111 GetOptionalConstraintValueAsDouble(
112 constraints,
113 MediaStreamVideoSource::kMaxAspectRatio,
114 max_aspect_ratio);
117 // Returns true if |constraint| is fulfilled. |format| can be changed by a
118 // constraint, e.g. the frame rate can be changed by setting maxFrameRate.
119 bool UpdateFormatForConstraint(
120 const blink::WebMediaConstraint& constraint,
121 bool mandatory,
122 media::VideoCaptureFormat* format) {
123 DCHECK(format != NULL);
125 if (!format->IsValid())
126 return false;
128 std::string constraint_name = constraint.m_name.utf8();
129 std::string constraint_value = constraint.m_value.utf8();
131 if (constraint_name.find(kGooglePrefix) == 0) {
132 // These are actually options, not constraints, so they can be satisfied
133 // regardless of the format.
134 return true;
137 if (constraint_name == MediaStreamSource::kSourceId) {
138 // This is a constraint that doesn't affect the format.
139 return true;
142 // Ignore Chrome specific Tab capture constraints.
143 if (constraint_name == kMediaStreamSource ||
144 constraint_name == kMediaStreamSourceId)
145 return true;
147 if (constraint_name == MediaStreamVideoSource::kMinAspectRatio ||
148 constraint_name == MediaStreamVideoSource::kMaxAspectRatio) {
149 // These constraints are handled by cropping if the camera outputs the wrong
150 // aspect ratio.
151 double value;
152 return base::StringToDouble(constraint_value, &value);
155 double value = 0.0;
156 if (!base::StringToDouble(constraint_value, &value)) {
157 DLOG(WARNING) << "Can't parse MediaStream constraint. Name:"
158 << constraint_name << " Value:" << constraint_value;
159 return false;
162 if (constraint_name == MediaStreamVideoSource::kMinWidth) {
163 return (value <= format->frame_size.width());
164 } else if (constraint_name == MediaStreamVideoSource::kMaxWidth) {
165 return value > 0.0;
166 } else if (constraint_name == MediaStreamVideoSource::kMinHeight) {
167 return (value <= format->frame_size.height());
168 } else if (constraint_name == MediaStreamVideoSource::kMaxHeight) {
169 return value > 0.0;
170 } else if (constraint_name == MediaStreamVideoSource::kMinFrameRate) {
171 return (value > 0.0) && (value <= format->frame_rate);
172 } else if (constraint_name == MediaStreamVideoSource::kMaxFrameRate) {
173 if (value <= 0.0) {
174 // The frame rate is set by constraint.
175 // Don't allow 0 as frame rate if it is a mandatory constraint.
176 // Set the frame rate to 1 if it is not mandatory.
177 if (mandatory) {
178 return false;
179 } else {
180 value = 1.0;
183 format->frame_rate =
184 (format->frame_rate > value) ? value : format->frame_rate;
185 return true;
186 } else {
187 LOG(WARNING) << "Found unknown MediaStream constraint. Name:"
188 << constraint_name << " Value:" << constraint_value;
189 return false;
193 // Removes media::VideoCaptureFormats from |formats| that don't meet
194 // |constraint|.
195 void FilterFormatsByConstraint(
196 const blink::WebMediaConstraint& constraint,
197 bool mandatory,
198 media::VideoCaptureFormats* formats) {
199 DVLOG(3) << "FilterFormatsByConstraint("
200 << "{ constraint.m_name = " << constraint.m_name.utf8()
201 << " constraint.m_value = " << constraint.m_value.utf8()
202 << " mandatory = " << mandatory << "})";
203 media::VideoCaptureFormats::iterator format_it = formats->begin();
204 while (format_it != formats->end()) {
205 // Modify the format_it to fulfill the constraint if possible.
206 // Delete it otherwise.
207 if (!UpdateFormatForConstraint(constraint, mandatory, &(*format_it)))
208 format_it = formats->erase(format_it);
209 else
210 ++format_it;
214 // Returns the media::VideoCaptureFormats that matches |constraints|.
215 media::VideoCaptureFormats FilterFormats(
216 const blink::WebMediaConstraints& constraints,
217 const media::VideoCaptureFormats& supported_formats,
218 blink::WebString* unsatisfied_constraint) {
219 if (constraints.isNull())
220 return supported_formats;
222 double max_aspect_ratio;
223 double min_aspect_ratio;
224 GetDesiredMinAndMaxAspectRatio(constraints,
225 &min_aspect_ratio,
226 &max_aspect_ratio);
228 if (min_aspect_ratio > max_aspect_ratio || max_aspect_ratio < 0.05f) {
229 DLOG(WARNING) << "Wrong requested aspect ratio.";
230 return media::VideoCaptureFormats();
233 int min_width = 0;
234 GetMandatoryConstraintValueAsInteger(constraints,
235 MediaStreamVideoSource::kMinWidth,
236 &min_width);
237 int min_height = 0;
238 GetMandatoryConstraintValueAsInteger(constraints,
239 MediaStreamVideoSource::kMinHeight,
240 &min_height);
241 int max_width;
242 int max_height;
243 GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
245 if (min_width > max_width || min_height > max_height)
246 return media::VideoCaptureFormats();
248 double min_frame_rate = 0.0f;
249 double max_frame_rate = 0.0f;
250 if (GetConstraintValueAsDouble(constraints,
251 MediaStreamVideoSource::kMaxFrameRate,
252 &max_frame_rate) &&
253 GetConstraintValueAsDouble(constraints,
254 MediaStreamVideoSource::kMinFrameRate,
255 &min_frame_rate)) {
256 if (min_frame_rate > max_frame_rate) {
257 DLOG(WARNING) << "Wrong requested frame rate.";
258 return media::VideoCaptureFormats();
262 blink::WebVector<blink::WebMediaConstraint> mandatory;
263 blink::WebVector<blink::WebMediaConstraint> optional;
264 constraints.getMandatoryConstraints(mandatory);
265 constraints.getOptionalConstraints(optional);
266 media::VideoCaptureFormats candidates = supported_formats;
267 for (size_t i = 0; i < mandatory.size(); ++i) {
268 FilterFormatsByConstraint(mandatory[i], true, &candidates);
269 if (candidates.empty()) {
270 *unsatisfied_constraint = mandatory[i].m_name;
271 return candidates;
275 if (candidates.empty())
276 return candidates;
278 // Ok - all mandatory checked and we still have candidates.
279 // Let's try filtering using the optional constraints. The optional
280 // constraints must be filtered in the order they occur in |optional|.
281 // But if a constraint produce zero candidates, the constraint is ignored and
282 // the next constraint is tested.
283 // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#idl-def-Constraints
284 for (size_t i = 0; i < optional.size(); ++i) {
285 media::VideoCaptureFormats current_candidates = candidates;
286 FilterFormatsByConstraint(optional[i], false, &current_candidates);
287 if (!current_candidates.empty())
288 candidates = current_candidates;
291 // We have done as good as we can to filter the supported resolutions.
292 return candidates;
295 const media::VideoCaptureFormat& GetBestFormatBasedOnArea(
296 const media::VideoCaptureFormats& formats,
297 int area) {
298 media::VideoCaptureFormats::const_iterator it = formats.begin();
299 media::VideoCaptureFormats::const_iterator best_it = formats.begin();
300 int best_diff = std::numeric_limits<int>::max();
301 for (; it != formats.end(); ++it) {
302 const int diff = abs(area - it->frame_size.GetArea());
303 if (diff < best_diff) {
304 best_diff = diff;
305 best_it = it;
308 return *best_it;
311 // Find the format that best matches the default video size.
312 // This algorithm is chosen since a resolution must be picked even if no
313 // constraints are provided. We don't just select the maximum supported
314 // resolution since higher resolutions cost more in terms of complexity and
315 // many cameras have lower frame rate and have more noise in the image at
316 // their maximum supported resolution.
317 void GetBestCaptureFormat(
318 const media::VideoCaptureFormats& formats,
319 const blink::WebMediaConstraints& constraints,
320 media::VideoCaptureFormat* capture_format) {
321 DCHECK(!formats.empty());
323 int max_width;
324 int max_height;
325 GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
327 *capture_format = GetBestFormatBasedOnArea(
328 formats,
329 std::min(max_width,
330 static_cast<int>(MediaStreamVideoSource::kDefaultWidth)) *
331 std::min(max_height,
332 static_cast<int>(MediaStreamVideoSource::kDefaultHeight)));
335 } // anonymous namespace
337 // static
338 MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource(
339 const blink::WebMediaStreamSource& source) {
340 return static_cast<MediaStreamVideoSource*>(source.extraData());
343 // static
344 bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) {
345 for (const char* constraint : kSupportedConstraints) {
346 if (constraint == name)
347 return true;
349 return false;
352 MediaStreamVideoSource::MediaStreamVideoSource()
353 : state_(NEW),
354 track_adapter_(
355 new VideoTrackAdapter(ChildProcess::current()->io_task_runner())),
356 weak_factory_(this) {
359 MediaStreamVideoSource::~MediaStreamVideoSource() {
360 DCHECK(CalledOnValidThread());
363 void MediaStreamVideoSource::AddTrack(
364 MediaStreamVideoTrack* track,
365 const VideoCaptureDeliverFrameCB& frame_callback,
366 const blink::WebMediaConstraints& constraints,
367 const ConstraintsCallback& callback) {
368 DCHECK(CalledOnValidThread());
369 DCHECK(!constraints.isNull());
370 DCHECK(std::find(tracks_.begin(), tracks_.end(),
371 track) == tracks_.end());
372 tracks_.push_back(track);
374 requested_constraints_.push_back(
375 RequestedConstraints(track, frame_callback, constraints, callback));
377 switch (state_) {
378 case NEW: {
379 // Tab capture and Screen capture needs the maximum requested height
380 // and width to decide on the resolution.
381 int max_requested_width = 0;
382 GetMandatoryConstraintValueAsInteger(constraints, kMaxWidth,
383 &max_requested_width);
385 int max_requested_height = 0;
386 GetMandatoryConstraintValueAsInteger(constraints, kMaxHeight,
387 &max_requested_height);
389 double max_requested_frame_rate;
390 if (!GetConstraintValueAsDouble(constraints, kMaxFrameRate,
391 &max_requested_frame_rate)) {
392 max_requested_frame_rate = kDefaultFrameRate;
395 state_ = RETRIEVING_CAPABILITIES;
396 GetCurrentSupportedFormats(
397 max_requested_width,
398 max_requested_height,
399 max_requested_frame_rate,
400 base::Bind(&MediaStreamVideoSource::OnSupportedFormats,
401 weak_factory_.GetWeakPtr()));
403 break;
405 case STARTING:
406 case RETRIEVING_CAPABILITIES: {
407 // The |callback| will be triggered once the source has started or
408 // the capabilities have been retrieved.
409 break;
411 case ENDED:
412 case STARTED: {
413 // Currently, reconfiguring the source is not supported.
414 FinalizeAddTrack();
419 void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
420 DCHECK(CalledOnValidThread());
421 std::vector<MediaStreamVideoTrack*>::iterator it =
422 std::find(tracks_.begin(), tracks_.end(), video_track);
423 DCHECK(it != tracks_.end());
424 tracks_.erase(it);
426 // Check if |video_track| is waiting for applying new constraints and remove
427 // the request in that case.
428 for (std::vector<RequestedConstraints>::iterator it =
429 requested_constraints_.begin();
430 it != requested_constraints_.end(); ++it) {
431 if (it->track == video_track) {
432 requested_constraints_.erase(it);
433 break;
436 // Call |frame_adapter_->RemoveTrack| here even if adding the track has
437 // failed and |frame_adapter_->AddCallback| has not been called.
438 track_adapter_->RemoveTrack(video_track);
440 if (tracks_.empty())
441 StopSource();
444 base::SingleThreadTaskRunner* MediaStreamVideoSource::io_task_runner() const {
445 DCHECK(CalledOnValidThread());
446 return track_adapter_->io_task_runner();
449 void MediaStreamVideoSource::DoStopSource() {
450 DCHECK(CalledOnValidThread());
451 DVLOG(3) << "DoStopSource()";
452 if (state_ == ENDED)
453 return;
454 track_adapter_->StopFrameMonitoring();
455 StopSourceImpl();
456 state_ = ENDED;
457 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
460 void MediaStreamVideoSource::OnSupportedFormats(
461 const media::VideoCaptureFormats& formats) {
462 DCHECK(CalledOnValidThread());
463 DCHECK_EQ(RETRIEVING_CAPABILITIES, state_);
465 supported_formats_ = formats;
466 blink::WebMediaConstraints fulfilled_constraints;
467 if (!FindBestFormatWithConstraints(supported_formats_,
468 &current_format_,
469 &fulfilled_constraints)) {
470 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
471 // This object can be deleted after calling FinalizeAddTrack. See comment
472 // in the header file.
473 FinalizeAddTrack();
474 return;
477 state_ = STARTING;
478 DVLOG(3) << "Starting the capturer with "
479 << media::VideoCaptureFormat::ToString(current_format_);
481 StartSourceImpl(
482 current_format_,
483 fulfilled_constraints,
484 base::Bind(&VideoTrackAdapter::DeliverFrameOnIO, track_adapter_));
487 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
488 const media::VideoCaptureFormats& formats,
489 media::VideoCaptureFormat* best_format,
490 blink::WebMediaConstraints* fulfilled_constraints) {
491 DCHECK(CalledOnValidThread());
492 // Find the first constraints that we can fulfill.
493 for (const auto& request : requested_constraints_) {
494 const blink::WebMediaConstraints& requested_constraints =
495 request.constraints;
497 // If the source doesn't support capability enumeration it is still ok if
498 // no mandatory constraints have been specified. That just means that
499 // we will start with whatever format is native to the source.
500 if (formats.empty() && !HasMandatoryConstraints(requested_constraints)) {
501 *fulfilled_constraints = requested_constraints;
502 *best_format = media::VideoCaptureFormat();
503 return true;
505 blink::WebString unsatisfied_constraint;
506 media::VideoCaptureFormats filtered_formats =
507 FilterFormats(requested_constraints, formats, &unsatisfied_constraint);
508 if (filtered_formats.size() > 0) {
509 // A request with constraints that can be fulfilled.
510 *fulfilled_constraints = requested_constraints;
511 GetBestCaptureFormat(filtered_formats,
512 requested_constraints,
513 best_format);
514 return true;
517 return false;
520 void MediaStreamVideoSource::OnStartDone(MediaStreamRequestResult result) {
521 DCHECK(CalledOnValidThread());
522 DVLOG(3) << "OnStartDone({result =" << result << "})";
523 if (result == MEDIA_DEVICE_OK) {
524 DCHECK_EQ(STARTING, state_);
525 state_ = STARTED;
526 SetReadyState(blink::WebMediaStreamSource::ReadyStateLive);
528 track_adapter_->StartFrameMonitoring(
529 current_format_.frame_rate,
530 base::Bind(&MediaStreamVideoSource::SetMutedState,
531 weak_factory_.GetWeakPtr()));
533 } else {
534 StopSource();
537 // This object can be deleted after calling FinalizeAddTrack. See comment in
538 // the header file.
539 FinalizeAddTrack();
542 void MediaStreamVideoSource::FinalizeAddTrack() {
543 DCHECK(CalledOnValidThread());
544 media::VideoCaptureFormats formats;
545 formats.push_back(current_format_);
547 std::vector<RequestedConstraints> callbacks;
548 callbacks.swap(requested_constraints_);
549 for (const auto& request : callbacks) {
550 MediaStreamRequestResult result = MEDIA_DEVICE_OK;
551 blink::WebString unsatisfied_constraint;
553 if (HasMandatoryConstraints(request.constraints) &&
554 FilterFormats(request.constraints, formats,
555 &unsatisfied_constraint).empty()) {
556 result = MEDIA_DEVICE_CONSTRAINT_NOT_SATISFIED;
559 if (state_ != STARTED && result == MEDIA_DEVICE_OK)
560 result = MEDIA_DEVICE_TRACK_START_FAILURE;
562 if (result == MEDIA_DEVICE_OK) {
563 int max_width;
564 int max_height;
565 GetDesiredMaxWidthAndHeight(request.constraints, &max_width, &max_height);
566 double max_aspect_ratio;
567 double min_aspect_ratio;
568 GetDesiredMinAndMaxAspectRatio(request.constraints,
569 &min_aspect_ratio,
570 &max_aspect_ratio);
571 double max_frame_rate = 0.0f;
572 GetConstraintValueAsDouble(request.constraints,
573 kMaxFrameRate, &max_frame_rate);
575 track_adapter_->AddTrack(request.track, request.frame_callback,
576 max_width, max_height,
577 min_aspect_ratio, max_aspect_ratio,
578 max_frame_rate);
581 DVLOG(3) << "FinalizeAddTrack() result " << result;
583 if (!request.callback.is_null()) {
584 request.callback.Run(this, result, unsatisfied_constraint);
589 void MediaStreamVideoSource::SetReadyState(
590 blink::WebMediaStreamSource::ReadyState state) {
591 DVLOG(3) << "MediaStreamVideoSource::SetReadyState state " << state;
592 DCHECK(CalledOnValidThread());
593 if (!owner().isNull())
594 owner().setReadyState(state);
595 for (const auto& track : tracks_)
596 track->OnReadyStateChanged(state);
599 void MediaStreamVideoSource::SetMutedState(bool muted_state) {
600 DVLOG(3) << "MediaStreamVideoSource::SetMutedState state=" << muted_state;
601 DCHECK(CalledOnValidThread());
602 if (!owner().isNull()) {
603 owner().setReadyState(muted_state
604 ? blink::WebMediaStreamSource::ReadyStateMuted
605 : blink::WebMediaStreamSource::ReadyStateLive);
609 MediaStreamVideoSource::RequestedConstraints::RequestedConstraints(
610 MediaStreamVideoTrack* track,
611 const VideoCaptureDeliverFrameCB& frame_callback,
612 const blink::WebMediaConstraints& constraints,
613 const ConstraintsCallback& callback)
614 : track(track),
615 frame_callback(frame_callback),
616 constraints(constraints),
617 callback(callback) {
620 MediaStreamVideoSource::RequestedConstraints::~RequestedConstraints() {
623 } // namespace content