Roll src/third_party/WebKit a452221:9ff6d11 (svn 202117:202119)
[chromium-blink-merge.git] / content / renderer / media / media_stream_video_source.cc
blob1274ecf705d6ce55f6aa9fd918fed9516af2b2a7
1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/media_stream_video_source.h"
7 #include <algorithm>
8 #include <limits>
9 #include <string>
11 #include "base/logging.h"
12 #include "base/strings/string_number_conversions.h"
13 #include "base/trace_event/trace_event.h"
14 #include "content/child/child_process.h"
15 #include "content/renderer/media/media_stream_constraints_util.h"
16 #include "content/renderer/media/media_stream_video_track.h"
17 #include "content/renderer/media/video_track_adapter.h"
19 namespace content {
21 // Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b
22 const char MediaStreamVideoSource::kMinAspectRatio[] = "minAspectRatio";
23 const char MediaStreamVideoSource::kMaxAspectRatio[] = "maxAspectRatio";
24 const char MediaStreamVideoSource::kMaxWidth[] = "maxWidth";
25 const char MediaStreamVideoSource::kMinWidth[] = "minWidth";
26 const char MediaStreamVideoSource::kMaxHeight[] = "maxHeight";
27 const char MediaStreamVideoSource::kMinHeight[] = "minHeight";
28 const char MediaStreamVideoSource::kMaxFrameRate[] = "maxFrameRate";
29 const char MediaStreamVideoSource::kMinFrameRate[] = "minFrameRate";
31 // TODO(mcasas): Find a way to guarantee all constraints are added to the array.
32 const char* kSupportedConstraints[] = {
33 MediaStreamVideoSource::kMaxAspectRatio,
34 MediaStreamVideoSource::kMinAspectRatio,
35 MediaStreamVideoSource::kMaxWidth,
36 MediaStreamVideoSource::kMinWidth,
37 MediaStreamVideoSource::kMaxHeight,
38 MediaStreamVideoSource::kMinHeight,
39 MediaStreamVideoSource::kMaxFrameRate,
40 MediaStreamVideoSource::kMinFrameRate,
43 namespace {
45 // Google-specific key prefix. Constraints with this prefix are ignored if they
46 // are unknown.
47 const char kGooglePrefix[] = "goog";
49 // Returns true if |constraint| has mandatory constraints.
50 bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
51 blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
52 constraints.getMandatoryConstraints(mandatory_constraints);
53 return !mandatory_constraints.isEmpty();
56 // Retrieve the desired max width and height from |constraints|. If not set,
57 // the |desired_width| and |desired_height| are set to
58 // std::numeric_limits<int>::max();
59 // If either max width or height is set as a mandatory constraint, the optional
60 // constraints are not checked.
61 void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
62 int* desired_width, int* desired_height) {
63 *desired_width = std::numeric_limits<int>::max();
64 *desired_height = std::numeric_limits<int>::max();
66 bool mandatory = GetMandatoryConstraintValueAsInteger(
67 constraints,
68 MediaStreamVideoSource::kMaxWidth,
69 desired_width);
70 mandatory |= GetMandatoryConstraintValueAsInteger(
71 constraints,
72 MediaStreamVideoSource::kMaxHeight,
73 desired_height);
74 if (mandatory)
75 return;
77 GetOptionalConstraintValueAsInteger(constraints,
78 MediaStreamVideoSource::kMaxWidth,
79 desired_width);
80 GetOptionalConstraintValueAsInteger(constraints,
81 MediaStreamVideoSource::kMaxHeight,
82 desired_height);
85 // Retrieve the desired max and min aspect ratio from |constraints|. If not set,
86 // the |min_aspect_ratio| is set to 0 and |max_aspect_ratio| is set to
87 // std::numeric_limits<double>::max();
88 // If either min or max aspect ratio is set as a mandatory constraint, the
89 // optional constraints are not checked.
90 void GetDesiredMinAndMaxAspectRatio(
91 const blink::WebMediaConstraints& constraints,
92 double* min_aspect_ratio,
93 double* max_aspect_ratio) {
94 *min_aspect_ratio = 0;
95 *max_aspect_ratio = std::numeric_limits<double>::max();
97 bool mandatory = GetMandatoryConstraintValueAsDouble(
98 constraints,
99 MediaStreamVideoSource::kMinAspectRatio,
100 min_aspect_ratio);
101 mandatory |= GetMandatoryConstraintValueAsDouble(
102 constraints,
103 MediaStreamVideoSource::kMaxAspectRatio,
104 max_aspect_ratio);
105 if (mandatory)
106 return;
108 GetOptionalConstraintValueAsDouble(constraints,
109 MediaStreamVideoSource::kMinAspectRatio,
110 min_aspect_ratio);
111 GetOptionalConstraintValueAsDouble(constraints,
112 MediaStreamVideoSource::kMaxAspectRatio,
113 max_aspect_ratio);
116 // Returns true if |constraint| is fulfilled. |format| can be changed by a
117 // constraint, e.g. the frame rate can be changed by setting maxFrameRate.
118 bool UpdateFormatForConstraint(const blink::WebMediaConstraint& constraint,
119 bool mandatory,
120 media::VideoCaptureFormat* format) {
121 DCHECK(format != NULL);
123 if (!format->IsValid())
124 return false;
126 const std::string constraint_name = constraint.m_name.utf8();
127 const std::string constraint_value = constraint.m_value.utf8();
129 if (constraint_name.find(kGooglePrefix) == 0) {
130 // These are actually options, not constraints, so they can be satisfied
131 // regardless of the format.
132 return true;
135 if (constraint_name == MediaStreamSource::kSourceId) {
136 // This is a constraint that doesn't affect the format.
137 return true;
140 // Ignore Chrome specific Tab capture constraints.
141 if (constraint_name == kMediaStreamSource ||
142 constraint_name == kMediaStreamSourceId)
143 return true;
145 if (constraint_name == MediaStreamVideoSource::kMinAspectRatio ||
146 constraint_name == MediaStreamVideoSource::kMaxAspectRatio) {
147 // These constraints are handled by cropping if the camera outputs the wrong
148 // aspect ratio.
149 double value;
150 return base::StringToDouble(constraint_value, &value);
153 double value = 0.0;
154 if (!base::StringToDouble(constraint_value, &value)) {
155 DLOG(WARNING) << "Can't parse MediaStream constraint. Name:"
156 << constraint_name << " Value:" << constraint_value;
157 return false;
160 if (constraint_name == MediaStreamVideoSource::kMinWidth) {
161 return (value <= format->frame_size.width());
162 } else if (constraint_name == MediaStreamVideoSource::kMaxWidth) {
163 return value > 0.0;
164 } else if (constraint_name == MediaStreamVideoSource::kMinHeight) {
165 return (value <= format->frame_size.height());
166 } else if (constraint_name == MediaStreamVideoSource::kMaxHeight) {
167 return value > 0.0;
168 } else if (constraint_name == MediaStreamVideoSource::kMinFrameRate) {
169 return (value > 0.0) && (value <= format->frame_rate);
170 } else if (constraint_name == MediaStreamVideoSource::kMaxFrameRate) {
171 if (value <= 0.0) {
172 // The frame rate is set by constraint.
173 // Don't allow 0 as frame rate if it is a mandatory constraint.
174 // Set the frame rate to 1 if it is not mandatory.
175 if (mandatory) {
176 return false;
177 } else {
178 value = 1.0;
181 format->frame_rate =
182 (format->frame_rate > value) ? value : format->frame_rate;
183 return true;
184 } else {
185 LOG(WARNING) << "Found unknown MediaStream constraint. Name:"
186 << constraint_name << " Value:" << constraint_value;
187 return false;
191 // Removes media::VideoCaptureFormats from |formats| that don't meet
192 // |constraint|.
193 void FilterFormatsByConstraint(const blink::WebMediaConstraint& constraint,
194 bool mandatory,
195 media::VideoCaptureFormats* formats) {
196 DVLOG(3) << "FilterFormatsByConstraint("
197 << "{ constraint.m_name = " << constraint.m_name.utf8()
198 << " constraint.m_value = " << constraint.m_value.utf8()
199 << " mandatory = " << mandatory << "})";
200 media::VideoCaptureFormats::iterator format_it = formats->begin();
201 while (format_it != formats->end()) {
202 // Modify the format_it to fulfill the constraint if possible.
203 // Delete it otherwise.
204 if (!UpdateFormatForConstraint(constraint, mandatory, &(*format_it)))
205 format_it = formats->erase(format_it);
206 else
207 ++format_it;
211 // Returns the media::VideoCaptureFormats that matches |constraints|.
212 media::VideoCaptureFormats FilterFormats(
213 const blink::WebMediaConstraints& constraints,
214 const media::VideoCaptureFormats& supported_formats,
215 blink::WebString* unsatisfied_constraint) {
216 if (constraints.isNull())
217 return supported_formats;
219 double max_aspect_ratio;
220 double min_aspect_ratio;
221 GetDesiredMinAndMaxAspectRatio(constraints,
222 &min_aspect_ratio,
223 &max_aspect_ratio);
225 if (min_aspect_ratio > max_aspect_ratio || max_aspect_ratio < 0.05f) {
226 DLOG(WARNING) << "Wrong requested aspect ratio.";
227 return media::VideoCaptureFormats();
230 int min_width = 0;
231 GetMandatoryConstraintValueAsInteger(constraints,
232 MediaStreamVideoSource::kMinWidth,
233 &min_width);
234 int min_height = 0;
235 GetMandatoryConstraintValueAsInteger(constraints,
236 MediaStreamVideoSource::kMinHeight,
237 &min_height);
238 int max_width;
239 int max_height;
240 GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
242 if (min_width > max_width || min_height > max_height)
243 return media::VideoCaptureFormats();
245 double min_frame_rate = 0.0f;
246 double max_frame_rate = 0.0f;
247 if (GetConstraintValueAsDouble(constraints,
248 MediaStreamVideoSource::kMaxFrameRate,
249 &max_frame_rate) &&
250 GetConstraintValueAsDouble(constraints,
251 MediaStreamVideoSource::kMinFrameRate,
252 &min_frame_rate)) {
253 if (min_frame_rate > max_frame_rate) {
254 DLOG(WARNING) << "Wrong requested frame rate.";
255 return media::VideoCaptureFormats();
259 blink::WebVector<blink::WebMediaConstraint> mandatory;
260 blink::WebVector<blink::WebMediaConstraint> optional;
261 constraints.getMandatoryConstraints(mandatory);
262 constraints.getOptionalConstraints(optional);
263 media::VideoCaptureFormats candidates = supported_formats;
264 for (const auto& constraint : mandatory) {
265 FilterFormatsByConstraint(constraint, true, &candidates);
266 if (candidates.empty()) {
267 *unsatisfied_constraint = constraint.m_name;
268 return candidates;
272 if (candidates.empty())
273 return candidates;
275 // Ok - all mandatory checked and we still have candidates.
276 // Let's try filtering using the optional constraints. The optional
277 // constraints must be filtered in the order they occur in |optional|.
278 // But if a constraint produce zero candidates, the constraint is ignored and
279 // the next constraint is tested.
280 // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#idl-def-Constraints
281 for (const auto& constraint : optional) {
282 media::VideoCaptureFormats current_candidates = candidates;
283 FilterFormatsByConstraint(constraint, false, &current_candidates);
284 if (!current_candidates.empty())
285 candidates = current_candidates;
288 // We have done as good as we can to filter the supported resolutions.
289 return candidates;
292 media::VideoCaptureFormat GetBestFormatBasedOnArea(
293 const media::VideoCaptureFormats& formats,
294 int area) {
295 DCHECK(!formats.empty());
296 const media::VideoCaptureFormat* best_format = nullptr;
297 int best_diff = std::numeric_limits<int>::max();
298 for (const auto& format : formats) {
299 const int diff = abs(area - format.frame_size.GetArea());
300 if (diff < best_diff) {
301 best_diff = diff;
302 best_format = &format;
305 return *best_format;
308 // Find the format that best matches the default video size.
309 // This algorithm is chosen since a resolution must be picked even if no
310 // constraints are provided. We don't just select the maximum supported
311 // resolution since higher resolutions cost more in terms of complexity and
312 // many cameras have lower frame rate and have more noise in the image at
313 // their maximum supported resolution.
314 media::VideoCaptureFormat GetBestCaptureFormat(
315 const media::VideoCaptureFormats& formats,
316 const blink::WebMediaConstraints& constraints) {
317 DCHECK(!formats.empty());
319 int max_width;
320 int max_height;
321 GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
322 const int area =
323 std::min(max_width,
324 static_cast<int>(MediaStreamVideoSource::kDefaultWidth)) *
325 std::min(max_height,
326 static_cast<int>(MediaStreamVideoSource::kDefaultHeight));
328 return GetBestFormatBasedOnArea(formats, area);
331 } // anonymous namespace
333 // static
334 MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource(
335 const blink::WebMediaStreamSource& source) {
336 return static_cast<MediaStreamVideoSource*>(source.extraData());
339 // static
340 bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) {
341 return std::find(kSupportedConstraints,
342 kSupportedConstraints + arraysize(kSupportedConstraints),
343 name) !=
344 kSupportedConstraints + arraysize(kSupportedConstraints);
347 MediaStreamVideoSource::MediaStreamVideoSource()
348 : state_(NEW),
349 track_adapter_(
350 new VideoTrackAdapter(ChildProcess::current()->io_task_runner())),
351 weak_factory_(this) {
354 MediaStreamVideoSource::~MediaStreamVideoSource() {
355 DCHECK(CalledOnValidThread());
358 void MediaStreamVideoSource::AddTrack(
359 MediaStreamVideoTrack* track,
360 const VideoCaptureDeliverFrameCB& frame_callback,
361 const blink::WebMediaConstraints& constraints,
362 const ConstraintsCallback& callback) {
363 DCHECK(CalledOnValidThread());
364 DCHECK(!constraints.isNull());
365 DCHECK(std::find(tracks_.begin(), tracks_.end(), track) == tracks_.end());
366 tracks_.push_back(track);
368 track_descriptors_.push_back(
369 TrackDescriptor(track, frame_callback, constraints, callback));
371 switch (state_) {
372 case NEW: {
373 // Tab capture and Screen capture needs the maximum requested height
374 // and width to decide on the resolution.
375 int max_requested_width = 0;
376 GetMandatoryConstraintValueAsInteger(constraints, kMaxWidth,
377 &max_requested_width);
379 int max_requested_height = 0;
380 GetMandatoryConstraintValueAsInteger(constraints, kMaxHeight,
381 &max_requested_height);
383 double max_requested_frame_rate = kDefaultFrameRate;
384 GetConstraintValueAsDouble(constraints, kMaxFrameRate,
385 &max_requested_frame_rate);
387 state_ = RETRIEVING_CAPABILITIES;
388 GetCurrentSupportedFormats(
389 max_requested_width,
390 max_requested_height,
391 max_requested_frame_rate,
392 base::Bind(&MediaStreamVideoSource::OnSupportedFormats,
393 weak_factory_.GetWeakPtr()));
395 break;
397 case STARTING:
398 case RETRIEVING_CAPABILITIES: {
399 // The |callback| will be triggered once the source has started or
400 // the capabilities have been retrieved.
401 break;
403 case ENDED:
404 case STARTED: {
405 // Currently, reconfiguring the source is not supported.
406 FinalizeAddTrack();
411 void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
412 DCHECK(CalledOnValidThread());
413 std::vector<MediaStreamVideoTrack*>::iterator it =
414 std::find(tracks_.begin(), tracks_.end(), video_track);
415 DCHECK(it != tracks_.end());
416 tracks_.erase(it);
418 for (std::vector<TrackDescriptor>::iterator it = track_descriptors_.begin();
419 it != track_descriptors_.end(); ++it) {
420 if (it->track == video_track) {
421 track_descriptors_.erase(it);
422 break;
426 // Call |frame_adapter_->RemoveTrack| here even if adding the track has
427 // failed and |frame_adapter_->AddCallback| has not been called.
428 track_adapter_->RemoveTrack(video_track);
430 if (tracks_.empty())
431 StopSource();
434 base::SingleThreadTaskRunner* MediaStreamVideoSource::io_task_runner() const {
435 DCHECK(CalledOnValidThread());
436 return track_adapter_->io_task_runner();
439 void MediaStreamVideoSource::DoStopSource() {
440 DCHECK(CalledOnValidThread());
441 DVLOG(3) << "DoStopSource()";
442 if (state_ == ENDED)
443 return;
444 track_adapter_->StopFrameMonitoring();
445 StopSourceImpl();
446 state_ = ENDED;
447 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
450 void MediaStreamVideoSource::OnSupportedFormats(
451 const media::VideoCaptureFormats& formats) {
452 DCHECK(CalledOnValidThread());
453 DCHECK_EQ(RETRIEVING_CAPABILITIES, state_);
455 supported_formats_ = formats;
456 blink::WebMediaConstraints fulfilled_constraints;
457 if (!FindBestFormatWithConstraints(supported_formats_,
458 &current_format_,
459 &fulfilled_constraints)) {
460 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
461 // This object can be deleted after calling FinalizeAddTrack. See comment
462 // in the header file.
463 FinalizeAddTrack();
464 return;
467 state_ = STARTING;
468 DVLOG(3) << "Starting the capturer with "
469 << media::VideoCaptureFormat::ToString(current_format_);
471 StartSourceImpl(
472 current_format_,
473 fulfilled_constraints,
474 base::Bind(&VideoTrackAdapter::DeliverFrameOnIO, track_adapter_));
477 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
478 const media::VideoCaptureFormats& formats,
479 media::VideoCaptureFormat* best_format,
480 blink::WebMediaConstraints* fulfilled_constraints) {
481 DCHECK(CalledOnValidThread());
482 // Find the first constraints that we can fulfill.
483 for (const auto& track : track_descriptors_) {
484 const blink::WebMediaConstraints& track_constraints = track.constraints;
486 // If the source doesn't support capability enumeration it is still ok if
487 // no mandatory constraints have been specified. That just means that
488 // we will start with whatever format is native to the source.
489 if (formats.empty() && !HasMandatoryConstraints(track_constraints)) {
490 *fulfilled_constraints = track_constraints;
491 *best_format = media::VideoCaptureFormat();
492 return true;
494 blink::WebString unsatisfied_constraint;
495 const media::VideoCaptureFormats filtered_formats =
496 FilterFormats(track_constraints, formats, &unsatisfied_constraint);
497 if (filtered_formats.empty())
498 continue;
500 // A request with constraints that can be fulfilled.
501 *fulfilled_constraints = track_constraints;
502 *best_format = GetBestCaptureFormat(filtered_formats, track_constraints);
503 return true;
505 return false;
508 void MediaStreamVideoSource::OnStartDone(MediaStreamRequestResult result) {
509 DCHECK(CalledOnValidThread());
510 DVLOG(3) << "OnStartDone({result =" << result << "})";
511 if (result == MEDIA_DEVICE_OK) {
512 DCHECK_EQ(STARTING, state_);
513 state_ = STARTED;
514 SetReadyState(blink::WebMediaStreamSource::ReadyStateLive);
516 track_adapter_->StartFrameMonitoring(
517 current_format_.frame_rate,
518 base::Bind(&MediaStreamVideoSource::SetMutedState,
519 weak_factory_.GetWeakPtr()));
521 } else {
522 StopSource();
525 // This object can be deleted after calling FinalizeAddTrack. See comment in
526 // the header file.
527 FinalizeAddTrack();
530 void MediaStreamVideoSource::FinalizeAddTrack() {
531 DCHECK(CalledOnValidThread());
532 const media::VideoCaptureFormats formats(1, current_format_);
534 std::vector<TrackDescriptor> track_descriptors;
535 track_descriptors.swap(track_descriptors_);
536 for (const auto& track : track_descriptors) {
537 MediaStreamRequestResult result = MEDIA_DEVICE_OK;
538 blink::WebString unsatisfied_constraint;
540 if (HasMandatoryConstraints(track.constraints) &&
541 FilterFormats(track.constraints, formats, &unsatisfied_constraint)
542 .empty()) {
543 result = MEDIA_DEVICE_CONSTRAINT_NOT_SATISFIED;
546 if (state_ != STARTED && result == MEDIA_DEVICE_OK)
547 result = MEDIA_DEVICE_TRACK_START_FAILURE;
549 if (result == MEDIA_DEVICE_OK) {
550 int max_width;
551 int max_height;
552 GetDesiredMaxWidthAndHeight(track.constraints, &max_width, &max_height);
553 double max_aspect_ratio;
554 double min_aspect_ratio;
555 GetDesiredMinAndMaxAspectRatio(track.constraints,
556 &min_aspect_ratio,
557 &max_aspect_ratio);
558 double max_frame_rate = 0.0f;
559 GetConstraintValueAsDouble(track.constraints,
560 kMaxFrameRate, &max_frame_rate);
562 track_adapter_->AddTrack(track.track, track.frame_callback, max_width,
563 max_height, min_aspect_ratio, max_aspect_ratio,
564 max_frame_rate);
567 DVLOG(3) << "FinalizeAddTrack() result " << result;
569 if (!track.callback.is_null())
570 track.callback.Run(this, result, unsatisfied_constraint);
574 void MediaStreamVideoSource::SetReadyState(
575 blink::WebMediaStreamSource::ReadyState state) {
576 DVLOG(3) << "MediaStreamVideoSource::SetReadyState state " << state;
577 DCHECK(CalledOnValidThread());
578 if (!owner().isNull())
579 owner().setReadyState(state);
580 for (const auto& track : tracks_)
581 track->OnReadyStateChanged(state);
584 void MediaStreamVideoSource::SetMutedState(bool muted_state) {
585 DVLOG(3) << "MediaStreamVideoSource::SetMutedState state=" << muted_state;
586 DCHECK(CalledOnValidThread());
587 if (!owner().isNull()) {
588 owner().setReadyState(muted_state
589 ? blink::WebMediaStreamSource::ReadyStateMuted
590 : blink::WebMediaStreamSource::ReadyStateLive);
594 MediaStreamVideoSource::TrackDescriptor::TrackDescriptor(
595 MediaStreamVideoTrack* track,
596 const VideoCaptureDeliverFrameCB& frame_callback,
597 const blink::WebMediaConstraints& constraints,
598 const ConstraintsCallback& callback)
599 : track(track),
600 frame_callback(frame_callback),
601 constraints(constraints),
602 callback(callback) {
605 MediaStreamVideoSource::TrackDescriptor::~TrackDescriptor() {
608 } // namespace content