1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/media_stream_impl.h"
9 #include "base/logging.h"
10 #include "base/string_number_conversions.h"
11 #include "base/stringprintf.h"
12 #include "base/utf_string_conversions.h"
13 #include "content/renderer/media/media_stream_dependency_factory.h"
14 #include "content/renderer/media/media_stream_dispatcher.h"
15 #include "content/renderer/media/media_stream_extra_data.h"
16 #include "content/renderer/media/media_stream_source_extra_data.h"
17 #include "content/renderer/media/rtc_video_renderer.h"
18 #include "content/renderer/media/video_capture_impl_manager.h"
19 #include "content/renderer/media/webrtc_audio_capturer.h"
20 #include "content/renderer/media/webrtc_audio_renderer.h"
21 #include "content/renderer/media/webrtc_local_audio_renderer.h"
22 #include "content/renderer/media/webrtc_uma_histograms.h"
23 #include "third_party/WebKit/Source/Platform/chromium/public/WebMediaConstraints.h"
24 #include "third_party/WebKit/Source/Platform/chromium/public/WebMediaStreamSource.h"
25 #include "third_party/WebKit/Source/Platform/chromium/public/WebMediaStreamTrack.h"
26 #include "third_party/WebKit/Source/Platform/chromium/public/WebVector.h"
27 #include "third_party/WebKit/Source/WebKit/chromium/public/WebDocument.h"
28 #include "third_party/WebKit/Source/WebKit/chromium/public/WebFrame.h"
29 #include "third_party/WebKit/Source/WebKit/chromium/public/WebMediaStreamRegistry.h"
30 #include "third_party/WebKit/Source/WebKit/chromium/public/WebSecurityOrigin.h"
31 #include "webkit/media/media_stream_audio_renderer.h"
36 std::string
GetMandatoryStreamConstraint(
37 const WebKit::WebMediaConstraints
& constraints
, const std::string
& key
) {
38 if (constraints
.isNull())
41 WebKit::WebString value
;
42 constraints
.getMandatoryConstraintValue(UTF8ToUTF16(key
), value
);
43 return UTF16ToUTF8(value
);
46 void UpdateRequestOptions(
47 const WebKit::WebUserMediaRequest
& user_media_request
,
48 StreamOptions
* options
) {
49 if (options
->audio_type
!= content::MEDIA_NO_SERVICE
) {
50 std::string audio_stream_source
= GetMandatoryStreamConstraint(
51 user_media_request
.audioConstraints(), kMediaStreamSource
);
52 if (audio_stream_source
== kMediaStreamSourceTab
) {
53 options
->audio_type
= content::MEDIA_TAB_AUDIO_CAPTURE
;
54 options
->audio_device_id
= GetMandatoryStreamConstraint(
55 user_media_request
.audioConstraints(),
56 kMediaStreamSourceId
);
60 if (options
->video_type
!= content::MEDIA_NO_SERVICE
) {
61 std::string video_stream_source
= GetMandatoryStreamConstraint(
62 user_media_request
.videoConstraints(), kMediaStreamSource
);
63 if (video_stream_source
== kMediaStreamSourceTab
) {
64 options
->video_type
= content::MEDIA_TAB_VIDEO_CAPTURE
;
65 options
->video_device_id
= GetMandatoryStreamConstraint(
66 user_media_request
.videoConstraints(),
67 kMediaStreamSourceId
);
68 } else if (video_stream_source
== kMediaStreamSourceScreen
) {
69 options
->video_type
= content::MEDIA_SCREEN_VIDEO_CAPTURE
;
74 static int g_next_request_id
= 0;
76 // Creates a WebKit representation of a stream sources based on
77 // |devices| from the MediaStreamDispatcher.
78 void CreateWebKitSourceVector(
79 const std::string
& label
,
80 const StreamDeviceInfoArray
& devices
,
81 WebKit::WebMediaStreamSource::Type type
,
82 WebKit::WebVector
<WebKit::WebMediaStreamSource
>& webkit_sources
) {
83 CHECK_EQ(devices
.size(), webkit_sources
.size());
84 for (size_t i
= 0; i
< devices
.size(); ++i
) {
85 const char* track_type
=
86 (type
== WebKit::WebMediaStreamSource::TypeAudio
) ? "a" : "v";
87 std::string source_id
= base::StringPrintf("%s%s%u", label
.c_str(),
89 static_cast<unsigned int>(i
));
90 webkit_sources
[i
].initialize(
91 UTF8ToUTF16(source_id
),
93 UTF8ToUTF16(devices
[i
].device
.name
));
94 webkit_sources
[i
].setExtraData(
95 new content::MediaStreamSourceExtraData(devices
[i
]));
96 webkit_sources
[i
].setDeviceId(UTF8ToUTF16(devices
[i
].device
.id
.c_str()));
100 webrtc::MediaStreamInterface
* GetNativeMediaStream(
101 const WebKit::WebMediaStream
& descriptor
) {
102 content::MediaStreamExtraData
* extra_data
=
103 static_cast<content::MediaStreamExtraData
*>(descriptor
.extraData());
106 return extra_data
->stream();
111 MediaStreamImpl::MediaStreamImpl(
112 RenderView
* render_view
,
113 MediaStreamDispatcher
* media_stream_dispatcher
,
114 VideoCaptureImplManager
* vc_manager
,
115 MediaStreamDependencyFactory
* dependency_factory
)
116 : RenderViewObserver(render_view
),
117 dependency_factory_(dependency_factory
),
118 media_stream_dispatcher_(media_stream_dispatcher
),
119 vc_manager_(vc_manager
) {
122 MediaStreamImpl::~MediaStreamImpl() {
125 void MediaStreamImpl::OnLocalMediaStreamStop(
126 const std::string
& label
) {
127 DVLOG(1) << "MediaStreamImpl::OnLocalMediaStreamStop(" << label
<< ")";
129 UserMediaRequestInfo
* user_media_request
= FindUserMediaRequestInfo(label
);
130 if (user_media_request
) {
131 dependency_factory_
->StopLocalAudioSource(user_media_request
->descriptor
);
133 media_stream_dispatcher_
->StopStream(label
);
134 DeleteUserMediaRequestInfo(user_media_request
);
136 DVLOG(1) << "MediaStreamImpl::OnLocalMediaStreamStop: the stream has "
137 << "already been stopped.";
141 void MediaStreamImpl::requestUserMedia(
142 const WebKit::WebUserMediaRequest
& user_media_request
,
143 const WebKit::WebVector
<WebKit::WebMediaStreamSource
>& audio_sources
,
144 const WebKit::WebVector
<WebKit::WebMediaStreamSource
>& video_sources
) {
145 // Save histogram data so we can see how much GetUserMedia is used.
146 // The histogram counts the number of calls to the JS API
148 UpdateWebRTCMethodCount(WEBKIT_GET_USER_MEDIA
);
149 DCHECK(CalledOnValidThread());
150 int request_id
= g_next_request_id
++;
151 StreamOptions
options(MEDIA_NO_SERVICE
, MEDIA_NO_SERVICE
);
152 WebKit::WebFrame
* frame
= NULL
;
153 GURL security_origin
;
155 // |user_media_request| can't be mocked. So in order to test at all we check
157 if (user_media_request
.isNull()) {
159 if (audio_sources
.size() > 0)
160 options
.audio_type
= MEDIA_DEVICE_AUDIO_CAPTURE
;
161 if (video_sources
.size() > 0)
162 options
.video_type
= MEDIA_DEVICE_VIDEO_CAPTURE
;
164 if (user_media_request
.audio())
165 options
.audio_type
= MEDIA_DEVICE_AUDIO_CAPTURE
;
166 if (user_media_request
.video())
167 options
.video_type
= MEDIA_DEVICE_VIDEO_CAPTURE
;
169 security_origin
= GURL(user_media_request
.securityOrigin().toString());
170 // Get the WebFrame that requested a MediaStream.
171 // The frame is needed to tell the MediaStreamDispatcher when a stream goes
173 frame
= user_media_request
.ownerDocument().frame();
176 UpdateRequestOptions(user_media_request
, &options
);
179 DVLOG(1) << "MediaStreamImpl::requestUserMedia(" << request_id
<< ", [ "
180 << "audio=" << (options
.audio_type
)
181 << ", video=" << (options
.video_type
) << " ], "
182 << security_origin
.spec() << ")";
184 user_media_requests_
.push_back(
185 new UserMediaRequestInfo(request_id
, frame
, user_media_request
));
187 media_stream_dispatcher_
->GenerateStream(
194 void MediaStreamImpl::cancelUserMediaRequest(
195 const WebKit::WebUserMediaRequest
& user_media_request
) {
196 DCHECK(CalledOnValidThread());
197 UserMediaRequestInfo
* request
= FindUserMediaRequestInfo(user_media_request
);
199 // We can't abort the stream generation process.
200 // Instead, erase the request. Once the stream is generated we will stop the
201 // stream if the request does not exist.
202 DeleteUserMediaRequestInfo(request
);
206 WebKit::WebMediaStream
MediaStreamImpl::GetMediaStream(
208 return WebKit::WebMediaStreamRegistry::lookupMediaStreamDescriptor(url
);
211 bool MediaStreamImpl::IsMediaStream(const GURL
& url
) {
212 return CheckMediaStream(url
);
216 bool MediaStreamImpl::CheckMediaStream(const GURL
& url
) {
217 WebKit::WebMediaStream
descriptor(
218 WebKit::WebMediaStreamRegistry::lookupMediaStreamDescriptor(url
));
220 if (descriptor
.isNull() || !descriptor
.extraData())
221 return false; // This is not a valid stream.
223 webrtc::MediaStreamInterface
* stream
= GetNativeMediaStream(descriptor
);
225 (!stream
->GetVideoTracks().empty() || !stream
->GetAudioTracks().empty()));
228 scoped_refptr
<webkit_media::VideoFrameProvider
>
229 MediaStreamImpl::GetVideoFrameProvider(
231 const base::Closure
& error_cb
,
232 const webkit_media::VideoFrameProvider::RepaintCB
& repaint_cb
) {
233 DCHECK(CalledOnValidThread());
234 WebKit::WebMediaStream
descriptor(GetMediaStream(url
));
236 if (descriptor
.isNull() || !descriptor
.extraData())
237 return NULL
; // This is not a valid stream.
239 DVLOG(1) << "MediaStreamImpl::GetVideoFrameProvider stream:"
240 << UTF16ToUTF8(descriptor
.label());
242 webrtc::MediaStreamInterface
* stream
= GetNativeMediaStream(descriptor
);
244 return CreateVideoFrameProvider(stream
, error_cb
, repaint_cb
);
249 scoped_refptr
<webkit_media::MediaStreamAudioRenderer
>
250 MediaStreamImpl::GetAudioRenderer(const GURL
& url
) {
251 DCHECK(CalledOnValidThread());
252 WebKit::WebMediaStream
descriptor(GetMediaStream(url
));
254 if (descriptor
.isNull() || !descriptor
.extraData())
255 return NULL
; // This is not a valid stream.
257 DVLOG(1) << "MediaStreamImpl::GetAudioRenderer stream:"
258 << UTF16ToUTF8(descriptor
.label());
260 MediaStreamExtraData
* extra_data
=
261 static_cast<MediaStreamExtraData
*>(descriptor
.extraData());
263 if (extra_data
->is_local()) {
264 // Create the local audio renderer if the stream contains audio tracks.
265 return CreateLocalAudioRenderer(extra_data
->stream());
268 webrtc::MediaStreamInterface
* stream
= extra_data
->stream();
269 if (!stream
|| stream
->GetAudioTracks().empty())
272 // This is a remote media stream.
273 WebRtcAudioDeviceImpl
* audio_device
=
274 dependency_factory_
->GetWebRtcAudioDevice();
276 // Share the existing renderer if any, otherwise create a new one.
277 scoped_refptr
<WebRtcAudioRenderer
> renderer(audio_device
->renderer());
279 renderer
= CreateRemoteAudioRenderer(extra_data
->stream());
281 if (renderer
&& !audio_device
->SetAudioRenderer(renderer
))
287 // Callback from MediaStreamDispatcher.
288 // The requested stream have been generated by the MediaStreamDispatcher.
289 void MediaStreamImpl::OnStreamGenerated(
291 const std::string
& label
,
292 const StreamDeviceInfoArray
& audio_array
,
293 const StreamDeviceInfoArray
& video_array
) {
294 DCHECK(CalledOnValidThread());
295 DVLOG(1) << "MediaStreamImpl::OnStreamGenerated stream:" << label
;
297 UserMediaRequestInfo
* request_info
= FindUserMediaRequestInfo(request_id
);
299 // This can happen if the request is canceled or the frame reloads while
300 // MediaStreamDispatcher is processing the request.
301 // We need to tell the dispatcher to stop the stream.
302 media_stream_dispatcher_
->StopStream(label
);
303 DVLOG(1) << "Request ID not found";
306 request_info
->generated
= true;
308 WebKit::WebVector
<WebKit::WebMediaStreamSource
> audio_source_vector(
310 CreateWebKitSourceVector(label
, audio_array
,
311 WebKit::WebMediaStreamSource::TypeAudio
,
312 audio_source_vector
);
313 WebKit::WebVector
<WebKit::WebMediaStreamSource
> video_source_vector(
315 CreateWebKitSourceVector(label
, video_array
,
316 WebKit::WebMediaStreamSource::TypeVideo
,
317 video_source_vector
);
319 WebKit::WebUserMediaRequest
* request
= &(request_info
->request
);
320 WebKit::WebString webkit_label
= UTF8ToUTF16(label
);
321 WebKit::WebMediaStream
* description
= &(request_info
->descriptor
);
323 description
->initialize(webkit_label
, audio_source_vector
,
324 video_source_vector
);
326 // WebUserMediaRequest don't have an implementation in unit tests.
327 // Therefore we need to check for isNull here.
328 WebKit::WebMediaConstraints audio_constraints
= request
->isNull() ?
329 WebKit::WebMediaConstraints() : request
->audioConstraints();
330 WebKit::WebMediaConstraints video_constraints
= request
->isNull() ?
331 WebKit::WebMediaConstraints() : request
->videoConstraints();
333 dependency_factory_
->CreateNativeMediaSources(
334 audio_constraints
, video_constraints
, description
,
335 base::Bind(&MediaStreamImpl::OnCreateNativeSourcesComplete
, AsWeakPtr()));
338 // Callback from MediaStreamDispatcher.
339 // The requested stream failed to be generated.
340 void MediaStreamImpl::OnStreamGenerationFailed(int request_id
) {
341 DCHECK(CalledOnValidThread());
342 DVLOG(1) << "MediaStreamImpl::OnStreamGenerationFailed("
343 << request_id
<< ")";
344 UserMediaRequestInfo
* request_info
= FindUserMediaRequestInfo(request_id
);
346 // This can happen if the request is canceled or the frame reloads while
347 // MediaStreamDispatcher is processing the request.
348 DVLOG(1) << "Request ID not found";
351 CompleteGetUserMediaRequest(request_info
->descriptor
,
352 &request_info
->request
,
354 DeleteUserMediaRequestInfo(request_info
);
357 // Callback from MediaStreamDependencyFactory when the sources in |description|
358 // have been generated.
359 void MediaStreamImpl::OnCreateNativeSourcesComplete(
360 WebKit::WebMediaStream
* description
,
361 bool request_succeeded
) {
362 DVLOG(1) << "MediaStreamImpl::OnCreateNativeSourcesComplete stream:"
363 << UTF16ToUTF8(description
->label());
364 UserMediaRequestInfo
* request_info
= FindUserMediaRequestInfo(description
);
366 // This can happen if the request is canceled or the frame reloads while
367 // MediaStreamDependencyFactory is creating the sources.
368 DVLOG(1) << "Request ID not found";
372 // Create a native representation of the stream.
373 if (request_succeeded
) {
374 dependency_factory_
->CreateNativeLocalMediaStream(
376 base::Bind(&MediaStreamImpl::OnLocalMediaStreamStop
, AsWeakPtr()));
378 CompleteGetUserMediaRequest(request_info
->descriptor
, &request_info
->request
,
380 if (!request_succeeded
) {
381 OnLocalMediaStreamStop(UTF16ToUTF8(description
->label()));
385 void MediaStreamImpl::OnDevicesEnumerated(
387 const StreamDeviceInfoArray
& device_array
) {
388 DVLOG(1) << "MediaStreamImpl::OnDevicesEnumerated("
389 << request_id
<< ")";
393 void MediaStreamImpl::OnDevicesEnumerationFailed(int request_id
) {
394 DVLOG(1) << "MediaStreamImpl::OnDevicesEnumerationFailed("
395 << request_id
<< ")";
399 void MediaStreamImpl::OnDeviceOpened(
401 const std::string
& label
,
402 const StreamDeviceInfo
& video_device
) {
403 DVLOG(1) << "MediaStreamImpl::OnDeviceOpened("
404 << request_id
<< ", " << label
<< ")";
408 void MediaStreamImpl::OnDeviceOpenFailed(int request_id
) {
409 DVLOG(1) << "MediaStreamImpl::VideoDeviceOpenFailed("
410 << request_id
<< ")";
414 void MediaStreamImpl::CompleteGetUserMediaRequest(
415 const WebKit::WebMediaStream
& stream
,
416 WebKit::WebUserMediaRequest
* request_info
,
417 bool request_succeeded
) {
418 if (request_succeeded
) {
419 request_info
->requestSucceeded(stream
);
421 request_info
->requestFailed();
425 MediaStreamImpl::UserMediaRequestInfo
*
426 MediaStreamImpl::FindUserMediaRequestInfo(int request_id
) {
427 UserMediaRequests::iterator it
= user_media_requests_
.begin();
428 for (; it
!= user_media_requests_
.end(); ++it
) {
429 if ((*it
)->request_id
== request_id
)
435 MediaStreamImpl::UserMediaRequestInfo
*
436 MediaStreamImpl::FindUserMediaRequestInfo(
437 const WebKit::WebUserMediaRequest
& request
) {
438 UserMediaRequests::iterator it
= user_media_requests_
.begin();
439 for (; it
!= user_media_requests_
.end(); ++it
) {
440 if ((*it
)->request
== request
)
446 MediaStreamImpl::UserMediaRequestInfo
*
447 MediaStreamImpl::FindUserMediaRequestInfo(const std::string
& label
) {
448 UserMediaRequests::iterator it
= user_media_requests_
.begin();
449 for (; it
!= user_media_requests_
.end(); ++it
) {
450 if ((*it
)->generated
&& (*it
)->descriptor
.label() == UTF8ToUTF16(label
))
456 MediaStreamImpl::UserMediaRequestInfo
*
457 MediaStreamImpl::FindUserMediaRequestInfo(
458 WebKit::WebMediaStream
* descriptor
) {
459 UserMediaRequests::iterator it
= user_media_requests_
.begin();
460 for (; it
!= user_media_requests_
.end(); ++it
) {
461 if (&((*it
)->descriptor
) == descriptor
)
467 void MediaStreamImpl::DeleteUserMediaRequestInfo(
468 UserMediaRequestInfo
* request
) {
469 UserMediaRequests::iterator it
= user_media_requests_
.begin();
470 for (; it
!= user_media_requests_
.end(); ++it
) {
471 if ((*it
) == request
) {
472 user_media_requests_
.erase(it
);
479 void MediaStreamImpl::FrameWillClose(WebKit::WebFrame
* frame
) {
480 // Loop through all UserMediaRequests and find the requests that belong to the
481 // frame that is being closed.
482 UserMediaRequests::iterator request_it
= user_media_requests_
.begin();
484 while (request_it
!= user_media_requests_
.end()) {
485 if ((*request_it
)->frame
== frame
) {
486 DVLOG(1) << "MediaStreamImpl::FrameWillClose: "
487 << "Cancel user media request " << (*request_it
)->request_id
;
488 // If the request is generated, it means that the MediaStreamDispatcher
489 // has generated a stream for us and we need to let the
490 // MediaStreamDispatcher know that the stream is no longer wanted.
491 // If not, we cancel the request and delete the request object.
492 if ((*request_it
)->generated
) {
493 // Stop the local audio track before closing the device in the browser.
494 dependency_factory_
->StopLocalAudioSource((*request_it
)->descriptor
);
496 media_stream_dispatcher_
->StopStream(
497 UTF16ToUTF8((*request_it
)->descriptor
.label()));
499 media_stream_dispatcher_
->CancelGenerateStream(
500 (*request_it
)->request_id
);
502 request_it
= user_media_requests_
.erase(request_it
);
509 scoped_refptr
<webkit_media::VideoFrameProvider
>
510 MediaStreamImpl::CreateVideoFrameProvider(
511 webrtc::MediaStreamInterface
* stream
,
512 const base::Closure
& error_cb
,
513 const webkit_media::VideoFrameProvider::RepaintCB
& repaint_cb
) {
514 if (stream
->GetVideoTracks().empty())
517 DVLOG(1) << "MediaStreamImpl::CreateRemoteVideoFrameProvider label:"
520 return new RTCVideoRenderer(
521 stream
->GetVideoTracks()[0],
526 scoped_refptr
<WebRtcAudioRenderer
> MediaStreamImpl::CreateRemoteAudioRenderer(
527 webrtc::MediaStreamInterface
* stream
) {
528 if (stream
->GetAudioTracks().empty())
531 DVLOG(1) << "MediaStreamImpl::CreateRemoteAudioRenderer label:"
534 return new WebRtcAudioRenderer(RenderViewObserver::routing_id());
537 scoped_refptr
<WebRtcLocalAudioRenderer
>
538 MediaStreamImpl::CreateLocalAudioRenderer(
539 webrtc::MediaStreamInterface
* stream
) {
540 if (stream
->GetAudioTracks().empty())
543 DVLOG(1) << "MediaStreamImpl::CreateLocalAudioRenderer label:"
546 scoped_refptr
<WebRtcAudioCapturer
> source
=
547 dependency_factory_
->GetWebRtcAudioDevice()->capturer();
552 webrtc::AudioTrackVector audio_tracks
= stream
->GetAudioTracks();
553 DCHECK_EQ(audio_tracks
.size(), 1u);
554 webrtc::AudioTrackInterface
* audio_track
= audio_tracks
[0];
555 DVLOG(1) << "audio_track.kind : " << audio_track
->kind()
556 << "audio_track.id : " << audio_track
->id()
557 << "audio_track.enabled: " << audio_track
->enabled();
559 // Create a new WebRtcLocalAudioRenderer instance and connect it to the
560 // existing WebRtcAudioCapturer so that the renderer can use it as source.
561 return new WebRtcLocalAudioRenderer(source
, audio_track
,
562 RenderViewObserver::routing_id());
565 MediaStreamSourceExtraData::MediaStreamSourceExtraData(
566 const StreamDeviceInfo
& device_info
)
567 : device_info_(device_info
) {
570 MediaStreamSourceExtraData::MediaStreamSourceExtraData(
571 media::AudioCapturerSource
* source
)
572 : audio_source_(source
) {
575 MediaStreamSourceExtraData::~MediaStreamSourceExtraData() {}
577 MediaStreamExtraData::MediaStreamExtraData(
578 webrtc::MediaStreamInterface
* stream
, bool is_local
)
580 is_local_(is_local
) {
583 MediaStreamExtraData::~MediaStreamExtraData() {
586 void MediaStreamExtraData::SetLocalStreamStopCallback(
587 const StreamStopCallback
& stop_callback
) {
588 stream_stop_callback_
= stop_callback
;
591 void MediaStreamExtraData::OnLocalStreamStop() {
592 if (!stream_stop_callback_
.is_null())
593 stream_stop_callback_
.Run(stream_
->label());
596 } // namespace content