content/renderer/media/remote_media_stream_impl.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/remote_media_stream_impl.h"
   6
   7 #include <string>
   8
   9 #include "base/bind.h"
  10 #include "base/location.h"
  11 #include "base/logging.h"
  12 #include "base/strings/utf_string_conversions.h"
  13 #include "base/thread_task_runner_handle.h"
  14 #include "content/renderer/media/media_stream.h"
  15 #include "content/renderer/media/media_stream_track.h"
  16 #include "content/renderer/media/media_stream_video_track.h"
  17 #include "content/renderer/media/webrtc/media_stream_remote_video_source.h"
  18 #include "content/renderer/media/webrtc/peer_connection_dependency_factory.h"
  19 #include "content/renderer/media/webrtc/track_observer.h"
  20 #include "third_party/WebKit/public/platform/WebString.h"
  21
  22 namespace content {
  23 namespace {
  24
  25 template <typename WebRtcTrackVector, typename AdapterType>
  26 void CreateAdaptersForTracks(
  27     const WebRtcTrackVector& tracks,
  28     std::vector<scoped_refptr<AdapterType>>* observers,
  29     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread) {
  30   for (auto& track : tracks)
  31     observers->push_back(new AdapterType(main_thread, track));
  32 }
  33
  34 template<typename VectorType>
  35 bool IsTrackInVector(const VectorType& v, const std::string& id) {
  36   for (const auto& t : v) {
  37     if (t->id() == id)
  38       return true;
  39   }
  40   return false;
  41 }
  42 }  // namespace
  43
  44 // TODO(tommi): Move this class to a separate set of files.
  45 class RemoteMediaStreamAudioTrack : public MediaStreamTrack {
  46  public:
  47   RemoteMediaStreamAudioTrack(
  48       const scoped_refptr<webrtc::AudioTrackInterface>& track,
  49       const scoped_refptr<base::SingleThreadTaskRunner>& signaling_thread)
  50       : MediaStreamTrack(false), track_(track),
  51         signaling_thread_(signaling_thread) {
  52   }
  53
  54   ~RemoteMediaStreamAudioTrack() override {}
  55
  56  private:
  57   void SetEnabled(bool enabled) override {
  58     track_->set_enabled(enabled);
  59   }
  60
  61   void Stop() override {
  62     // Stop means that a track should be stopped permanently. But
  63     // since there is no proper way of doing that on a remote track, we can
  64     // at least disable the track. Blink will not call down to the content layer
  65     // after a track has been stopped.
  66     SetEnabled(false);
  67   }
  68
  69  private:
  70   const scoped_refptr<webrtc::AudioTrackInterface> track_;
  71   const scoped_refptr<base::SingleThreadTaskRunner> signaling_thread_;
  72 };
  73
  74 // Base class used for mapping between webrtc and blink MediaStream tracks.
  75 // An instance of a RemoteMediaStreamTrackAdapter is stored in
  76 // RemoteMediaStreamImpl per remote audio and video track.
  77 template<typename WebRtcMediaStreamTrackType>
  78 class RemoteMediaStreamTrackAdapter
  79     : public base::RefCountedThreadSafe<
  80           RemoteMediaStreamTrackAdapter<WebRtcMediaStreamTrackType>> {
  81  public:
  82   RemoteMediaStreamTrackAdapter(
  83       const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
  84       WebRtcMediaStreamTrackType* webrtc_track)
  85       : main_thread_(main_thread), webrtc_track_(webrtc_track),
  86         id_(webrtc_track->id()) {
  87   }
  88
  89   const scoped_refptr<WebRtcMediaStreamTrackType>& observed_track() {
  90     return webrtc_track_;
  91   }
  92
  93   blink::WebMediaStreamTrack* webkit_track() {
  94     DCHECK(main_thread_->BelongsToCurrentThread());
  95     DCHECK(!webkit_track_.isNull());
  96     return &webkit_track_;
  97   }
  98
  99   const std::string& id() const { return id_; }
 100
 101   bool initialized() const {
 102     DCHECK(main_thread_->BelongsToCurrentThread());
 103     return !webkit_track_.isNull();
 104   }
 105
 106   void Initialize() {
 107     DCHECK(main_thread_->BelongsToCurrentThread());
 108     DCHECK(!initialized());
 109     webkit_initialize_.Run();
 110     webkit_initialize_.Reset();
 111     DCHECK(initialized());
 112   }
 113
 114  protected:
 115   friend class base::RefCountedThreadSafe<
 116       RemoteMediaStreamTrackAdapter<WebRtcMediaStreamTrackType>>;
 117
 118   virtual ~RemoteMediaStreamTrackAdapter() {
 119     DCHECK(main_thread_->BelongsToCurrentThread());
 120   }
 121
 122   void InitializeWebkitTrack(blink::WebMediaStreamSource::Type type) {
 123     DCHECK(main_thread_->BelongsToCurrentThread());
 124     DCHECK(webkit_track_.isNull());
 125
 126     blink::WebString webkit_track_id(base::UTF8ToUTF16(id_));
 127     blink::WebMediaStreamSource webkit_source;
 128     webkit_source.initialize(webkit_track_id, type, webkit_track_id,
 129                              true /* remote */, true /* readonly */);
 130     webkit_track_.initialize(webkit_track_id, webkit_source);
 131     DCHECK(!webkit_track_.isNull());
 132   }
 133
 134   const scoped_refptr<base::SingleThreadTaskRunner> main_thread_;
 135   // This callback will be run when Initialize() is called and then freed.
 136   // The callback is used by derived classes to bind objects that need to be
 137   // instantiated and initialized on the signaling thread but then moved to
 138   // and used on the main thread when initializing the webkit object(s).
 139   base::Callback<void()> webkit_initialize_;
 140
 141  private:
 142   const scoped_refptr<WebRtcMediaStreamTrackType> webrtc_track_;
 143   blink::WebMediaStreamTrack webkit_track_;
 144   // const copy of the webrtc track id that allows us to check it from both the
 145   // main and signaling threads without incurring a synchronous thread hop.
 146   const std::string id_;
 147
 148   DISALLOW_COPY_AND_ASSIGN(RemoteMediaStreamTrackAdapter);
 149 };
 150
 151 class RemoteVideoTrackAdapter
 152     : public RemoteMediaStreamTrackAdapter<webrtc::VideoTrackInterface> {
 153  public:
 154   // Called on the signaling thread
 155   RemoteVideoTrackAdapter(
 156       const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 157       webrtc::VideoTrackInterface* webrtc_track)
 158       : RemoteMediaStreamTrackAdapter(main_thread, webrtc_track) {
 159     scoped_ptr<TrackObserver> observer(
 160         new TrackObserver(main_thread, observed_track().get()));
 161     // Here, we use base::Unretained() to avoid a circular reference.
 162     webkit_initialize_ = base::Bind(
 163         &RemoteVideoTrackAdapter::InitializeWebkitVideoTrack,
 164         base::Unretained(this), base::Passed(&observer),
 165         observed_track()->enabled());
 166   }
 167
 168  protected:
 169   ~RemoteVideoTrackAdapter() override {}
 170
 171  private:
 172   void InitializeWebkitVideoTrack(scoped_ptr<TrackObserver> observer,
 173                                   bool enabled) {
 174     DCHECK(main_thread_->BelongsToCurrentThread());
 175     scoped_ptr<MediaStreamRemoteVideoSource> video_source(
 176         new MediaStreamRemoteVideoSource(observer.Pass()));
 177     InitializeWebkitTrack(blink::WebMediaStreamSource::TypeVideo);
 178     webkit_track()->source().setExtraData(video_source.get());
 179     // Initial constraints must be provided to a MediaStreamVideoTrack. But
 180     // no constraints are available initially on a remote video track.
 181     blink::WebMediaConstraints constraints;
 182     constraints.initialize();
 183     MediaStreamVideoTrack* media_stream_track =
 184         new MediaStreamVideoTrack(video_source.release(), constraints,
 185             MediaStreamVideoSource::ConstraintsCallback(), enabled);
 186     webkit_track()->setExtraData(media_stream_track);
 187   }
 188 };
 189
 190 // RemoteAudioTrackAdapter is responsible for listening on state
 191 // change notifications on a remote webrtc audio MediaStreamTracks and notify
 192 // WebKit.
 193 class RemoteAudioTrackAdapter
 194     : public RemoteMediaStreamTrackAdapter<webrtc::AudioTrackInterface>,
 195       public webrtc::ObserverInterface {
 196  public:
 197   RemoteAudioTrackAdapter(
 198       const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 199       webrtc::AudioTrackInterface* webrtc_track);
 200
 201   void Unregister();
 202
 203  protected:
 204   ~RemoteAudioTrackAdapter() override;
 205
 206  private:
 207   void InitializeWebkitAudioTrack(
 208       scoped_ptr<RemoteMediaStreamAudioTrack> media_stream_track);
 209
 210   // webrtc::ObserverInterface implementation.
 211   void OnChanged() override;
 212
 213   void OnChangedOnMainThread(
 214       webrtc::MediaStreamTrackInterface::TrackState state);
 215
 216 #if DCHECK_IS_ON()
 217   bool unregistered_;
 218 #endif
 219
 220   webrtc::MediaStreamTrackInterface::TrackState state_;
 221
 222   DISALLOW_COPY_AND_ASSIGN(RemoteAudioTrackAdapter);
 223 };
 224
 225 // Called on the signaling thread.
 226 RemoteAudioTrackAdapter::RemoteAudioTrackAdapter(
 227     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 228     webrtc::AudioTrackInterface* webrtc_track)
 229     : RemoteMediaStreamTrackAdapter(main_thread, webrtc_track),
 230 #if DCHECK_IS_ON()
 231       unregistered_(false),
 232 #endif
 233       state_(observed_track()->state()) {
 234   // TODO(tommi): Use TrackObserver instead.
 235   observed_track()->RegisterObserver(this);
 236   scoped_ptr<RemoteMediaStreamAudioTrack> media_stream_track(
 237       new RemoteMediaStreamAudioTrack(observed_track().get(),
 238           base::ThreadTaskRunnerHandle::Get()));
 239   // Here, we use base::Unretained() to avoid a circular reference.
 240   webkit_initialize_ = base::Bind(
 241           &RemoteAudioTrackAdapter::InitializeWebkitAudioTrack,
 242           base::Unretained(this), base::Passed(&media_stream_track));
 243 }
 244
 245 RemoteAudioTrackAdapter::~RemoteAudioTrackAdapter() {
 246 #if DCHECK_IS_ON()
 247   DCHECK(unregistered_);
 248 #endif
 249 }
 250
 251 void RemoteAudioTrackAdapter::Unregister() {
 252 #if DCHECK_IS_ON()
 253   DCHECK(!unregistered_);
 254   unregistered_ = true;
 255 #endif
 256   observed_track()->UnregisterObserver(this);
 257 }
 258
 259 void RemoteAudioTrackAdapter::InitializeWebkitAudioTrack(
 260     scoped_ptr<RemoteMediaStreamAudioTrack> media_stream_track) {
 261   InitializeWebkitTrack(blink::WebMediaStreamSource::TypeAudio);
 262   webkit_track()->setExtraData(media_stream_track.release());
 263 }
 264
 265 void RemoteAudioTrackAdapter::OnChanged() {
 266   main_thread_->PostTask(FROM_HERE,
 267       base::Bind(&RemoteAudioTrackAdapter::OnChangedOnMainThread,
 268           this, observed_track()->state()));
 269 }
 270
 271 void RemoteAudioTrackAdapter::OnChangedOnMainThread(
 272     webrtc::MediaStreamTrackInterface::TrackState state) {
 273   DCHECK(main_thread_->BelongsToCurrentThread());
 274
 275   if (state == state_ || !initialized())
 276     return;
 277
 278   state_ = state;
 279
 280   switch (state) {
 281     case webrtc::MediaStreamTrackInterface::kInitializing:
 282       // Ignore the kInitializing state since there is no match in
 283       // WebMediaStreamSource::ReadyState.
 284       break;
 285     case webrtc::MediaStreamTrackInterface::kLive:
 286       webkit_track()->source().setReadyState(
 287           blink::WebMediaStreamSource::ReadyStateLive);
 288       break;
 289     case webrtc::MediaStreamTrackInterface::kEnded:
 290       webkit_track()->source().setReadyState(
 291           blink::WebMediaStreamSource::ReadyStateEnded);
 292       break;
 293     default:
 294       NOTREACHED();
 295       break;
 296   }
 297 }
 298
 299 RemoteMediaStreamImpl::Observer::Observer(
 300     const base::WeakPtr<RemoteMediaStreamImpl>& media_stream,
 301     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 302     webrtc::MediaStreamInterface* webrtc_stream)
 303     : media_stream_(media_stream),
 304       main_thread_(main_thread),
 305       webrtc_stream_(webrtc_stream) {
 306   webrtc_stream_->RegisterObserver(this);
 307 }
 308
 309 RemoteMediaStreamImpl::Observer::~Observer() {
 310   DCHECK(!webrtc_stream_.get()) << "Unregister hasn't been called";
 311 }
 312
 313 void RemoteMediaStreamImpl::Observer::InitializeOnMainThread(
 314     const std::string& label) {
 315   DCHECK(main_thread_->BelongsToCurrentThread());
 316   if (media_stream_)
 317     media_stream_->InitializeOnMainThread(label);
 318 }
 319
 320 void RemoteMediaStreamImpl::Observer::Unregister() {
 321   DCHECK(main_thread_->BelongsToCurrentThread());
 322   webrtc_stream_->UnregisterObserver(this);
 323   // Since we're guaranteed to not get further notifications, it's safe to
 324   // release the webrtc_stream_ here.
 325   webrtc_stream_ = nullptr;
 326 }
 327
 328 void RemoteMediaStreamImpl::Observer::OnChanged() {
 329   scoped_ptr<RemoteAudioTrackAdapters> audio(new RemoteAudioTrackAdapters());
 330   scoped_ptr<RemoteVideoTrackAdapters> video(new RemoteVideoTrackAdapters());
 331
 332   CreateAdaptersForTracks(
 333       webrtc_stream_->GetAudioTracks(), audio.get(), main_thread_);
 334   CreateAdaptersForTracks(
 335       webrtc_stream_->GetVideoTracks(), video.get(), main_thread_);
 336
 337   main_thread_->PostTask(FROM_HERE,
 338       base::Bind(&RemoteMediaStreamImpl::Observer::OnChangedOnMainThread,
 339       this, base::Passed(&audio), base::Passed(&video)));
 340 }
 341
 342 void RemoteMediaStreamImpl::Observer::OnChangedOnMainThread(
 343     scoped_ptr<RemoteAudioTrackAdapters> audio_tracks,
 344     scoped_ptr<RemoteVideoTrackAdapters> video_tracks) {
 345   DCHECK(main_thread_->BelongsToCurrentThread());
 346   if (media_stream_)
 347     media_stream_->OnChanged(audio_tracks.Pass(), video_tracks.Pass());
 348 }
 349
 350 // Called on the signaling thread.
 351 RemoteMediaStreamImpl::RemoteMediaStreamImpl(
 352     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 353     webrtc::MediaStreamInterface* webrtc_stream)
 354     : signaling_thread_(base::ThreadTaskRunnerHandle::Get()),
 355       weak_factory_(this) {
 356   observer_ = new RemoteMediaStreamImpl::Observer(
 357       weak_factory_.GetWeakPtr(), main_thread, webrtc_stream);
 358   CreateAdaptersForTracks(webrtc_stream->GetAudioTracks(),
 359       &audio_track_observers_, main_thread);
 360   CreateAdaptersForTracks(webrtc_stream->GetVideoTracks(),
 361       &video_track_observers_, main_thread);
 362
 363   main_thread->PostTask(FROM_HERE,
 364       base::Bind(&RemoteMediaStreamImpl::Observer::InitializeOnMainThread,
 365           observer_, webrtc_stream->label()));
 366 }
 367
 368 RemoteMediaStreamImpl::~RemoteMediaStreamImpl() {
 369   DCHECK(observer_->main_thread()->BelongsToCurrentThread());
 370   for (auto& track : audio_track_observers_)
 371     track->Unregister();
 372   observer_->Unregister();
 373 }
 374
 375 void RemoteMediaStreamImpl::InitializeOnMainThread(const std::string& label) {
 376   DCHECK(observer_->main_thread()->BelongsToCurrentThread());
 377   blink::WebVector<blink::WebMediaStreamTrack> webkit_audio_tracks(
 378         audio_track_observers_.size());
 379   for (size_t i = 0; i < audio_track_observers_.size(); ++i) {
 380     audio_track_observers_[i]->Initialize();
 381     webkit_audio_tracks[i] = *audio_track_observers_[i]->webkit_track();
 382   }
 383
 384   blink::WebVector<blink::WebMediaStreamTrack> webkit_video_tracks(
 385         video_track_observers_.size());
 386   for (size_t i = 0; i < video_track_observers_.size(); ++i) {
 387     video_track_observers_[i]->Initialize();
 388     webkit_video_tracks[i] = *video_track_observers_[i]->webkit_track();
 389   }
 390
 391   webkit_stream_.initialize(base::UTF8ToUTF16(label),
 392                             webkit_audio_tracks, webkit_video_tracks);
 393   webkit_stream_.setExtraData(new MediaStream(observer_->stream().get()));
 394 }
 395
 396 void RemoteMediaStreamImpl::OnChanged(
 397     scoped_ptr<RemoteAudioTrackAdapters> audio_tracks,
 398     scoped_ptr<RemoteVideoTrackAdapters> video_tracks) {
 399   // Find removed tracks.
 400   auto audio_it = audio_track_observers_.begin();
 401   while (audio_it != audio_track_observers_.end()) {
 402     if (!IsTrackInVector(*audio_tracks.get(), (*audio_it)->id())) {
 403       (*audio_it)->Unregister();
 404        webkit_stream_.removeTrack(*(*audio_it)->webkit_track());
 405        audio_it = audio_track_observers_.erase(audio_it);
 406     } else {
 407       ++audio_it;
 408     }
 409   }
 410
 411   auto video_it = video_track_observers_.begin();
 412   while (video_it != video_track_observers_.end()) {
 413     if (!IsTrackInVector(*video_tracks.get(), (*video_it)->id())) {
 414       webkit_stream_.removeTrack(*(*video_it)->webkit_track());
 415       video_it = video_track_observers_.erase(video_it);
 416     } else {
 417       ++video_it;
 418     }
 419   }
 420
 421   // Find added tracks.
 422   for (auto& track : *audio_tracks.get()) {
 423     if (!IsTrackInVector(audio_track_observers_, track->id())) {
 424       track->Initialize();
 425       audio_track_observers_.push_back(track);
 426       webkit_stream_.addTrack(*track->webkit_track());
 427       // Set the track to null to avoid unregistering it below now that it's
 428       // been associated with a media stream.
 429       track = nullptr;
 430     }
 431   }
 432
 433   // Find added video tracks.
 434   for (const auto& track : *video_tracks.get()) {
 435     if (!IsTrackInVector(video_track_observers_, track->id())) {
 436       track->Initialize();
 437       video_track_observers_.push_back(track);
 438       webkit_stream_.addTrack(*track->webkit_track());
 439     }
 440   }
 441
 442   // Unregister all the audio track observers that were not used.
 443   // We need to do this before destruction since the observers can't unregister
 444   // from within the dtor due to a race.
 445   for (auto& track : *audio_tracks.get()) {
 446     if (track.get())
 447       track->Unregister();
 448   }
 449 }
 450
 451 }  // namespace content