content/renderer/media/remote_media_stream_impl.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/remote_media_stream_impl.h"
   6
   7 #include <string>
   8
   9 #include "base/bind.h"
  10 #include "base/location.h"
  11 #include "base/logging.h"
  12 #include "base/strings/utf_string_conversions.h"
  13 #include "base/thread_task_runner_handle.h"
  14 #include "content/renderer/media/media_stream.h"
  15 #include "content/renderer/media/media_stream_track.h"
  16 #include "content/renderer/media/media_stream_video_track.h"
  17 #include "content/renderer/media/webrtc/media_stream_remote_video_source.h"
  18 #include "content/renderer/media/webrtc/peer_connection_dependency_factory.h"
  19 #include "content/renderer/media/webrtc/track_observer.h"
  20 #include "third_party/WebKit/public/platform/WebMediaStreamSource.h"
  21 #include "third_party/WebKit/public/platform/WebMediaStreamTrack.h"
  22 #include "third_party/WebKit/public/platform/WebString.h"
  23
  24 namespace content {
  25 namespace {
  26
  27 template <typename WebRtcTrackVector, typename AdapterType>
  28 void CreateAdaptersForTracks(
  29     const WebRtcTrackVector& tracks,
  30     std::vector<scoped_refptr<AdapterType>>* observers,
  31     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread) {
  32   for (auto& track : tracks)
  33     observers->push_back(new AdapterType(main_thread, track));
  34 }
  35
  36 template<typename VectorType>
  37 bool IsTrackInVector(const VectorType& v, const std::string& id) {
  38   for (const auto& t : v) {
  39     if (t->id() == id)
  40       return true;
  41   }
  42   return false;
  43 }
  44 }  // namespace
  45
  46 // TODO(tommi): Move this class to a separate set of files.
  47 class RemoteMediaStreamAudioTrack : public MediaStreamTrack {
  48  public:
  49   RemoteMediaStreamAudioTrack(
  50       const scoped_refptr<webrtc::AudioTrackInterface>& track,
  51       const scoped_refptr<base::SingleThreadTaskRunner>& signaling_thread)
  52       : MediaStreamTrack(false), track_(track),
  53         signaling_thread_(signaling_thread) {
  54   }
  55
  56   ~RemoteMediaStreamAudioTrack() override {}
  57
  58  private:
  59   void SetEnabled(bool enabled) override {
  60     track_->set_enabled(enabled);
  61   }
  62
  63   void Stop() override {
  64     // Stop means that a track should be stopped permanently. But
  65     // since there is no proper way of doing that on a remote track, we can
  66     // at least disable the track. Blink will not call down to the content layer
  67     // after a track has been stopped.
  68     SetEnabled(false);
  69   }
  70
  71  private:
  72   const scoped_refptr<webrtc::AudioTrackInterface> track_;
  73   const scoped_refptr<base::SingleThreadTaskRunner> signaling_thread_;
  74 };
  75
  76 // Base class used for mapping between webrtc and blink MediaStream tracks.
  77 // An instance of a RemoteMediaStreamTrackAdapter is stored in
  78 // RemoteMediaStreamImpl per remote audio and video track.
  79 template<typename WebRtcMediaStreamTrackType>
  80 class RemoteMediaStreamTrackAdapter
  81     : public base::RefCountedThreadSafe<
  82           RemoteMediaStreamTrackAdapter<WebRtcMediaStreamTrackType>> {
  83  public:
  84   RemoteMediaStreamTrackAdapter(
  85       const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
  86       WebRtcMediaStreamTrackType* webrtc_track)
  87       : main_thread_(main_thread), webrtc_track_(webrtc_track),
  88         id_(webrtc_track->id()) {
  89   }
  90
  91   const scoped_refptr<WebRtcMediaStreamTrackType>& observed_track() {
  92     return webrtc_track_;
  93   }
  94
  95   blink::WebMediaStreamTrack* webkit_track() {
  96     DCHECK(main_thread_->BelongsToCurrentThread());
  97     DCHECK(!webkit_track_.isNull());
  98     return &webkit_track_;
  99   }
 100
 101   const std::string& id() const { return id_; }
 102
 103   bool initialized() const {
 104     DCHECK(main_thread_->BelongsToCurrentThread());
 105     return !webkit_track_.isNull();
 106   }
 107
 108   void Initialize() {
 109     DCHECK(main_thread_->BelongsToCurrentThread());
 110     DCHECK(!initialized());
 111     webkit_initialize_.Run();
 112     webkit_initialize_.Reset();
 113     DCHECK(initialized());
 114   }
 115
 116  protected:
 117   friend class base::RefCountedThreadSafe<
 118       RemoteMediaStreamTrackAdapter<WebRtcMediaStreamTrackType>>;
 119
 120   virtual ~RemoteMediaStreamTrackAdapter() {
 121     DCHECK(main_thread_->BelongsToCurrentThread());
 122   }
 123
 124   void InitializeWebkitTrack(blink::WebMediaStreamSource::Type type) {
 125     DCHECK(main_thread_->BelongsToCurrentThread());
 126     DCHECK(webkit_track_.isNull());
 127
 128     blink::WebString webkit_track_id(base::UTF8ToUTF16(id_));
 129     blink::WebMediaStreamSource webkit_source;
 130     webkit_source.initialize(webkit_track_id, type, webkit_track_id,
 131                              true /* remote */, true /* readonly */);
 132     webkit_track_.initialize(webkit_track_id, webkit_source);
 133     DCHECK(!webkit_track_.isNull());
 134   }
 135
 136   const scoped_refptr<base::SingleThreadTaskRunner> main_thread_;
 137   // This callback will be run when Initialize() is called and then freed.
 138   // The callback is used by derived classes to bind objects that need to be
 139   // instantiated and initialized on the signaling thread but then moved to
 140   // and used on the main thread when initializing the webkit object(s).
 141   base::Callback<void()> webkit_initialize_;
 142
 143  private:
 144   const scoped_refptr<WebRtcMediaStreamTrackType> webrtc_track_;
 145   blink::WebMediaStreamTrack webkit_track_;
 146   // const copy of the webrtc track id that allows us to check it from both the
 147   // main and signaling threads without incurring a synchronous thread hop.
 148   const std::string id_;
 149
 150   DISALLOW_COPY_AND_ASSIGN(RemoteMediaStreamTrackAdapter);
 151 };
 152
 153 class RemoteVideoTrackAdapter
 154     : public RemoteMediaStreamTrackAdapter<webrtc::VideoTrackInterface> {
 155  public:
 156   // Called on the signaling thread
 157   RemoteVideoTrackAdapter(
 158       const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 159       webrtc::VideoTrackInterface* webrtc_track)
 160       : RemoteMediaStreamTrackAdapter(main_thread, webrtc_track) {
 161     scoped_ptr<TrackObserver> observer(
 162         new TrackObserver(main_thread, observed_track().get()));
 163     // Here, we use base::Unretained() to avoid a circular reference.
 164     webkit_initialize_ = base::Bind(
 165         &RemoteVideoTrackAdapter::InitializeWebkitVideoTrack,
 166         base::Unretained(this), base::Passed(&observer),
 167         observed_track()->enabled());
 168   }
 169
 170  protected:
 171   ~RemoteVideoTrackAdapter() override {}
 172
 173  private:
 174   void InitializeWebkitVideoTrack(scoped_ptr<TrackObserver> observer,
 175                                   bool enabled) {
 176     DCHECK(main_thread_->BelongsToCurrentThread());
 177     scoped_ptr<MediaStreamRemoteVideoSource> video_source(
 178         new MediaStreamRemoteVideoSource(observer.Pass()));
 179     InitializeWebkitTrack(blink::WebMediaStreamSource::TypeVideo);
 180     webkit_track()->source().setExtraData(video_source.get());
 181     // Initial constraints must be provided to a MediaStreamVideoTrack. But
 182     // no constraints are available initially on a remote video track.
 183     blink::WebMediaConstraints constraints;
 184     constraints.initialize();
 185     MediaStreamVideoTrack* media_stream_track =
 186         new MediaStreamVideoTrack(video_source.release(), constraints,
 187             MediaStreamVideoSource::ConstraintsCallback(), enabled);
 188     webkit_track()->setExtraData(media_stream_track);
 189   }
 190 };
 191
 192 // RemoteAudioTrackAdapter is responsible for listening on state
 193 // change notifications on a remote webrtc audio MediaStreamTracks and notify
 194 // WebKit.
 195 class RemoteAudioTrackAdapter
 196     : public RemoteMediaStreamTrackAdapter<webrtc::AudioTrackInterface>,
 197       public webrtc::ObserverInterface {
 198  public:
 199   RemoteAudioTrackAdapter(
 200       const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 201       webrtc::AudioTrackInterface* webrtc_track);
 202
 203   void Unregister();
 204
 205  protected:
 206   ~RemoteAudioTrackAdapter() override;
 207
 208  private:
 209   void InitializeWebkitAudioTrack(
 210       scoped_ptr<RemoteMediaStreamAudioTrack> media_stream_track);
 211
 212   // webrtc::ObserverInterface implementation.
 213   void OnChanged() override;
 214
 215   void OnChangedOnMainThread(
 216       webrtc::MediaStreamTrackInterface::TrackState state);
 217
 218 #if DCHECK_IS_ON()
 219   bool unregistered_;
 220 #endif
 221
 222   webrtc::MediaStreamTrackInterface::TrackState state_;
 223
 224   DISALLOW_COPY_AND_ASSIGN(RemoteAudioTrackAdapter);
 225 };
 226
 227 // Called on the signaling thread.
 228 RemoteAudioTrackAdapter::RemoteAudioTrackAdapter(
 229     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 230     webrtc::AudioTrackInterface* webrtc_track)
 231     : RemoteMediaStreamTrackAdapter(main_thread, webrtc_track),
 232 #if DCHECK_IS_ON()
 233       unregistered_(false),
 234 #endif
 235       state_(observed_track()->state()) {
 236   // TODO(tommi): Use TrackObserver instead.
 237   observed_track()->RegisterObserver(this);
 238   scoped_ptr<RemoteMediaStreamAudioTrack> media_stream_track(
 239       new RemoteMediaStreamAudioTrack(observed_track().get(),
 240           base::ThreadTaskRunnerHandle::Get()));
 241   // Here, we use base::Unretained() to avoid a circular reference.
 242   webkit_initialize_ = base::Bind(
 243           &RemoteAudioTrackAdapter::InitializeWebkitAudioTrack,
 244           base::Unretained(this), base::Passed(&media_stream_track));
 245 }
 246
 247 RemoteAudioTrackAdapter::~RemoteAudioTrackAdapter() {
 248 #if DCHECK_IS_ON()
 249   DCHECK(unregistered_);
 250 #endif
 251 }
 252
 253 void RemoteAudioTrackAdapter::Unregister() {
 254 #if DCHECK_IS_ON()
 255   DCHECK(!unregistered_);
 256   unregistered_ = true;
 257 #endif
 258   observed_track()->UnregisterObserver(this);
 259 }
 260
 261 void RemoteAudioTrackAdapter::InitializeWebkitAudioTrack(
 262     scoped_ptr<RemoteMediaStreamAudioTrack> media_stream_track) {
 263   InitializeWebkitTrack(blink::WebMediaStreamSource::TypeAudio);
 264   webkit_track()->setExtraData(media_stream_track.release());
 265 }
 266
 267 void RemoteAudioTrackAdapter::OnChanged() {
 268   main_thread_->PostTask(FROM_HERE,
 269       base::Bind(&RemoteAudioTrackAdapter::OnChangedOnMainThread,
 270           this, observed_track()->state()));
 271 }
 272
 273 void RemoteAudioTrackAdapter::OnChangedOnMainThread(
 274     webrtc::MediaStreamTrackInterface::TrackState state) {
 275   DCHECK(main_thread_->BelongsToCurrentThread());
 276
 277   if (state == state_ || !initialized())
 278     return;
 279
 280   state_ = state;
 281
 282   switch (state) {
 283     case webrtc::MediaStreamTrackInterface::kInitializing:
 284       // Ignore the kInitializing state since there is no match in
 285       // WebMediaStreamSource::ReadyState.
 286       break;
 287     case webrtc::MediaStreamTrackInterface::kLive:
 288       webkit_track()->source().setReadyState(
 289           blink::WebMediaStreamSource::ReadyStateLive);
 290       break;
 291     case webrtc::MediaStreamTrackInterface::kEnded:
 292       webkit_track()->source().setReadyState(
 293           blink::WebMediaStreamSource::ReadyStateEnded);
 294       break;
 295     default:
 296       NOTREACHED();
 297       break;
 298   }
 299 }
 300
 301 RemoteMediaStreamImpl::Observer::Observer(
 302     const base::WeakPtr<RemoteMediaStreamImpl>& media_stream,
 303     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 304     webrtc::MediaStreamInterface* webrtc_stream)
 305     : media_stream_(media_stream),
 306       main_thread_(main_thread),
 307       webrtc_stream_(webrtc_stream) {
 308   webrtc_stream_->RegisterObserver(this);
 309 }
 310
 311 RemoteMediaStreamImpl::Observer::~Observer() {
 312   DCHECK(!webrtc_stream_.get()) << "Unregister hasn't been called";
 313 }
 314
 315 void RemoteMediaStreamImpl::Observer::InitializeOnMainThread(
 316     const std::string& label) {
 317   DCHECK(main_thread_->BelongsToCurrentThread());
 318   if (media_stream_)
 319     media_stream_->InitializeOnMainThread(label);
 320 }
 321
 322 void RemoteMediaStreamImpl::Observer::Unregister() {
 323   DCHECK(main_thread_->BelongsToCurrentThread());
 324   webrtc_stream_->UnregisterObserver(this);
 325   // Since we're guaranteed to not get further notifications, it's safe to
 326   // release the webrtc_stream_ here.
 327   webrtc_stream_ = nullptr;
 328 }
 329
 330 void RemoteMediaStreamImpl::Observer::OnChanged() {
 331   scoped_ptr<RemoteAudioTrackAdapters> audio(new RemoteAudioTrackAdapters());
 332   scoped_ptr<RemoteVideoTrackAdapters> video(new RemoteVideoTrackAdapters());
 333
 334   CreateAdaptersForTracks(
 335       webrtc_stream_->GetAudioTracks(), audio.get(), main_thread_);
 336   CreateAdaptersForTracks(
 337       webrtc_stream_->GetVideoTracks(), video.get(), main_thread_);
 338
 339   main_thread_->PostTask(FROM_HERE,
 340       base::Bind(&RemoteMediaStreamImpl::Observer::OnChangedOnMainThread,
 341       this, base::Passed(&audio), base::Passed(&video)));
 342 }
 343
 344 void RemoteMediaStreamImpl::Observer::OnChangedOnMainThread(
 345     scoped_ptr<RemoteAudioTrackAdapters> audio_tracks,
 346     scoped_ptr<RemoteVideoTrackAdapters> video_tracks) {
 347   DCHECK(main_thread_->BelongsToCurrentThread());
 348   if (media_stream_)
 349     media_stream_->OnChanged(audio_tracks.Pass(), video_tracks.Pass());
 350 }
 351
 352 // Called on the signaling thread.
 353 RemoteMediaStreamImpl::RemoteMediaStreamImpl(
 354     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 355     webrtc::MediaStreamInterface* webrtc_stream)
 356     : signaling_thread_(base::ThreadTaskRunnerHandle::Get()),
 357       weak_factory_(this) {
 358   observer_ = new RemoteMediaStreamImpl::Observer(
 359       weak_factory_.GetWeakPtr(), main_thread, webrtc_stream);
 360   CreateAdaptersForTracks(webrtc_stream->GetAudioTracks(),
 361       &audio_track_observers_, main_thread);
 362   CreateAdaptersForTracks(webrtc_stream->GetVideoTracks(),
 363       &video_track_observers_, main_thread);
 364
 365   main_thread->PostTask(FROM_HERE,
 366       base::Bind(&RemoteMediaStreamImpl::Observer::InitializeOnMainThread,
 367           observer_, webrtc_stream->label()));
 368 }
 369
 370 RemoteMediaStreamImpl::~RemoteMediaStreamImpl() {
 371   DCHECK(observer_->main_thread()->BelongsToCurrentThread());
 372   for (auto& track : audio_track_observers_)
 373     track->Unregister();
 374   observer_->Unregister();
 375 }
 376
 377 void RemoteMediaStreamImpl::InitializeOnMainThread(const std::string& label) {
 378   DCHECK(observer_->main_thread()->BelongsToCurrentThread());
 379   blink::WebVector<blink::WebMediaStreamTrack> webkit_audio_tracks(
 380         audio_track_observers_.size());
 381   for (size_t i = 0; i < audio_track_observers_.size(); ++i) {
 382     audio_track_observers_[i]->Initialize();
 383     webkit_audio_tracks[i] = *audio_track_observers_[i]->webkit_track();
 384   }
 385
 386   blink::WebVector<blink::WebMediaStreamTrack> webkit_video_tracks(
 387         video_track_observers_.size());
 388   for (size_t i = 0; i < video_track_observers_.size(); ++i) {
 389     video_track_observers_[i]->Initialize();
 390     webkit_video_tracks[i] = *video_track_observers_[i]->webkit_track();
 391   }
 392
 393   webkit_stream_.initialize(base::UTF8ToUTF16(label),
 394                             webkit_audio_tracks, webkit_video_tracks);
 395   webkit_stream_.setExtraData(new MediaStream(observer_->stream().get()));
 396 }
 397
 398 void RemoteMediaStreamImpl::OnChanged(
 399     scoped_ptr<RemoteAudioTrackAdapters> audio_tracks,
 400     scoped_ptr<RemoteVideoTrackAdapters> video_tracks) {
 401   // Find removed tracks.
 402   auto audio_it = audio_track_observers_.begin();
 403   while (audio_it != audio_track_observers_.end()) {
 404     if (!IsTrackInVector(*audio_tracks.get(), (*audio_it)->id())) {
 405       (*audio_it)->Unregister();
 406        webkit_stream_.removeTrack(*(*audio_it)->webkit_track());
 407        audio_it = audio_track_observers_.erase(audio_it);
 408     } else {
 409       ++audio_it;
 410     }
 411   }
 412
 413   auto video_it = video_track_observers_.begin();
 414   while (video_it != video_track_observers_.end()) {
 415     if (!IsTrackInVector(*video_tracks.get(), (*video_it)->id())) {
 416       webkit_stream_.removeTrack(*(*video_it)->webkit_track());
 417       video_it = video_track_observers_.erase(video_it);
 418     } else {
 419       ++video_it;
 420     }
 421   }
 422
 423   // Find added tracks.
 424   for (auto& track : *audio_tracks.get()) {
 425     if (!IsTrackInVector(audio_track_observers_, track->id())) {
 426       track->Initialize();
 427       audio_track_observers_.push_back(track);
 428       webkit_stream_.addTrack(*track->webkit_track());
 429       // Set the track to null to avoid unregistering it below now that it's
 430       // been associated with a media stream.
 431       track = nullptr;
 432     }
 433   }
 434
 435   // Find added video tracks.
 436   for (const auto& track : *video_tracks.get()) {
 437     if (!IsTrackInVector(video_track_observers_, track->id())) {
 438       track->Initialize();
 439       video_track_observers_.push_back(track);
 440       webkit_stream_.addTrack(*track->webkit_track());
 441     }
 442   }
 443
 444   // Unregister all the audio track observers that were not used.
 445   // We need to do this before destruction since the observers can't unregister
 446   // from within the dtor due to a race.
 447   for (auto& track : *audio_tracks.get()) {
 448     if (track.get())
 449       track->Unregister();
 450   }
 451 }
 452
 453 }  // namespace content