dom/media/DOMMediaStream.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #include "DOMMediaStream.h"
   7
   8 #include "AudioCaptureTrack.h"
   9 #include "AudioChannelAgent.h"
  10 #include "AudioStreamTrack.h"
  11 #include "MediaTrackGraph.h"
  12 #include "MediaTrackGraphImpl.h"
  13 #include "MediaTrackListener.h"
  14 #include "Tracing.h"
  15 #include "VideoStreamTrack.h"
  16 #include "mozilla/dom/AudioTrack.h"
  17 #include "mozilla/dom/AudioTrackList.h"
  18 #include "mozilla/dom/DocGroup.h"
  19 #include "mozilla/dom/HTMLCanvasElement.h"
  20 #include "mozilla/dom/MediaStreamBinding.h"
  21 #include "mozilla/dom/MediaStreamTrackEvent.h"
  22 #include "mozilla/dom/Promise.h"
  23 #include "mozilla/dom/VideoTrack.h"
  24 #include "mozilla/dom/VideoTrackList.h"
  25 #include "mozilla/media/MediaUtils.h"
  26 #include "nsContentUtils.h"
  27 #include "nsGlobalWindowInner.h"
  28 #include "nsIUUIDGenerator.h"
  29 #include "nsPIDOMWindow.h"
  30 #include "nsProxyRelease.h"
  31 #include "nsRFPService.h"
  32 #include "nsServiceManagerUtils.h"
  33
  34 #ifdef LOG
  35 #  undef LOG
  36 #endif
  37
  38 using namespace mozilla;
  39 using namespace mozilla::dom;
  40 using namespace mozilla::layers;
  41 using namespace mozilla::media;
  42
  43 static LazyLogModule gMediaStreamLog("MediaStream");
  44 #define LOG(type, msg) MOZ_LOG(gMediaStreamLog, type, msg)
  45
  46 static bool ContainsLiveTracks(
  47     const nsTArray<RefPtr<MediaStreamTrack>>& aTracks) {
  48   for (const auto& track : aTracks) {
  49     if (track->ReadyState() == MediaStreamTrackState::Live) {
  50       return true;
  51     }
  52   }
  53
  54   return false;
  55 }
  56
  57 static bool ContainsLiveAudioTracks(
  58     const nsTArray<RefPtr<MediaStreamTrack>>& aTracks) {
  59   for (const auto& track : aTracks) {
  60     if (track->AsAudioStreamTrack() &&
  61         track->ReadyState() == MediaStreamTrackState::Live) {
  62       return true;
  63     }
  64   }
  65
  66   return false;
  67 }
  68
  69 class DOMMediaStream::PlaybackTrackListener : public MediaStreamTrackConsumer {
  70  public:
  71   NS_INLINE_DECL_REFCOUNTING(PlaybackTrackListener)
  72
  73   explicit PlaybackTrackListener(DOMMediaStream* aStream) : mStream(aStream) {}
  74
  75   void NotifyEnded(MediaStreamTrack* aTrack) override {
  76     if (!mStream) {
  77       return;
  78     }
  79
  80     if (!aTrack) {
  81       MOZ_ASSERT(false);
  82       return;
  83     }
  84
  85     MOZ_ASSERT(mStream->HasTrack(*aTrack));
  86     mStream->NotifyTrackRemoved(aTrack);
  87   }
  88
  89  protected:
  90   virtual ~PlaybackTrackListener() = default;
  91
  92   WeakPtr<DOMMediaStream> mStream;
  93 };
  94
  95 NS_IMPL_CYCLE_COLLECTION_CLASS(DOMMediaStream)
  96
  97 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(DOMMediaStream,
  98                                                 DOMEventTargetHelper)
  99   tmp->Destroy();
 100   NS_IMPL_CYCLE_COLLECTION_UNLINK(mTracks)
 101   NS_IMPL_CYCLE_COLLECTION_UNLINK(mConsumersToKeepAlive)
 102   NS_IMPL_CYCLE_COLLECTION_UNLINK(mTrackListeners)
 103   NS_IMPL_CYCLE_COLLECTION_UNLINK_WEAK_PTR
 104 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
 105
 106 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(DOMMediaStream,
 107                                                   DOMEventTargetHelper)
 108   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTracks)
 109   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mConsumersToKeepAlive)
 110   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTrackListeners)
 111 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
 112
 113 NS_IMPL_ADDREF_INHERITED(DOMMediaStream, DOMEventTargetHelper)
 114 NS_IMPL_RELEASE_INHERITED(DOMMediaStream, DOMEventTargetHelper)
 115
 116 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(DOMMediaStream)
 117   NS_INTERFACE_MAP_ENTRY_CONCRETE(DOMMediaStream)
 118 NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper)
 119
 120 NS_IMPL_CYCLE_COLLECTION(DOMMediaStream::TrackListener)
 121 NS_IMPL_CYCLE_COLLECTING_ADDREF(DOMMediaStream::TrackListener)
 122 NS_IMPL_CYCLE_COLLECTING_RELEASE(DOMMediaStream::TrackListener)
 123 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(DOMMediaStream::TrackListener)
 124   NS_INTERFACE_MAP_ENTRY(nsISupports)
 125 NS_INTERFACE_MAP_END
 126
 127 DOMMediaStream::DOMMediaStream(nsPIDOMWindowInner* aWindow)
 128     : DOMEventTargetHelper(aWindow),
 129       mPlaybackTrackListener(MakeAndAddRef<PlaybackTrackListener>(this)) {
 130   nsresult rv;
 131   nsCOMPtr<nsIUUIDGenerator> uuidgen =
 132       do_GetService("@mozilla.org/uuid-generator;1", &rv);
 133
 134   if (NS_SUCCEEDED(rv) && uuidgen) {
 135     nsID uuid;
 136     memset(&uuid, 0, sizeof(uuid));
 137     rv = uuidgen->GenerateUUIDInPlace(&uuid);
 138     if (NS_SUCCEEDED(rv)) {
 139       char buffer[NSID_LENGTH];
 140       uuid.ToProvidedString(buffer);
 141       mID = NS_ConvertASCIItoUTF16(buffer);
 142     }
 143   }
 144 }
 145
 146 DOMMediaStream::~DOMMediaStream() { Destroy(); }
 147
 148 void DOMMediaStream::Destroy() {
 149   LOG(LogLevel::Debug, ("DOMMediaStream %p Being destroyed.", this));
 150   for (const auto& track : mTracks) {
 151     // We must remove ourselves from each track's principal change observer list
 152     // before we die.
 153     if (!track->Ended()) {
 154       track->RemoveConsumer(mPlaybackTrackListener);
 155     }
 156   }
 157   mTrackListeners.Clear();
 158 }
 159
 160 JSObject* DOMMediaStream::WrapObject(JSContext* aCx,
 161                                      JS::Handle<JSObject*> aGivenProto) {
 162   return dom::MediaStream_Binding::Wrap(aCx, this, aGivenProto);
 163 }
 164
 165 /* static */
 166 already_AddRefed<DOMMediaStream> DOMMediaStream::Constructor(
 167     const GlobalObject& aGlobal, ErrorResult& aRv) {
 168   Sequence<OwningNonNull<MediaStreamTrack>> emptyTrackSeq;
 169   return Constructor(aGlobal, emptyTrackSeq, aRv);
 170 }
 171
 172 /* static */
 173 already_AddRefed<DOMMediaStream> DOMMediaStream::Constructor(
 174     const GlobalObject& aGlobal, const DOMMediaStream& aStream,
 175     ErrorResult& aRv) {
 176   nsTArray<RefPtr<MediaStreamTrack>> tracks;
 177   aStream.GetTracks(tracks);
 178
 179   Sequence<OwningNonNull<MediaStreamTrack>> nonNullTrackSeq;
 180   if (!nonNullTrackSeq.SetLength(tracks.Length(), fallible)) {
 181     MOZ_ASSERT(false);
 182     aRv.Throw(NS_ERROR_OUT_OF_MEMORY);
 183     return nullptr;
 184   }
 185
 186   for (size_t i = 0; i < tracks.Length(); ++i) {
 187     nonNullTrackSeq[i] = tracks[i];
 188   }
 189
 190   return Constructor(aGlobal, nonNullTrackSeq, aRv);
 191 }
 192
 193 /* static */
 194 already_AddRefed<DOMMediaStream> DOMMediaStream::Constructor(
 195     const GlobalObject& aGlobal,
 196     const Sequence<OwningNonNull<MediaStreamTrack>>& aTracks,
 197     ErrorResult& aRv) {
 198   nsCOMPtr<nsPIDOMWindowInner> ownerWindow =
 199       do_QueryInterface(aGlobal.GetAsSupports());
 200   if (!ownerWindow) {
 201     aRv.Throw(NS_ERROR_FAILURE);
 202     return nullptr;
 203   }
 204
 205   auto newStream = MakeRefPtr<DOMMediaStream>(ownerWindow);
 206   for (MediaStreamTrack& track : aTracks) {
 207     newStream->AddTrack(track);
 208   }
 209   return newStream.forget();
 210 }
 211
 212 already_AddRefed<Promise> DOMMediaStream::CountUnderlyingStreams(
 213     const GlobalObject& aGlobal, ErrorResult& aRv) {
 214   nsCOMPtr<nsPIDOMWindowInner> window =
 215       do_QueryInterface(aGlobal.GetAsSupports());
 216   if (!window) {
 217     aRv.Throw(NS_ERROR_UNEXPECTED);
 218     return nullptr;
 219   }
 220
 221   nsCOMPtr<nsIGlobalObject> go = do_QueryInterface(aGlobal.GetAsSupports());
 222   if (!go) {
 223     aRv.Throw(NS_ERROR_UNEXPECTED);
 224     return nullptr;
 225   }
 226
 227   RefPtr<Promise> p = Promise::Create(go, aRv);
 228   if (aRv.Failed()) {
 229     return nullptr;
 230   }
 231
 232   MediaTrackGraph* graph = MediaTrackGraph::GetInstanceIfExists(
 233       window, MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
 234       MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
 235   if (!graph) {
 236     p->MaybeResolve(0);
 237     return p.forget();
 238   }
 239
 240   auto* graphImpl = static_cast<MediaTrackGraphImpl*>(graph);
 241
 242   class Counter : public ControlMessage {
 243    public:
 244     Counter(MediaTrackGraphImpl* aGraph, const RefPtr<Promise>& aPromise)
 245         : ControlMessage(nullptr), mGraph(aGraph), mPromise(aPromise) {
 246       MOZ_ASSERT(NS_IsMainThread());
 247     }
 248
 249     void Run() override {
 250       TRACE("DOMMediaStream::Counter")
 251       uint32_t streams =
 252           mGraph->mTracks.Length() + mGraph->mSuspendedTracks.Length();
 253       mGraph->DispatchToMainThreadStableState(NS_NewRunnableFunction(
 254           "DOMMediaStream::CountUnderlyingStreams (stable state)",
 255           [promise = std::move(mPromise), streams]() mutable {
 256             NS_DispatchToMainThread(NS_NewRunnableFunction(
 257                 "DOMMediaStream::CountUnderlyingStreams",
 258                 [promise = std::move(promise), streams]() {
 259                   promise->MaybeResolve(streams);
 260                 }));
 261           }));
 262     }
 263
 264     // mPromise can only be AddRefed/Released on main thread.
 265     // In case of shutdown, Run() does not run, so we dispatch mPromise to be
 266     // released on main thread here.
 267     void RunDuringShutdown() override {
 268       NS_ReleaseOnMainThread(
 269           "DOMMediaStream::CountUnderlyingStreams::Counter::RunDuringShutdown",
 270           mPromise.forget());
 271     }
 272
 273    private:
 274     // mGraph owns this Counter instance and decides its lifetime.
 275     MediaTrackGraphImpl* mGraph;
 276     RefPtr<Promise> mPromise;
 277   };
 278   graphImpl->AppendMessage(MakeUnique<Counter>(graphImpl, p));
 279
 280   return p.forget();
 281 }
 282
 283 void DOMMediaStream::GetId(nsAString& aID) const { aID = mID; }
 284
 285 void DOMMediaStream::GetAudioTracks(
 286     nsTArray<RefPtr<AudioStreamTrack>>& aTracks) const {
 287   for (const auto& track : mTracks) {
 288     if (AudioStreamTrack* t = track->AsAudioStreamTrack()) {
 289       aTracks.AppendElement(t);
 290     }
 291   }
 292 }
 293
 294 void DOMMediaStream::GetAudioTracks(
 295     nsTArray<RefPtr<MediaStreamTrack>>& aTracks) const {
 296   for (const auto& track : mTracks) {
 297     if (track->AsAudioStreamTrack()) {
 298       aTracks.AppendElement(track);
 299     }
 300   }
 301 }
 302
 303 void DOMMediaStream::GetVideoTracks(
 304     nsTArray<RefPtr<VideoStreamTrack>>& aTracks) const {
 305   for (const auto& track : mTracks) {
 306     if (VideoStreamTrack* t = track->AsVideoStreamTrack()) {
 307       aTracks.AppendElement(t);
 308     }
 309   }
 310 }
 311
 312 void DOMMediaStream::GetVideoTracks(
 313     nsTArray<RefPtr<MediaStreamTrack>>& aTracks) const {
 314   for (const auto& track : mTracks) {
 315     if (track->AsVideoStreamTrack()) {
 316       aTracks.AppendElement(track);
 317     }
 318   }
 319 }
 320
 321 void DOMMediaStream::GetTracks(
 322     nsTArray<RefPtr<MediaStreamTrack>>& aTracks) const {
 323   for (const auto& track : mTracks) {
 324     aTracks.AppendElement(track);
 325   }
 326 }
 327
 328 void DOMMediaStream::AddTrack(MediaStreamTrack& aTrack) {
 329   LOG(LogLevel::Info, ("DOMMediaStream %p Adding track %p (from track %p)",
 330                        this, &aTrack, aTrack.GetTrack()));
 331
 332   if (HasTrack(aTrack)) {
 333     LOG(LogLevel::Debug,
 334         ("DOMMediaStream %p already contains track %p", this, &aTrack));
 335     return;
 336   }
 337
 338   mTracks.AppendElement(&aTrack);
 339
 340   if (!aTrack.Ended()) {
 341     NotifyTrackAdded(&aTrack);
 342   }
 343 }
 344
 345 void DOMMediaStream::RemoveTrack(MediaStreamTrack& aTrack) {
 346   if (static_cast<LogModule*>(gMediaStreamLog)->ShouldLog(LogLevel::Info)) {
 347     if (aTrack.Ended()) {
 348       LOG(LogLevel::Info,
 349           ("DOMMediaStream %p Removing (ended) track %p", this, &aTrack));
 350     } else {
 351       LOG(LogLevel::Info,
 352           ("DOMMediaStream %p Removing track %p (from track %p)", this, &aTrack,
 353            aTrack.GetTrack()));
 354     }
 355   }
 356
 357   if (!mTracks.RemoveElement(&aTrack)) {
 358     LOG(LogLevel::Debug,
 359         ("DOMMediaStream %p does not contain track %p", this, &aTrack));
 360     return;
 361   }
 362
 363   if (!aTrack.Ended()) {
 364     NotifyTrackRemoved(&aTrack);
 365   }
 366 }
 367
 368 already_AddRefed<DOMMediaStream> DOMMediaStream::Clone() {
 369   auto newStream = MakeRefPtr<DOMMediaStream>(GetOwnerWindow());
 370
 371   LOG(LogLevel::Info,
 372       ("DOMMediaStream %p created clone %p", this, newStream.get()));
 373
 374   for (const auto& track : mTracks) {
 375     LOG(LogLevel::Debug,
 376         ("DOMMediaStream %p forwarding external track %p to clone %p", this,
 377          track.get(), newStream.get()));
 378     RefPtr<MediaStreamTrack> clone = track->Clone();
 379     newStream->AddTrack(*clone);
 380   }
 381
 382   return newStream.forget();
 383 }
 384
 385 bool DOMMediaStream::Active() const { return mActive; }
 386 bool DOMMediaStream::Audible() const { return mAudible; }
 387
 388 MediaStreamTrack* DOMMediaStream::GetTrackById(const nsAString& aId) const {
 389   for (const auto& track : mTracks) {
 390     nsString id;
 391     track->GetId(id);
 392     if (id == aId) {
 393       return track;
 394     }
 395   }
 396   return nullptr;
 397 }
 398
 399 bool DOMMediaStream::HasTrack(const MediaStreamTrack& aTrack) const {
 400   return mTracks.Contains(&aTrack);
 401 }
 402
 403 void DOMMediaStream::AddTrackInternal(MediaStreamTrack* aTrack) {
 404   LOG(LogLevel::Debug,
 405       ("DOMMediaStream %p Adding owned track %p", this, aTrack));
 406   AddTrack(*aTrack);
 407   DispatchTrackEvent(u"addtrack"_ns, aTrack);
 408 }
 409
 410 void DOMMediaStream::RemoveTrackInternal(MediaStreamTrack* aTrack) {
 411   LOG(LogLevel::Debug,
 412       ("DOMMediaStream %p Removing owned track %p", this, aTrack));
 413   if (!HasTrack(*aTrack)) {
 414     return;
 415   }
 416   RemoveTrack(*aTrack);
 417   DispatchTrackEvent(u"removetrack"_ns, aTrack);
 418 }
 419
 420 already_AddRefed<nsIPrincipal> DOMMediaStream::GetPrincipal() {
 421   nsGlobalWindowInner* win = GetOwnerWindow();
 422   if (!win) {
 423     return nullptr;
 424   }
 425   nsCOMPtr<nsIPrincipal> principal = win->GetPrincipal();
 426   for (const auto& t : mTracks) {
 427     if (t->Ended()) {
 428       continue;
 429     }
 430     nsContentUtils::CombineResourcePrincipals(&principal, t->GetPrincipal());
 431   }
 432   return principal.forget();
 433 }
 434
 435 void DOMMediaStream::NotifyActive() {
 436   LOG(LogLevel::Info, ("DOMMediaStream %p NotifyActive(). ", this));
 437
 438   MOZ_ASSERT(mActive);
 439   for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
 440     mTrackListeners[i]->NotifyActive();
 441   }
 442 }
 443
 444 void DOMMediaStream::NotifyInactive() {
 445   LOG(LogLevel::Info, ("DOMMediaStream %p NotifyInactive(). ", this));
 446
 447   MOZ_ASSERT(!mActive);
 448   for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
 449     mTrackListeners[i]->NotifyInactive();
 450   }
 451 }
 452
 453 void DOMMediaStream::NotifyAudible() {
 454   LOG(LogLevel::Info, ("DOMMediaStream %p NotifyAudible(). ", this));
 455
 456   MOZ_ASSERT(mAudible);
 457   for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
 458     mTrackListeners[i]->NotifyAudible();
 459   }
 460 }
 461
 462 void DOMMediaStream::NotifyInaudible() {
 463   LOG(LogLevel::Info, ("DOMMediaStream %p NotifyInaudible(). ", this));
 464
 465   MOZ_ASSERT(!mAudible);
 466   for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
 467     mTrackListeners[i]->NotifyInaudible();
 468   }
 469 }
 470
 471 void DOMMediaStream::RegisterTrackListener(TrackListener* aListener) {
 472   MOZ_ASSERT(NS_IsMainThread());
 473
 474   mTrackListeners.AppendElement(aListener);
 475 }
 476
 477 void DOMMediaStream::UnregisterTrackListener(TrackListener* aListener) {
 478   MOZ_ASSERT(NS_IsMainThread());
 479   mTrackListeners.RemoveElement(aListener);
 480 }
 481
 482 void DOMMediaStream::NotifyTrackAdded(const RefPtr<MediaStreamTrack>& aTrack) {
 483   MOZ_ASSERT(NS_IsMainThread());
 484
 485   aTrack->AddConsumer(mPlaybackTrackListener);
 486
 487   for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
 488     mTrackListeners[i]->NotifyTrackAdded(aTrack);
 489   }
 490
 491   if (!mActive) {
 492     // Check if we became active.
 493     if (ContainsLiveTracks(mTracks)) {
 494       mActive = true;
 495       NotifyActive();
 496     }
 497   }
 498
 499   if (!mAudible) {
 500     // Check if we became audible.
 501     if (ContainsLiveAudioTracks(mTracks)) {
 502       mAudible = true;
 503       NotifyAudible();
 504     }
 505   }
 506 }
 507
 508 void DOMMediaStream::NotifyTrackRemoved(
 509     const RefPtr<MediaStreamTrack>& aTrack) {
 510   MOZ_ASSERT(NS_IsMainThread());
 511
 512   if (aTrack) {
 513     // aTrack may be null to allow HTMLMediaElement::MozCaptureStream streams
 514     // to be played until the source media element has ended. The source media
 515     // element will then call NotifyTrackRemoved(nullptr) to signal that we can
 516     // go inactive, regardless of the timing of the last track ending.
 517
 518     aTrack->RemoveConsumer(mPlaybackTrackListener);
 519
 520     for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
 521       mTrackListeners[i]->NotifyTrackRemoved(aTrack);
 522     }
 523
 524     if (!mActive) {
 525       NS_ASSERTION(false, "Shouldn't remove a live track if already inactive");
 526       return;
 527     }
 528   }
 529
 530   if (mAudible) {
 531     // Check if we became inaudible.
 532     if (!ContainsLiveAudioTracks(mTracks)) {
 533       mAudible = false;
 534       NotifyInaudible();
 535     }
 536   }
 537
 538   // Check if we became inactive.
 539   if (!ContainsLiveTracks(mTracks)) {
 540     mActive = false;
 541     NotifyInactive();
 542   }
 543 }
 544
 545 nsresult DOMMediaStream::DispatchTrackEvent(
 546     const nsAString& aName, const RefPtr<MediaStreamTrack>& aTrack) {
 547   MediaStreamTrackEventInit init;
 548   init.mTrack = aTrack;
 549
 550   RefPtr<MediaStreamTrackEvent> event =
 551       MediaStreamTrackEvent::Constructor(this, aName, init);
 552
 553   return DispatchTrustedEvent(event);
 554 }