Backed out changeset f594e6f00208 (bug 1940883) for causing crashes in bug 1941164.
[gecko.git] / dom / media / MediaTrackGraph.cpp
blobf3c92b74c4a198b5fed59fe94ffa13afe03b502e
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #include "MediaTrackGraphImpl.h"
7 #include "mozilla/MathAlgorithms.h"
8 #include "mozilla/Unused.h"
10 #include "AudioSegment.h"
11 #include "CrossGraphPort.h"
12 #include "VideoSegment.h"
13 #include "nsContentUtils.h"
14 #include "nsGlobalWindowInner.h"
15 #include "nsPrintfCString.h"
16 #include "nsServiceManagerUtils.h"
17 #include "prerror.h"
18 #include "mozilla/Logging.h"
19 #include "mozilla/Attributes.h"
20 #include "ForwardedInputTrack.h"
21 #include "ImageContainer.h"
22 #include "AudioCaptureTrack.h"
23 #include "AudioDeviceInfo.h"
24 #include "AudioNodeTrack.h"
25 #include "AudioNodeExternalInputTrack.h"
26 #if defined(MOZ_WEBRTC)
27 # include "MediaEngineWebRTCAudio.h"
28 #endif // MOZ_WEBRTC
29 #include "MediaTrackListener.h"
30 #include "mozilla/dom/BaseAudioContextBinding.h"
31 #include "mozilla/dom/Document.h"
32 #include "mozilla/dom/WorkletThread.h"
33 #include "mozilla/media/MediaUtils.h"
34 #include <algorithm>
35 #include "GeckoProfiler.h"
36 #include "VideoFrameContainer.h"
37 #include "mozilla/AbstractThread.h"
38 #include "mozilla/StaticPrefs_dom.h"
39 #include "mozilla/StaticPrefs_media.h"
40 #include "transport/runnable_utils.h"
41 #include "VideoUtils.h"
42 #include "GraphRunner.h"
43 #include "Tracing.h"
44 #include "UnderrunHandler.h"
45 #include "mozilla/CycleCollectedJSRuntime.h"
46 #include "mozilla/Preferences.h"
48 #include "webaudio/blink/DenormalDisabler.h"
49 #include "webaudio/blink/HRTFDatabaseLoader.h"
51 using namespace mozilla::layers;
52 using namespace mozilla::dom;
53 using namespace mozilla::gfx;
54 using namespace mozilla::media;
56 namespace mozilla {
58 using AudioDeviceID = CubebUtils::AudioDeviceID;
59 using IsInShutdown = MediaTrack::IsInShutdown;
61 LazyLogModule gMediaTrackGraphLog("MediaTrackGraph");
62 #ifdef LOG
63 # undef LOG
64 #endif // LOG
65 #define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg)
67 NativeInputTrack* DeviceInputTrackManager::GetNativeInputTrack() {
68 return mNativeInputTrack.get();
71 DeviceInputTrack* DeviceInputTrackManager::GetDeviceInputTrack(
72 CubebUtils::AudioDeviceID aID) {
73 if (mNativeInputTrack && mNativeInputTrack->mDeviceId == aID) {
74 return mNativeInputTrack.get();
76 for (const RefPtr<NonNativeInputTrack>& t : mNonNativeInputTracks) {
77 if (t->mDeviceId == aID) {
78 return t.get();
81 return nullptr;
84 NonNativeInputTrack* DeviceInputTrackManager::GetFirstNonNativeInputTrack() {
85 if (mNonNativeInputTracks.IsEmpty()) {
86 return nullptr;
88 return mNonNativeInputTracks[0].get();
91 void DeviceInputTrackManager::Add(DeviceInputTrack* aTrack) {
92 if (NativeInputTrack* native = aTrack->AsNativeInputTrack()) {
93 MOZ_ASSERT(!mNativeInputTrack);
94 mNativeInputTrack = native;
95 } else {
96 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
97 MOZ_ASSERT(nonNative);
98 struct DeviceTrackComparator {
99 public:
100 bool Equals(const RefPtr<NonNativeInputTrack>& aTrack,
101 CubebUtils::AudioDeviceID aDeviceId) const {
102 return aTrack->mDeviceId == aDeviceId;
105 MOZ_ASSERT(!mNonNativeInputTracks.Contains(aTrack->mDeviceId,
106 DeviceTrackComparator()));
107 mNonNativeInputTracks.AppendElement(nonNative);
111 void DeviceInputTrackManager::Remove(DeviceInputTrack* aTrack) {
112 if (aTrack->AsNativeInputTrack()) {
113 MOZ_ASSERT(mNativeInputTrack);
114 MOZ_ASSERT(mNativeInputTrack.get() == aTrack->AsNativeInputTrack());
115 mNativeInputTrack = nullptr;
116 } else {
117 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
118 MOZ_ASSERT(nonNative);
119 DebugOnly<bool> removed = mNonNativeInputTracks.RemoveElement(nonNative);
120 MOZ_ASSERT(removed);
125 * A hash table containing the graph instances, one per Window ID,
126 * sample rate, and device ID combination.
129 struct MediaTrackGraphImpl::Lookup final {
130 HashNumber Hash() const {
131 return HashGeneric(mWindowID, mSampleRate, mOutputDeviceID);
133 const uint64_t mWindowID;
134 const TrackRate mSampleRate;
135 const CubebUtils::AudioDeviceID mOutputDeviceID;
138 // Implicit to support GraphHashSet.lookup(*graph).
139 MOZ_IMPLICIT MediaTrackGraphImpl::operator MediaTrackGraphImpl::Lookup() const {
140 return {mWindowID, mSampleRate, PrimaryOutputDeviceID()};
143 namespace {
144 struct GraphHasher { // for HashSet
145 using Lookup = const MediaTrackGraphImpl::Lookup;
147 static HashNumber hash(const Lookup& aLookup) { return aLookup.Hash(); }
149 static bool match(const MediaTrackGraphImpl* aGraph, const Lookup& aLookup) {
150 return aGraph->mWindowID == aLookup.mWindowID &&
151 aGraph->GraphRate() == aLookup.mSampleRate &&
152 aGraph->PrimaryOutputDeviceID() == aLookup.mOutputDeviceID;
156 // The weak reference to the graph is removed when its last track is removed.
157 using GraphHashSet =
158 HashSet<MediaTrackGraphImpl*, GraphHasher, InfallibleAllocPolicy>;
159 GraphHashSet* Graphs() {
160 MOZ_ASSERT(NS_IsMainThread());
161 static GraphHashSet sGraphs(4); // 4 is minimum HashSet capacity
162 return &sGraphs;
164 } // anonymous namespace
166 static void ApplyTrackDisabling(DisabledTrackMode aDisabledMode,
167 MediaSegment* aSegment,
168 MediaSegment* aRawSegment) {
169 if (aDisabledMode == DisabledTrackMode::ENABLED) {
170 return;
172 if (aDisabledMode == DisabledTrackMode::SILENCE_BLACK) {
173 aSegment->ReplaceWithDisabled();
174 if (aRawSegment) {
175 aRawSegment->ReplaceWithDisabled();
177 } else if (aDisabledMode == DisabledTrackMode::SILENCE_FREEZE) {
178 aSegment->ReplaceWithNull();
179 if (aRawSegment) {
180 aRawSegment->ReplaceWithNull();
182 } else {
183 MOZ_CRASH("Unsupported mode");
187 MediaTrackGraphImpl::~MediaTrackGraphImpl() {
188 MOZ_ASSERT(mTracks.IsEmpty() && mSuspendedTracks.IsEmpty(),
189 "All tracks should have been destroyed by messages from the main "
190 "thread");
191 LOG(LogLevel::Debug, ("MediaTrackGraph %p destroyed", this));
192 LOG(LogLevel::Debug, ("MediaTrackGraphImpl::~MediaTrackGraphImpl"));
195 void MediaTrackGraphImpl::AddTrackGraphThread(MediaTrack* aTrack) {
196 MOZ_ASSERT(OnGraphThreadOrNotRunning());
197 aTrack->mStartTime = mProcessedTime;
199 if (aTrack->IsSuspended()) {
200 mSuspendedTracks.AppendElement(aTrack);
201 LOG(LogLevel::Debug,
202 ("%p: Adding media track %p, in the suspended track array", this,
203 aTrack));
204 } else {
205 mTracks.AppendElement(aTrack);
206 LOG(LogLevel::Debug, ("%p: Adding media track %p, count %zu", this, aTrack,
207 mTracks.Length()));
210 SetTrackOrderDirty();
213 void MediaTrackGraphImpl::RemoveTrackGraphThread(MediaTrack* aTrack) {
214 MOZ_ASSERT(OnGraphThreadOrNotRunning());
215 // Remove references in mTrackUpdates before we allow aTrack to die.
216 // Pending updates are not needed (since the main thread has already given
217 // up the track) so we will just drop them.
219 MonitorAutoLock lock(mMonitor);
220 for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) {
221 if (mTrackUpdates[i].mTrack == aTrack) {
222 mTrackUpdates[i].mTrack = nullptr;
227 // Ensure that mFirstCycleBreaker is updated when necessary.
228 SetTrackOrderDirty();
230 UnregisterAllAudioOutputs(aTrack);
232 if (aTrack->IsSuspended()) {
233 mSuspendedTracks.RemoveElement(aTrack);
234 } else {
235 mTracks.RemoveElement(aTrack);
238 LOG(LogLevel::Debug, ("%p: Removed media track %p, count %zu", this, aTrack,
239 mTracks.Length()));
241 NS_RELEASE(aTrack); // probably destroying it
244 TrackTime MediaTrackGraphImpl::GraphTimeToTrackTimeWithBlocking(
245 const MediaTrack* aTrack, GraphTime aTime) const {
246 MOZ_ASSERT(
247 aTime <= mStateComputedTime,
248 "Don't ask about times where we haven't made blocking decisions yet");
249 return std::max<TrackTime>(
250 0, std::min(aTime, aTrack->mStartBlocking) - aTrack->mStartTime);
253 void MediaTrackGraphImpl::UpdateCurrentTimeForTracks(
254 GraphTime aPrevCurrentTime) {
255 MOZ_ASSERT(OnGraphThread());
256 for (MediaTrack* track : AllTracks()) {
257 // Shouldn't have already notified of ended *and* have output!
258 MOZ_ASSERT_IF(track->mStartBlocking > aPrevCurrentTime,
259 !track->mNotifiedEnded);
261 // Calculate blocked time and fire Blocked/Unblocked events
262 GraphTime blockedTime = mStateComputedTime - track->mStartBlocking;
263 NS_ASSERTION(blockedTime >= 0, "Error in blocking time");
264 track->AdvanceTimeVaryingValuesToCurrentTime(mStateComputedTime,
265 blockedTime);
266 LOG(LogLevel::Verbose,
267 ("%p: MediaTrack %p bufferStartTime=%f blockedTime=%f", this, track,
268 MediaTimeToSeconds(track->mStartTime),
269 MediaTimeToSeconds(blockedTime)));
270 track->mStartBlocking = mStateComputedTime;
272 TrackTime trackCurrentTime =
273 track->GraphTimeToTrackTime(mStateComputedTime);
274 if (track->mEnded) {
275 MOZ_ASSERT(track->GetEnd() <= trackCurrentTime);
276 if (!track->mNotifiedEnded) {
277 // Playout of this track ended and listeners have not been notified.
278 track->mNotifiedEnded = true;
279 SetTrackOrderDirty();
280 for (const auto& listener : track->mTrackListeners) {
281 listener->NotifyOutput(this, track->GetEnd());
282 listener->NotifyEnded(this);
285 } else {
286 for (const auto& listener : track->mTrackListeners) {
287 listener->NotifyOutput(this, trackCurrentTime);
293 template <typename C, typename Chunk>
294 void MediaTrackGraphImpl::ProcessChunkMetadataForInterval(MediaTrack* aTrack,
295 C& aSegment,
296 TrackTime aStart,
297 TrackTime aEnd) {
298 MOZ_ASSERT(OnGraphThreadOrNotRunning());
299 MOZ_ASSERT(aTrack);
301 TrackTime offset = 0;
302 for (typename C::ConstChunkIterator chunk(aSegment); !chunk.IsEnded();
303 chunk.Next()) {
304 if (offset >= aEnd) {
305 break;
307 offset += chunk->GetDuration();
308 if (chunk->IsNull() || offset < aStart) {
309 continue;
311 const PrincipalHandle& principalHandle = chunk->GetPrincipalHandle();
312 if (principalHandle != aSegment.GetLastPrincipalHandle()) {
313 aSegment.SetLastPrincipalHandle(principalHandle);
314 LOG(LogLevel::Debug,
315 ("%p: MediaTrack %p, principalHandle "
316 "changed in %sChunk with duration %lld",
317 this, aTrack,
318 aSegment.GetType() == MediaSegment::AUDIO ? "Audio" : "Video",
319 (long long)chunk->GetDuration()));
320 for (const auto& listener : aTrack->mTrackListeners) {
321 listener->NotifyPrincipalHandleChanged(this, principalHandle);
327 void MediaTrackGraphImpl::ProcessChunkMetadata(GraphTime aPrevCurrentTime) {
328 MOZ_ASSERT(OnGraphThreadOrNotRunning());
329 for (MediaTrack* track : AllTracks()) {
330 TrackTime iterationStart = track->GraphTimeToTrackTime(aPrevCurrentTime);
331 TrackTime iterationEnd = track->GraphTimeToTrackTime(mProcessedTime);
332 if (!track->mSegment) {
333 continue;
335 if (track->mType == MediaSegment::AUDIO) {
336 ProcessChunkMetadataForInterval<AudioSegment, AudioChunk>(
337 track, *track->GetData<AudioSegment>(), iterationStart, iterationEnd);
338 } else if (track->mType == MediaSegment::VIDEO) {
339 ProcessChunkMetadataForInterval<VideoSegment, VideoChunk>(
340 track, *track->GetData<VideoSegment>(), iterationStart, iterationEnd);
341 } else {
342 MOZ_CRASH("Unknown track type");
347 GraphTime MediaTrackGraphImpl::WillUnderrun(MediaTrack* aTrack,
348 GraphTime aEndBlockingDecisions) {
349 // Ended tracks can't underrun. ProcessedMediaTracks also can't cause
350 // underrun currently, since we'll always be able to produce data for them
351 // unless they block on some other track.
352 if (aTrack->mEnded || aTrack->AsProcessedTrack()) {
353 return aEndBlockingDecisions;
355 // This track isn't ended or suspended. We don't need to call
356 // TrackTimeToGraphTime since an underrun is the only thing that can block
357 // it.
358 GraphTime bufferEnd = aTrack->GetEnd() + aTrack->mStartTime;
359 #ifdef DEBUG
360 if (bufferEnd < mProcessedTime) {
361 LOG(LogLevel::Error, ("%p: MediaTrack %p underrun, "
362 "bufferEnd %f < mProcessedTime %f (%" PRId64
363 " < %" PRId64 "), TrackTime %" PRId64,
364 this, aTrack, MediaTimeToSeconds(bufferEnd),
365 MediaTimeToSeconds(mProcessedTime), bufferEnd,
366 mProcessedTime, aTrack->GetEnd()));
367 NS_ASSERTION(bufferEnd >= mProcessedTime, "Buffer underran");
369 #endif
370 return std::min(bufferEnd, aEndBlockingDecisions);
373 namespace {
374 // Value of mCycleMarker for unvisited tracks in cycle detection.
375 const uint32_t NOT_VISITED = UINT32_MAX;
376 // Value of mCycleMarker for ordered tracks in muted cycles.
377 const uint32_t IN_MUTED_CYCLE = 1;
378 } // namespace
380 bool MediaTrackGraphImpl::AudioTrackPresent() {
381 MOZ_ASSERT(OnGraphThreadOrNotRunning());
383 bool audioTrackPresent = false;
384 for (MediaTrack* track : mTracks) {
385 if (track->AsAudioNodeTrack()) {
386 audioTrackPresent = true;
387 break;
390 if (track->mType == MediaSegment::AUDIO && !track->mNotifiedEnded) {
391 audioTrackPresent = true;
392 break;
396 // We may not have audio input device when we only have AudioNodeTracks. But
397 // if audioTrackPresent is false, we must have no input device.
398 MOZ_DIAGNOSTIC_ASSERT_IF(
399 !audioTrackPresent,
400 !mDeviceInputTrackManagerGraphThread.GetNativeInputTrack());
402 return audioTrackPresent;
405 void MediaTrackGraphImpl::CheckDriver() {
406 MOZ_ASSERT(OnGraphThread());
407 // An offline graph has only one driver.
408 // Otherwise, if a switch is already pending, let that happen.
409 if (!mRealtime || Switching()) {
410 return;
413 AudioCallbackDriver* audioCallbackDriver =
414 CurrentDriver()->AsAudioCallbackDriver();
415 if (audioCallbackDriver && !audioCallbackDriver->OnFallback()) {
416 for (PendingResumeOperation& op : mPendingResumeOperations) {
417 op.Apply(this);
419 mPendingResumeOperations.Clear();
422 // Note that this looks for any audio tracks, input or output, and switches
423 // to a SystemClockDriver if there are none active or no resume operations
424 // to make any active.
425 bool needAudioCallbackDriver =
426 !mPendingResumeOperations.IsEmpty() || AudioTrackPresent();
427 if (!needAudioCallbackDriver) {
428 if (audioCallbackDriver && audioCallbackDriver->IsStarted()) {
429 SwitchAtNextIteration(
430 new SystemClockDriver(this, CurrentDriver(), mSampleRate));
432 return;
435 NativeInputTrack* native =
436 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
437 CubebUtils::AudioDeviceID inputDevice = native ? native->mDeviceId : nullptr;
438 uint32_t inputChannelCount = AudioInputChannelCount(inputDevice);
439 AudioInputType inputPreference = AudioInputDevicePreference(inputDevice);
440 Maybe<AudioInputProcessingParamsRequest> processingRequest =
441 ToMaybeRef(native).map([](auto& native) {
442 return native.UpdateRequestedProcessingParams();
445 uint32_t primaryOutputChannelCount = PrimaryOutputChannelCount();
446 if (!audioCallbackDriver) {
447 if (primaryOutputChannelCount > 0) {
448 AudioCallbackDriver* driver = new AudioCallbackDriver(
449 this, CurrentDriver(), mSampleRate, primaryOutputChannelCount,
450 inputChannelCount, PrimaryOutputDeviceID(), inputDevice,
451 inputPreference, processingRequest);
452 SwitchAtNextIteration(driver);
454 return;
457 bool needInputProcessingParamUpdate =
458 processingRequest &&
459 processingRequest->mGeneration !=
460 audioCallbackDriver->RequestedInputProcessingParams().mGeneration;
462 // Check if this graph should switch to a different number of output channels.
463 // Generally, a driver switch is explicitly made by an event (e.g., setting
464 // the AudioDestinationNode channelCount), but if an HTMLMediaElement is
465 // directly playing back via another HTMLMediaElement, the number of channels
466 // of the media determines how many channels to output, and it can change
467 // dynamically.
468 if (primaryOutputChannelCount != audioCallbackDriver->OutputChannelCount()) {
469 if (needInputProcessingParamUpdate) {
470 needInputProcessingParamUpdate = false;
472 AudioCallbackDriver* driver = new AudioCallbackDriver(
473 this, CurrentDriver(), mSampleRate, primaryOutputChannelCount,
474 inputChannelCount, PrimaryOutputDeviceID(), inputDevice,
475 inputPreference, processingRequest);
476 SwitchAtNextIteration(driver);
479 if (needInputProcessingParamUpdate) {
480 needInputProcessingParamUpdate = false;
481 LOG(LogLevel::Debug,
482 ("%p: Setting on the fly requested processing params %s (Gen %d)", this,
483 CubebUtils::ProcessingParamsToString(processingRequest->mParams).get(),
484 processingRequest->mGeneration));
485 audioCallbackDriver->RequestInputProcessingParams(*processingRequest);
489 void MediaTrackGraphImpl::UpdateTrackOrder() {
490 if (!mTrackOrderDirty) {
491 return;
494 mTrackOrderDirty = false;
496 // The algorithm for finding cycles is based on Tim Leslie's iterative
497 // implementation [1][2] of Pearce's variant [3] of Tarjan's strongly
498 // connected components (SCC) algorithm. There are variations (a) to
499 // distinguish whether tracks in SCCs of size 1 are in a cycle and (b) to
500 // re-run the algorithm over SCCs with breaks at DelayNodes.
502 // [1] http://www.timl.id.au/?p=327
503 // [2]
504 // https://github.com/scipy/scipy/blob/e2c502fca/scipy/sparse/csgraph/_traversal.pyx#L582
505 // [3] http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.102.1707
507 // There are two stacks. One for the depth-first search (DFS),
508 mozilla::LinkedList<MediaTrack> dfsStack;
509 // and another for tracks popped from the DFS stack, but still being
510 // considered as part of SCCs involving tracks on the stack.
511 mozilla::LinkedList<MediaTrack> sccStack;
513 // An index into mTracks for the next track found with no unsatisfied
514 // upstream dependencies.
515 uint32_t orderedTrackCount = 0;
517 for (uint32_t i = 0; i < mTracks.Length(); ++i) {
518 MediaTrack* t = mTracks[i];
519 ProcessedMediaTrack* pt = t->AsProcessedTrack();
520 if (pt) {
521 // The dfsStack initially contains a list of all processed tracks in
522 // unchanged order.
523 dfsStack.insertBack(t);
524 pt->mCycleMarker = NOT_VISITED;
525 } else {
526 // SourceMediaTracks have no inputs and so can be ordered now.
527 mTracks[orderedTrackCount] = t;
528 ++orderedTrackCount;
532 // mNextStackMarker corresponds to "index" in Tarjan's algorithm. It is a
533 // counter to label mCycleMarker on the next visited track in the DFS
534 // uniquely in the set of visited tracks that are still being considered.
536 // In this implementation, the counter descends so that the values are
537 // strictly greater than the values that mCycleMarker takes when the track
538 // has been ordered (0 or IN_MUTED_CYCLE).
540 // Each new track labelled, as the DFS searches upstream, receives a value
541 // less than those used for all other tracks being considered.
542 uint32_t nextStackMarker = NOT_VISITED - 1;
543 // Reset list of DelayNodes in cycles stored at the tail of mTracks.
544 mFirstCycleBreaker = mTracks.Length();
546 // Rearrange dfsStack order as required to DFS upstream and pop tracks
547 // in processing order to place in mTracks.
548 while (auto pt = static_cast<ProcessedMediaTrack*>(dfsStack.getFirst())) {
549 const auto& inputs = pt->mInputs;
550 MOZ_ASSERT(pt->AsProcessedTrack());
551 if (pt->mCycleMarker == NOT_VISITED) {
552 // Record the position on the visited stack, so that any searches
553 // finding this track again know how much of the stack is in the cycle.
554 pt->mCycleMarker = nextStackMarker;
555 --nextStackMarker;
556 // Not-visited input tracks should be processed first.
557 // SourceMediaTracks have already been ordered.
558 for (uint32_t i = inputs.Length(); i--;) {
559 if (inputs[i]->GetSource()->IsSuspended()) {
560 continue;
562 auto input = inputs[i]->GetSource()->AsProcessedTrack();
563 if (input && input->mCycleMarker == NOT_VISITED) {
564 // It can be that this track has an input which is from a suspended
565 // AudioContext.
566 if (input->isInList()) {
567 input->remove();
568 dfsStack.insertFront(input);
572 continue;
575 // Returning from DFS. Pop from dfsStack.
576 pt->remove();
578 // cycleStackMarker keeps track of the highest marker value on any
579 // upstream track, if any, found receiving input, directly or indirectly,
580 // from the visited stack (and so from |ps|, making a cycle). In a
581 // variation from Tarjan's SCC algorithm, this does not include |ps|
582 // unless it is part of the cycle.
583 uint32_t cycleStackMarker = 0;
584 for (uint32_t i = inputs.Length(); i--;) {
585 if (inputs[i]->GetSource()->IsSuspended()) {
586 continue;
588 auto input = inputs[i]->GetSource()->AsProcessedTrack();
589 if (input) {
590 cycleStackMarker = std::max(cycleStackMarker, input->mCycleMarker);
594 if (cycleStackMarker <= IN_MUTED_CYCLE) {
595 // All inputs have been ordered and their stack markers have been removed.
596 // This track is not part of a cycle. It can be processed next.
597 pt->mCycleMarker = 0;
598 mTracks[orderedTrackCount] = pt;
599 ++orderedTrackCount;
600 continue;
603 // A cycle has been found. Record this track for ordering when all
604 // tracks in this SCC have been popped from the DFS stack.
605 sccStack.insertFront(pt);
607 if (cycleStackMarker > pt->mCycleMarker) {
608 // Cycles have been found that involve tracks that remain on the stack.
609 // Leave mCycleMarker indicating the most downstream (last) track on
610 // the stack known to be part of this SCC. In this way, any searches on
611 // other paths that find |ps| will know (without having to traverse from
612 // this track again) that they are part of this SCC (i.e. part of an
613 // intersecting cycle).
614 pt->mCycleMarker = cycleStackMarker;
615 continue;
618 // |pit| is the root of an SCC involving no other tracks on dfsStack, the
619 // complete SCC has been recorded, and tracks in this SCC are part of at
620 // least one cycle.
621 MOZ_ASSERT(cycleStackMarker == pt->mCycleMarker);
622 // If there are DelayNodes in this SCC, then they may break the cycles.
623 bool haveDelayNode = false;
624 auto next = sccStack.getFirst();
625 // Tracks in this SCC are identified by mCycleMarker <= cycleStackMarker.
626 // (There may be other tracks later in sccStack from other incompletely
627 // searched SCCs, involving tracks still on dfsStack.)
629 // DelayNodes in cycles must behave differently from those not in cycles,
630 // so all DelayNodes in the SCC must be identified.
631 while (next && static_cast<ProcessedMediaTrack*>(next)->mCycleMarker <=
632 cycleStackMarker) {
633 auto nt = next->AsAudioNodeTrack();
634 // Get next before perhaps removing from list below.
635 next = next->getNext();
636 if (nt && nt->Engine()->AsDelayNodeEngine()) {
637 haveDelayNode = true;
638 // DelayNodes break cycles by producing their output in a
639 // preprocessing phase; they do not need to be ordered before their
640 // consumers. Order them at the tail of mTracks so that they can be
641 // handled specially. Do so now, so that DFS ignores them.
642 nt->remove();
643 nt->mCycleMarker = 0;
644 --mFirstCycleBreaker;
645 mTracks[mFirstCycleBreaker] = nt;
648 auto after_scc = next;
649 while ((next = sccStack.getFirst()) != after_scc) {
650 next->remove();
651 auto removed = static_cast<ProcessedMediaTrack*>(next);
652 if (haveDelayNode) {
653 // Return tracks to the DFS stack again (to order and detect cycles
654 // without delayNodes). Any of these tracks that are still inputs
655 // for tracks on the visited stack must be returned to the front of
656 // the stack to be ordered before their dependents. We know that none
657 // of these tracks need input from tracks on the visited stack, so
658 // they can all be searched and ordered before the current stack head
659 // is popped.
660 removed->mCycleMarker = NOT_VISITED;
661 dfsStack.insertFront(removed);
662 } else {
663 // Tracks in cycles without any DelayNodes must be muted, and so do
664 // not need input and can be ordered now. They must be ordered before
665 // their consumers so that their muted output is available.
666 removed->mCycleMarker = IN_MUTED_CYCLE;
667 mTracks[orderedTrackCount] = removed;
668 ++orderedTrackCount;
673 MOZ_ASSERT(orderedTrackCount == mFirstCycleBreaker);
676 TrackTime MediaTrackGraphImpl::PlayAudio(const TrackAndVolume& aOutput,
677 GraphTime aPlayedTime,
678 uint32_t aOutputChannelCount) {
679 MOZ_ASSERT(OnGraphThread());
680 MOZ_ASSERT(mRealtime, "Should only attempt to play audio in realtime mode");
682 TrackTime ticksWritten = 0;
684 ticksWritten = 0;
685 MediaTrack* track = aOutput.mTrack;
686 AudioSegment* audio = track->GetData<AudioSegment>();
687 AudioSegment output;
689 TrackTime offset = track->GraphTimeToTrackTime(aPlayedTime);
691 // We don't update Track->mTracksStartTime here to account for time spent
692 // blocked. Instead, we'll update it in UpdateCurrentTimeForTracks after
693 // the blocked period has completed. But we do need to make sure we play
694 // from the right offsets in the track buffer, even if we've already
695 // written silence for some amount of blocked time after the current time.
696 GraphTime t = aPlayedTime;
697 while (t < mStateComputedTime) {
698 bool blocked = t >= track->mStartBlocking;
699 GraphTime end = blocked ? mStateComputedTime : track->mStartBlocking;
700 NS_ASSERTION(end <= mStateComputedTime, "mStartBlocking is wrong!");
702 // Check how many ticks of sound we can provide if we are blocked some
703 // time in the middle of this cycle.
704 TrackTime toWrite = end - t;
706 if (blocked) {
707 output.InsertNullDataAtStart(toWrite);
708 ticksWritten += toWrite;
709 LOG(LogLevel::Verbose,
710 ("%p: MediaTrack %p writing %" PRId64 " blocking-silence samples for "
711 "%f to %f (%" PRId64 " to %" PRId64 ")",
712 this, track, toWrite, MediaTimeToSeconds(t), MediaTimeToSeconds(end),
713 offset, offset + toWrite));
714 } else {
715 TrackTime endTicksNeeded = offset + toWrite;
716 TrackTime endTicksAvailable = audio->GetDuration();
718 if (endTicksNeeded <= endTicksAvailable) {
719 LOG(LogLevel::Verbose,
720 ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f "
721 "(samples %" PRId64 " to %" PRId64 ")",
722 this, track, toWrite, MediaTimeToSeconds(t),
723 MediaTimeToSeconds(end), offset, endTicksNeeded));
724 output.AppendSlice(*audio, offset, endTicksNeeded);
725 ticksWritten += toWrite;
726 offset = endTicksNeeded;
727 } else {
728 // MOZ_ASSERT(track->IsEnded(), "Not enough data, and track not
729 // ended."); If we are at the end of the track, maybe write the
730 // remaining samples, and pad with/output silence.
731 if (endTicksNeeded > endTicksAvailable && offset < endTicksAvailable) {
732 output.AppendSlice(*audio, offset, endTicksAvailable);
734 LOG(LogLevel::Verbose,
735 ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f "
736 "(samples %" PRId64 " to %" PRId64 ")",
737 this, track, toWrite, MediaTimeToSeconds(t),
738 MediaTimeToSeconds(end), offset, endTicksNeeded));
739 uint32_t available = endTicksAvailable - offset;
740 ticksWritten += available;
741 toWrite -= available;
742 offset = endTicksAvailable;
744 output.AppendNullData(toWrite);
745 LOG(LogLevel::Verbose,
746 ("%p MediaTrack %p writing %" PRId64 " padding slsamples for %f to "
747 "%f (samples %" PRId64 " to %" PRId64 ")",
748 this, track, toWrite, MediaTimeToSeconds(t),
749 MediaTimeToSeconds(end), offset, endTicksNeeded));
750 ticksWritten += toWrite;
752 output.ApplyVolume(mGlobalVolume * aOutput.mVolume);
754 t = end;
756 output.Mix(mMixer, aOutputChannelCount, mSampleRate);
758 return ticksWritten;
761 DeviceInputTrack* MediaTrackGraph::GetDeviceInputTrackMainThread(
762 CubebUtils::AudioDeviceID aID) {
763 MOZ_ASSERT(NS_IsMainThread());
764 auto* impl = static_cast<MediaTrackGraphImpl*>(this);
765 return impl->mDeviceInputTrackManagerMainThread.GetDeviceInputTrack(aID);
768 NativeInputTrack* MediaTrackGraph::GetNativeInputTrackMainThread() {
769 MOZ_ASSERT(NS_IsMainThread());
770 auto* impl = static_cast<MediaTrackGraphImpl*>(this);
771 return impl->mDeviceInputTrackManagerMainThread.GetNativeInputTrack();
774 void MediaTrackGraphImpl::OpenAudioInputImpl(DeviceInputTrack* aTrack) {
775 MOZ_ASSERT(OnGraphThread());
776 LOG(LogLevel::Debug,
777 ("%p OpenAudioInputImpl: device %p", this, aTrack->mDeviceId));
779 mDeviceInputTrackManagerGraphThread.Add(aTrack);
781 if (aTrack->AsNativeInputTrack()) {
782 // Switch Drivers since we're adding input (to input-only or full-duplex)
783 AudioCallbackDriver* driver = new AudioCallbackDriver(
784 this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(),
785 AudioInputChannelCount(aTrack->mDeviceId), PrimaryOutputDeviceID(),
786 aTrack->mDeviceId, AudioInputDevicePreference(aTrack->mDeviceId),
787 Some(aTrack->UpdateRequestedProcessingParams()));
788 LOG(LogLevel::Debug,
789 ("%p OpenAudioInputImpl: starting new AudioCallbackDriver(input) %p",
790 this, driver));
791 SwitchAtNextIteration(driver);
792 } else {
793 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
794 MOZ_ASSERT(nonNative);
795 // Start non-native input right away.
796 nonNative->StartAudio(MakeRefPtr<AudioInputSource>(
797 MakeRefPtr<AudioInputSourceListener>(nonNative),
798 nonNative->GenerateSourceId(), nonNative->mDeviceId,
799 AudioInputChannelCount(nonNative->mDeviceId),
800 AudioInputDevicePreference(nonNative->mDeviceId) ==
801 AudioInputType::Voice,
802 nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate()));
806 void MediaTrackGraphImpl::OpenAudioInput(DeviceInputTrack* aTrack) {
807 MOZ_ASSERT(NS_IsMainThread());
808 MOZ_ASSERT(aTrack);
810 LOG(LogLevel::Debug, ("%p OpenInput: DeviceInputTrack %p for device %p", this,
811 aTrack, aTrack->mDeviceId));
813 class Message : public ControlMessage {
814 public:
815 Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack)
816 : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {}
817 void Run() override {
818 TRACE("MTG::OpenAudioInputImpl ControlMessage");
819 mGraph->OpenAudioInputImpl(mInputTrack);
821 MediaTrackGraphImpl* mGraph;
822 DeviceInputTrack* mInputTrack;
825 mDeviceInputTrackManagerMainThread.Add(aTrack);
827 this->AppendMessage(MakeUnique<Message>(this, aTrack));
830 void MediaTrackGraphImpl::CloseAudioInputImpl(DeviceInputTrack* aTrack) {
831 MOZ_ASSERT(OnGraphThread());
833 LOG(LogLevel::Debug,
834 ("%p CloseAudioInputImpl: device %p", this, aTrack->mDeviceId));
836 if (NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack()) {
837 nonNative->StopAudio();
838 mDeviceInputTrackManagerGraphThread.Remove(aTrack);
839 return;
842 MOZ_ASSERT(aTrack->AsNativeInputTrack());
844 mDeviceInputTrackManagerGraphThread.Remove(aTrack);
846 // Switch Drivers since we're adding or removing an input (to nothing/system
847 // or output only)
848 bool audioTrackPresent = AudioTrackPresent();
850 GraphDriver* driver;
851 if (audioTrackPresent) {
852 // We still have audio output
853 LOG(LogLevel::Debug,
854 ("%p: CloseInput: output present (AudioCallback)", this));
856 driver = new AudioCallbackDriver(
857 this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(),
858 AudioInputChannelCount(aTrack->mDeviceId), PrimaryOutputDeviceID(),
859 nullptr, AudioInputDevicePreference(aTrack->mDeviceId),
860 Some(aTrack->UpdateRequestedProcessingParams()));
861 SwitchAtNextIteration(driver);
862 } else if (CurrentDriver()->AsAudioCallbackDriver()) {
863 LOG(LogLevel::Debug,
864 ("%p: CloseInput: no output present (SystemClockCallback)", this));
866 driver = new SystemClockDriver(this, CurrentDriver(), mSampleRate);
867 SwitchAtNextIteration(driver);
868 } // else SystemClockDriver->SystemClockDriver, no switch
871 void MediaTrackGraphImpl::UnregisterAllAudioOutputs(MediaTrack* aTrack) {
872 MOZ_ASSERT(OnGraphThreadOrNotRunning());
873 mOutputDevices.RemoveElementsBy([&](OutputDeviceEntry& aDeviceRef) {
874 aDeviceRef.mTrackOutputs.RemoveElement(aTrack);
875 // mReceiver is null for the primary output device, which is retained for
876 // AudioCallbackDriver output even when no tracks have audio outputs.
877 return aDeviceRef.mTrackOutputs.IsEmpty() && aDeviceRef.mReceiver;
881 void MediaTrackGraphImpl::CloseAudioInput(DeviceInputTrack* aTrack) {
882 MOZ_ASSERT(NS_IsMainThread());
883 MOZ_ASSERT(aTrack);
885 LOG(LogLevel::Debug, ("%p CloseInput: DeviceInputTrack %p for device %p",
886 this, aTrack, aTrack->mDeviceId));
888 class Message : public ControlMessage {
889 public:
890 Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack)
891 : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {}
892 void Run() override {
893 TRACE("MTG::CloseAudioInputImpl ControlMessage");
894 mGraph->CloseAudioInputImpl(mInputTrack);
896 MediaTrackGraphImpl* mGraph;
897 DeviceInputTrack* mInputTrack;
900 // DeviceInputTrack is still alive (in mTracks) even we remove it here, since
901 // aTrack->Destroy() is called after this. See DeviceInputTrack::CloseAudio
902 // for more details.
903 mDeviceInputTrackManagerMainThread.Remove(aTrack);
905 this->AppendMessage(MakeUnique<Message>(this, aTrack));
907 if (aTrack->AsNativeInputTrack()) {
908 LOG(LogLevel::Debug,
909 ("%p Native input device %p is closed!", this, aTrack->mDeviceId));
910 SetNewNativeInput();
914 // All AudioInput listeners get the same speaker data (at least for now).
915 void MediaTrackGraphImpl::NotifyOutputData(const AudioChunk& aChunk) {
916 if (!mDeviceInputTrackManagerGraphThread.GetNativeInputTrack()) {
917 return;
920 #if defined(MOZ_WEBRTC)
921 for (const auto& track : mTracks) {
922 if (const auto& t = track->AsAudioProcessingTrack()) {
923 t->NotifyOutputData(this, aChunk);
926 #endif
929 void MediaTrackGraphImpl::NotifyInputStopped() {
930 NativeInputTrack* native =
931 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
932 if (!native) {
933 return;
935 native->NotifyInputStopped(this);
938 void MediaTrackGraphImpl::NotifyInputData(const AudioDataValue* aBuffer,
939 size_t aFrames, TrackRate aRate,
940 uint32_t aChannels,
941 uint32_t aAlreadyBuffered) {
942 // Either we have an audio input device, or we just removed the audio input
943 // this iteration, and we're switching back to an output-only driver next
944 // iteration.
945 NativeInputTrack* native =
946 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
947 MOZ_ASSERT(native || Switching());
948 if (!native) {
949 return;
951 native->NotifyInputData(this, aBuffer, aFrames, aRate, aChannels,
952 aAlreadyBuffered);
955 void MediaTrackGraphImpl::NotifySetRequestedInputProcessingParamsResult(
956 AudioCallbackDriver* aDriver, int aGeneration,
957 Result<cubeb_input_processing_params, int>&& aResult) {
958 MOZ_ASSERT(NS_IsMainThread());
959 NativeInputTrack* native =
960 mDeviceInputTrackManagerMainThread.GetNativeInputTrack();
961 if (!native) {
962 return;
964 QueueControlMessageWithNoShutdown([this, self = RefPtr(this),
965 driver = RefPtr(aDriver), aGeneration,
966 result = std::move(aResult)]() mutable {
967 NativeInputTrack* native =
968 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
969 if (!native) {
970 return;
972 if (driver != mDriver) {
973 return;
975 native->NotifySetRequestedProcessingParamsResult(this, aGeneration, result);
979 void MediaTrackGraphImpl::DeviceChangedImpl() {
980 MOZ_ASSERT(OnGraphThread());
981 NativeInputTrack* native =
982 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
983 if (!native) {
984 return;
986 native->DeviceChanged(this);
989 void MediaTrackGraphImpl::SetMaxOutputChannelCount(uint32_t aMaxChannelCount) {
990 MOZ_ASSERT(OnGraphThread());
991 mMaxOutputChannelCount = aMaxChannelCount;
994 void MediaTrackGraphImpl::DeviceChanged() {
995 // This is safe to be called from any thread: this message comes from an
996 // underlying platform API, and we don't have much guarantees. If it is not
997 // called from the main thread (and it probably will rarely be), it will post
998 // itself to the main thread, and the actual device change message will be ran
999 // and acted upon on the graph thread.
1000 if (!NS_IsMainThread()) {
1001 RefPtr<nsIRunnable> runnable = WrapRunnable(
1002 RefPtr<MediaTrackGraphImpl>(this), &MediaTrackGraphImpl::DeviceChanged);
1003 mMainThread->Dispatch(runnable.forget());
1004 return;
1007 class Message : public ControlMessage {
1008 public:
1009 explicit Message(MediaTrackGraph* aGraph)
1010 : ControlMessage(nullptr),
1011 mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)) {}
1012 void Run() override {
1013 TRACE("MTG::DeviceChangeImpl ControlMessage");
1014 mGraphImpl->DeviceChangedImpl();
1016 // We know that this is valid, because the graph can't shutdown if it has
1017 // messages.
1018 MediaTrackGraphImpl* mGraphImpl;
1021 if (mMainThreadTrackCount == 0 && mMainThreadPortCount == 0) {
1022 // This is a special case where the origin of this event cannot control the
1023 // lifetime of the graph, because the graph is controling the lifetime of
1024 // the AudioCallbackDriver where the event originated.
1025 // We know the graph is soon going away, so there's no need to notify about
1026 // this device change.
1027 return;
1030 // Reset the latency, it will get fetched again next time it's queried.
1031 MOZ_ASSERT(NS_IsMainThread());
1032 mAudioOutputLatency = 0.0;
1034 // Dispatch to the bg thread to do the (potentially expensive) query of the
1035 // maximum channel count, and then dispatch back to the main thread, then to
1036 // the graph, with the new info.
1037 RefPtr<MediaTrackGraphImpl> self = this;
1038 NS_DispatchBackgroundTask(NS_NewRunnableFunction(
1039 "MaxChannelCountUpdateOnBgThread", [self{std::move(self)}]() {
1040 uint32_t maxChannelCount = CubebUtils::MaxNumberOfChannels();
1041 self->Dispatch(NS_NewRunnableFunction(
1042 "MaxChannelCountUpdateToMainThread",
1043 [self{self}, maxChannelCount]() {
1044 class MessageToGraph : public ControlMessage {
1045 public:
1046 explicit MessageToGraph(MediaTrackGraph* aGraph,
1047 uint32_t aMaxChannelCount)
1048 : ControlMessage(nullptr),
1049 mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)),
1050 mMaxChannelCount(aMaxChannelCount) {}
1051 void Run() override {
1052 TRACE("MTG::SetMaxOutputChannelCount ControlMessage")
1053 mGraphImpl->SetMaxOutputChannelCount(mMaxChannelCount);
1055 MediaTrackGraphImpl* mGraphImpl;
1056 uint32_t mMaxChannelCount;
1058 self->AppendMessage(
1059 MakeUnique<MessageToGraph>(self, maxChannelCount));
1060 }));
1061 }));
1063 AppendMessage(MakeUnique<Message>(this));
1066 static const char* GetAudioInputTypeString(const AudioInputType& aType) {
1067 return aType == AudioInputType::Voice ? "Voice" : "Unknown";
1070 void MediaTrackGraph::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) {
1071 MOZ_ASSERT(OnGraphThread());
1072 auto* impl = static_cast<MediaTrackGraphImpl*>(this);
1073 impl->ReevaluateInputDevice(aID);
1076 void MediaTrackGraphImpl::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) {
1077 MOZ_ASSERT(OnGraphThread());
1078 LOG(LogLevel::Debug, ("%p: ReevaluateInputDevice: device %p", this, aID));
1080 DeviceInputTrack* track =
1081 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
1082 if (!track) {
1083 LOG(LogLevel::Debug,
1084 ("%p: No DeviceInputTrack for this device. Ignore", this));
1085 return;
1088 bool needToSwitch = false;
1090 if (NonNativeInputTrack* nonNative = track->AsNonNativeInputTrack()) {
1091 if (nonNative->NumberOfChannels() != AudioInputChannelCount(aID)) {
1092 LOG(LogLevel::Debug,
1093 ("%p: %u-channel non-native input device %p (track %p) is "
1094 "re-configured to %d-channel",
1095 this, nonNative->NumberOfChannels(), aID, track,
1096 AudioInputChannelCount(aID)));
1097 needToSwitch = true;
1099 if (nonNative->DevicePreference() != AudioInputDevicePreference(aID)) {
1100 LOG(LogLevel::Debug,
1101 ("%p: %s-type non-native input device %p (track %p) is re-configured "
1102 "to %s-type",
1103 this, GetAudioInputTypeString(nonNative->DevicePreference()), aID,
1104 track, GetAudioInputTypeString(AudioInputDevicePreference(aID))));
1105 needToSwitch = true;
1108 if (needToSwitch) {
1109 nonNative->StopAudio();
1110 nonNative->StartAudio(MakeRefPtr<AudioInputSource>(
1111 MakeRefPtr<AudioInputSourceListener>(nonNative),
1112 nonNative->GenerateSourceId(), aID, AudioInputChannelCount(aID),
1113 AudioInputDevicePreference(aID) == AudioInputType::Voice,
1114 nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate()));
1117 return;
1120 MOZ_ASSERT(track->AsNativeInputTrack());
1122 if (AudioCallbackDriver* audioCallbackDriver =
1123 CurrentDriver()->AsAudioCallbackDriver()) {
1124 if (audioCallbackDriver->InputChannelCount() !=
1125 AudioInputChannelCount(aID)) {
1126 LOG(LogLevel::Debug,
1127 ("%p: ReevaluateInputDevice: %u-channel AudioCallbackDriver %p is "
1128 "re-configured to %d-channel",
1129 this, audioCallbackDriver->InputChannelCount(), audioCallbackDriver,
1130 AudioInputChannelCount(aID)));
1131 needToSwitch = true;
1133 if (audioCallbackDriver->InputDevicePreference() !=
1134 AudioInputDevicePreference(aID)) {
1135 LOG(LogLevel::Debug,
1136 ("%p: ReevaluateInputDevice: %s-type AudioCallbackDriver %p is "
1137 "re-configured to %s-type",
1138 this,
1139 GetAudioInputTypeString(
1140 audioCallbackDriver->InputDevicePreference()),
1141 audioCallbackDriver,
1142 GetAudioInputTypeString(AudioInputDevicePreference(aID))));
1143 needToSwitch = true;
1145 } else if (Switching() && NextDriver()->AsAudioCallbackDriver()) {
1146 // We're already in the process of switching to a audio callback driver,
1147 // which will happen at the next iteration.
1148 // However, maybe it's not the correct number of channels. Re-query the
1149 // correct channel amount at this time.
1150 needToSwitch = true;
1153 if (needToSwitch) {
1154 AudioCallbackDriver* newDriver = new AudioCallbackDriver(
1155 this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(),
1156 AudioInputChannelCount(aID), PrimaryOutputDeviceID(), aID,
1157 AudioInputDevicePreference(aID),
1158 Some(track->UpdateRequestedProcessingParams()));
1159 SwitchAtNextIteration(newDriver);
1163 bool MediaTrackGraphImpl::OnGraphThreadOrNotRunning() const {
1164 // either we're on the right thread (and calling CurrentDriver() is safe),
1165 // or we're going to fail the assert anyway, so don't cross-check
1166 // via CurrentDriver().
1167 return mGraphDriverRunning ? OnGraphThread() : NS_IsMainThread();
1170 bool MediaTrackGraphImpl::OnGraphThread() const {
1171 // we're on the right thread (and calling mDriver is safe),
1172 MOZ_ASSERT(mDriver);
1173 if (mGraphRunner && mGraphRunner->OnThread()) {
1174 return true;
1176 return mDriver->OnThread();
1179 bool MediaTrackGraphImpl::Destroyed() const {
1180 MOZ_ASSERT(NS_IsMainThread());
1181 return !mSelfRef;
1184 bool MediaTrackGraphImpl::ShouldUpdateMainThread() {
1185 MOZ_ASSERT(OnGraphThreadOrNotRunning());
1186 if (mRealtime) {
1187 return true;
1190 TimeStamp now = TimeStamp::Now();
1191 // For offline graphs, update now if it has been long enough since the last
1192 // update, or if it has reached the end.
1193 if ((now - mLastMainThreadUpdate).ToMilliseconds() >
1194 CurrentDriver()->IterationDuration() ||
1195 mStateComputedTime >= mEndTime) {
1196 mLastMainThreadUpdate = now;
1197 return true;
1199 return false;
1202 void MediaTrackGraphImpl::PrepareUpdatesToMainThreadState(bool aFinalUpdate) {
1203 MOZ_ASSERT(OnGraphThreadOrNotRunning());
1204 mMonitor.AssertCurrentThreadOwns();
1206 // We don't want to frequently update the main thread about timing update
1207 // when we are not running in realtime.
1208 if (aFinalUpdate || ShouldUpdateMainThread()) {
1209 // Strip updates that will be obsoleted below, so as to keep the length of
1210 // mTrackUpdates sane.
1211 size_t keptUpdateCount = 0;
1212 for (size_t i = 0; i < mTrackUpdates.Length(); ++i) {
1213 MediaTrack* track = mTrackUpdates[i].mTrack;
1214 // RemoveTrackGraphThread() clears mTrack in updates for
1215 // tracks that are removed from the graph.
1216 MOZ_ASSERT(!track || track->GraphImpl() == this);
1217 if (!track || track->MainThreadNeedsUpdates()) {
1218 // Discard this update as it has either been cleared when the track
1219 // was destroyed or there will be a newer update below.
1220 continue;
1222 if (keptUpdateCount != i) {
1223 mTrackUpdates[keptUpdateCount] = std::move(mTrackUpdates[i]);
1224 MOZ_ASSERT(!mTrackUpdates[i].mTrack);
1226 ++keptUpdateCount;
1228 mTrackUpdates.TruncateLength(keptUpdateCount);
1230 mTrackUpdates.SetCapacity(mTrackUpdates.Length() + mTracks.Length() +
1231 mSuspendedTracks.Length());
1232 for (MediaTrack* track : AllTracks()) {
1233 if (!track->MainThreadNeedsUpdates()) {
1234 continue;
1236 TrackUpdate* update = mTrackUpdates.AppendElement();
1237 update->mTrack = track;
1238 // No blocking to worry about here, since we've passed
1239 // UpdateCurrentTimeForTracks.
1240 update->mNextMainThreadCurrentTime =
1241 track->GraphTimeToTrackTime(mProcessedTime);
1242 update->mNextMainThreadEnded = track->mNotifiedEnded;
1244 mNextMainThreadGraphTime = mProcessedTime;
1245 if (!mPendingUpdateRunnables.IsEmpty()) {
1246 mUpdateRunnables.AppendElements(std::move(mPendingUpdateRunnables));
1250 // If this is the final update, then a stable state event will soon be
1251 // posted just before this thread finishes, and so there is no need to also
1252 // post here.
1253 if (!aFinalUpdate &&
1254 // Don't send the message to the main thread if it's not going to have
1255 // any work to do.
1256 !(mUpdateRunnables.IsEmpty() && mTrackUpdates.IsEmpty())) {
1257 EnsureStableStateEventPosted();
1261 GraphTime MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(GraphTime aTime) {
1262 if (aTime % WEBAUDIO_BLOCK_SIZE == 0) {
1263 return aTime;
1265 return RoundUpToNextAudioBlock(aTime);
1268 GraphTime MediaTrackGraphImpl::RoundUpToNextAudioBlock(GraphTime aTime) {
1269 uint64_t block = aTime >> WEBAUDIO_BLOCK_SIZE_BITS;
1270 uint64_t nextBlock = block + 1;
1271 GraphTime nextTime = nextBlock << WEBAUDIO_BLOCK_SIZE_BITS;
1272 return nextTime;
1275 void MediaTrackGraphImpl::ProduceDataForTracksBlockByBlock(
1276 uint32_t aTrackIndex, TrackRate aSampleRate) {
1277 MOZ_ASSERT(OnGraphThread());
1278 MOZ_ASSERT(aTrackIndex <= mFirstCycleBreaker,
1279 "Cycle breaker is not AudioNodeTrack?");
1281 while (mProcessedTime < mStateComputedTime) {
1282 // Microtask checkpoints are in between render quanta.
1283 nsAutoMicroTask mt;
1285 GraphTime next = RoundUpToNextAudioBlock(mProcessedTime);
1286 for (uint32_t i = mFirstCycleBreaker; i < mTracks.Length(); ++i) {
1287 auto nt = static_cast<AudioNodeTrack*>(mTracks[i]);
1288 MOZ_ASSERT(nt->AsAudioNodeTrack());
1289 nt->ProduceOutputBeforeInput(mProcessedTime);
1291 for (uint32_t i = aTrackIndex; i < mTracks.Length(); ++i) {
1292 ProcessedMediaTrack* pt = mTracks[i]->AsProcessedTrack();
1293 if (pt) {
1294 pt->ProcessInput(
1295 mProcessedTime, next,
1296 (next == mStateComputedTime) ? ProcessedMediaTrack::ALLOW_END : 0);
1299 mProcessedTime = next;
1301 NS_ASSERTION(mProcessedTime == mStateComputedTime,
1302 "Something went wrong with rounding to block boundaries");
1305 void MediaTrackGraphImpl::RunMessageAfterProcessing(
1306 UniquePtr<ControlMessageInterface> aMessage) {
1307 MOZ_ASSERT(OnGraphThread());
1309 if (mFrontMessageQueue.IsEmpty()) {
1310 mFrontMessageQueue.AppendElement();
1313 // Only one block is used for messages from the graph thread.
1314 MOZ_ASSERT(mFrontMessageQueue.Length() == 1);
1315 mFrontMessageQueue[0].mMessages.AppendElement(std::move(aMessage));
1318 void MediaTrackGraphImpl::RunMessagesInQueue() {
1319 TRACE("MTG::RunMessagesInQueue");
1320 MOZ_ASSERT(OnGraphThread());
1321 // Calculate independent action times for each batch of messages (each
1322 // batch corresponding to an event loop task). This isolates the performance
1323 // of different scripts to some extent.
1324 for (uint32_t i = 0; i < mFrontMessageQueue.Length(); ++i) {
1325 nsTArray<UniquePtr<ControlMessageInterface>>& messages =
1326 mFrontMessageQueue[i].mMessages;
1328 for (uint32_t j = 0; j < messages.Length(); ++j) {
1329 TRACE("ControlMessage::Run");
1330 messages[j]->Run();
1333 mFrontMessageQueue.Clear();
1336 void MediaTrackGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions) {
1337 TRACE("MTG::UpdateGraph");
1338 MOZ_ASSERT(OnGraphThread());
1339 MOZ_ASSERT(aEndBlockingDecisions >= mProcessedTime);
1340 // The next state computed time can be the same as the previous: it
1341 // means the driver would have been blocking indefinitly, but the graph has
1342 // been woken up right after having been to sleep.
1343 MOZ_ASSERT(aEndBlockingDecisions >= mStateComputedTime);
1345 CheckDriver();
1346 UpdateTrackOrder();
1348 // Always do another iteration if there are tracks waiting to resume.
1349 bool ensureNextIteration = !mPendingResumeOperations.IsEmpty();
1351 for (MediaTrack* track : mTracks) {
1352 if (SourceMediaTrack* is = track->AsSourceTrack()) {
1353 ensureNextIteration |= is->PullNewData(aEndBlockingDecisions);
1354 is->ExtractPendingInput(mStateComputedTime, aEndBlockingDecisions);
1356 if (track->mEnded) {
1357 // The track's not suspended, and since it's ended, underruns won't
1358 // stop it playing out. So there's no blocking other than what we impose
1359 // here.
1360 GraphTime endTime = track->GetEnd() + track->mStartTime;
1361 if (endTime <= mStateComputedTime) {
1362 LOG(LogLevel::Verbose,
1363 ("%p: MediaTrack %p is blocked due to being ended", this, track));
1364 track->mStartBlocking = mStateComputedTime;
1365 } else {
1366 LOG(LogLevel::Verbose,
1367 ("%p: MediaTrack %p has ended, but is not blocked yet (current "
1368 "time %f, end at %f)",
1369 this, track, MediaTimeToSeconds(mStateComputedTime),
1370 MediaTimeToSeconds(endTime)));
1371 // Data can't be added to a ended track, so underruns are irrelevant.
1372 MOZ_ASSERT(endTime <= aEndBlockingDecisions);
1373 track->mStartBlocking = endTime;
1375 } else {
1376 track->mStartBlocking = WillUnderrun(track, aEndBlockingDecisions);
1378 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
1379 if (SourceMediaTrack* s = track->AsSourceTrack()) {
1380 if (s->Ended()) {
1381 continue;
1384 MutexAutoLock lock(s->mMutex);
1385 if (!s->mUpdateTrack->mPullingEnabled) {
1386 // The invariant that data must be provided is only enforced when
1387 // pulling.
1388 continue;
1391 if (track->GetEnd() <
1392 track->GraphTimeToTrackTime(aEndBlockingDecisions)) {
1393 LOG(LogLevel::Error,
1394 ("%p: SourceMediaTrack %p (%s) is live and pulled, "
1395 "but wasn't fed "
1396 "enough data. TrackListeners=%zu. Track-end=%f, "
1397 "Iteration-end=%f",
1398 this, track,
1399 (track->mType == MediaSegment::AUDIO ? "audio" : "video"),
1400 track->mTrackListeners.Length(),
1401 MediaTimeToSeconds(track->GetEnd()),
1402 MediaTimeToSeconds(
1403 track->GraphTimeToTrackTime(aEndBlockingDecisions))));
1404 MOZ_DIAGNOSTIC_CRASH(
1405 "A non-ended SourceMediaTrack wasn't fed "
1406 "enough data by NotifyPull");
1409 #endif /* MOZ_DIAGNOSTIC_ASSERT_ENABLED */
1413 for (MediaTrack* track : mSuspendedTracks) {
1414 track->mStartBlocking = mStateComputedTime;
1417 // If the loop is woken up so soon that IterationEnd() barely advances or
1418 // if an offline graph is not currently rendering, we end up having
1419 // aEndBlockingDecisions == mStateComputedTime.
1420 // Since the process interval [mStateComputedTime, aEndBlockingDecision) is
1421 // empty, Process() will not find any unblocked track and so will not
1422 // ensure another iteration. If the graph should be rendering, then ensure
1423 // another iteration to render.
1424 if (ensureNextIteration || (aEndBlockingDecisions == mStateComputedTime &&
1425 mStateComputedTime < mEndTime)) {
1426 EnsureNextIteration();
1430 void MediaTrackGraphImpl::SelectOutputDeviceForAEC() {
1431 MOZ_ASSERT(OnGraphThread());
1432 size_t currentDeviceIndex = mOutputDevices.IndexOf(mOutputDeviceForAEC);
1433 if (currentDeviceIndex == mOutputDevices.NoIndex) {
1434 // Outputs for this device have been removed.
1435 // Fall back to the primary output device.
1436 LOG(LogLevel::Info, ("%p: No remaining outputs to device %p. "
1437 "Switch to primary output device %p for AEC",
1438 this, mOutputDeviceForAEC, PrimaryOutputDeviceID()));
1439 mOutputDeviceForAEC = PrimaryOutputDeviceID();
1440 currentDeviceIndex = 0;
1441 MOZ_ASSERT(mOutputDevices[0].mDeviceID == mOutputDeviceForAEC);
1443 if (mOutputDevices.Length() == 1) {
1444 // No other output devices so there is no choice.
1445 return;
1448 // The output is considered silent intentionally only if the whole duration
1449 // (often more than just this processing interval) of audio data in the
1450 // MediaSegment is null so as to reduce switching between output devices
1451 // should there be short durations of silence.
1452 auto HasNonNullAudio = [](const TrackAndVolume& aTV) {
1453 return aTV.mVolume != 0 && !aTV.mTrack->IsSuspended() &&
1454 !aTV.mTrack->GetData()->IsNull();
1456 // Keep using the same output device stream if it has non-null data,
1457 // so as to stay with a stream having ongoing audio. If the output stream
1458 // is switched, the echo cancellation algorithm can take some time to adjust
1459 // to the change in delay, so there is less value in switching back and
1460 // forth between output devices for very short sounds.
1461 for (const auto& output : mOutputDevices[currentDeviceIndex].mTrackOutputs) {
1462 if (HasNonNullAudio(output)) {
1463 return;
1466 // The current output device is silent. Use another if it has non-null data.
1467 for (const auto& outputDeviceEntry : mOutputDevices) {
1468 for (const auto& output : outputDeviceEntry.mTrackOutputs) {
1469 if (HasNonNullAudio(output)) {
1470 // Switch to this device.
1471 LOG(LogLevel::Info,
1472 ("%p: Switch output device for AEC from silent %p to non-null %p",
1473 this, mOutputDeviceForAEC, outputDeviceEntry.mDeviceID));
1474 mOutputDeviceForAEC = outputDeviceEntry.mDeviceID;
1475 return;
1479 // Null data for all outputs. Keep using the same device.
1482 void MediaTrackGraphImpl::Process(MixerCallbackReceiver* aMixerReceiver) {
1483 TRACE("MTG::Process");
1484 MOZ_ASSERT(OnGraphThread());
1485 if (mStateComputedTime == mProcessedTime) { // No frames to render.
1486 return;
1489 // Play track contents.
1490 bool allBlockedForever = true;
1491 // True when we've done ProcessInput for all processed tracks.
1492 bool doneAllProducing = false;
1493 const GraphTime oldProcessedTime = mProcessedTime;
1495 // Figure out what each track wants to do
1496 for (uint32_t i = 0; i < mTracks.Length(); ++i) {
1497 MediaTrack* track = mTracks[i];
1498 if (!doneAllProducing) {
1499 ProcessedMediaTrack* pt = track->AsProcessedTrack();
1500 if (pt) {
1501 AudioNodeTrack* n = track->AsAudioNodeTrack();
1502 if (n) {
1503 #ifdef DEBUG
1504 // Verify that the sampling rate for all of the following tracks is
1505 // the same
1506 for (uint32_t j = i + 1; j < mTracks.Length(); ++j) {
1507 AudioNodeTrack* nextTrack = mTracks[j]->AsAudioNodeTrack();
1508 if (nextTrack) {
1509 MOZ_ASSERT(n->mSampleRate == nextTrack->mSampleRate,
1510 "All AudioNodeTracks in the graph must have the same "
1511 "sampling rate");
1514 #endif
1515 // Since an AudioNodeTrack is present, go ahead and
1516 // produce audio block by block for all the rest of the tracks.
1517 ProduceDataForTracksBlockByBlock(i, n->mSampleRate);
1518 doneAllProducing = true;
1519 } else {
1520 pt->ProcessInput(mProcessedTime, mStateComputedTime,
1521 ProcessedMediaTrack::ALLOW_END);
1522 // Assert that a live track produced enough data
1523 MOZ_ASSERT_IF(!track->mEnded,
1524 track->GetEnd() >= GraphTimeToTrackTimeWithBlocking(
1525 track, mStateComputedTime));
1529 if (track->mStartBlocking > oldProcessedTime) {
1530 allBlockedForever = false;
1533 mProcessedTime = mStateComputedTime;
1535 SelectOutputDeviceForAEC();
1536 for (const auto& outputDeviceEntry : mOutputDevices) {
1537 uint32_t outputChannelCount;
1538 if (!outputDeviceEntry.mReceiver) { // primary output
1539 if (!aMixerReceiver) {
1540 // Running off a system clock driver. No need to mix output.
1541 continue;
1543 MOZ_ASSERT(CurrentDriver()->AsAudioCallbackDriver(),
1544 "Driver must be AudioCallbackDriver if aMixerReceiver");
1545 // Use the number of channel the driver expects: this is the number of
1546 // channel that can be output by the underlying system level audio stream.
1547 outputChannelCount =
1548 CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
1549 } else {
1550 outputChannelCount = AudioOutputChannelCount(outputDeviceEntry);
1552 MOZ_ASSERT(mRealtime,
1553 "If there's an output device, this graph must be realtime");
1554 mMixer.StartMixing();
1555 // This is the number of frames that are written to the output buffer, for
1556 // this iteration.
1557 TrackTime ticksPlayed = 0;
1558 for (const auto& t : outputDeviceEntry.mTrackOutputs) {
1559 TrackTime ticksPlayedForThisTrack =
1560 PlayAudio(t, oldProcessedTime, outputChannelCount);
1561 if (ticksPlayed == 0) {
1562 ticksPlayed = ticksPlayedForThisTrack;
1563 } else {
1564 MOZ_ASSERT(
1565 !ticksPlayedForThisTrack || ticksPlayedForThisTrack == ticksPlayed,
1566 "Each track should have the same number of frames.");
1570 if (ticksPlayed == 0) {
1571 // Nothing was played, so the mixer doesn't know how many frames were
1572 // processed. We still tell it so AudioCallbackDriver knows how much has
1573 // been processed. (bug 1406027)
1574 mMixer.Mix(nullptr, outputChannelCount,
1575 mStateComputedTime - oldProcessedTime, mSampleRate);
1577 AudioChunk* outputChunk = mMixer.MixedChunk();
1578 if (outputDeviceEntry.mDeviceID == mOutputDeviceForAEC) {
1579 // Callback any observers for the AEC speaker data. Note that one
1580 // (maybe) of these will be full-duplex, the others will get their input
1581 // data off separate cubeb callbacks.
1582 NotifyOutputData(*outputChunk);
1584 if (!outputDeviceEntry.mReceiver) { // primary output
1585 aMixerReceiver->MixerCallback(outputChunk, mSampleRate);
1586 } else {
1587 outputDeviceEntry.mReceiver->EnqueueAudio(*outputChunk);
1591 if (!allBlockedForever) {
1592 EnsureNextIteration();
1596 bool MediaTrackGraphImpl::UpdateMainThreadState() {
1597 MOZ_ASSERT(OnGraphThread());
1598 if (mForceShutDownReceived) {
1599 for (MediaTrack* track : AllTracks()) {
1600 track->OnGraphThreadDone();
1604 MonitorAutoLock lock(mMonitor);
1605 bool finalUpdate =
1606 mForceShutDownReceived || (IsEmpty() && mBackMessageQueue.IsEmpty());
1607 PrepareUpdatesToMainThreadState(finalUpdate);
1608 if (!finalUpdate) {
1609 SwapMessageQueues();
1610 return true;
1612 // The JSContext will not be used again.
1613 // Clear main thread access while under monitor.
1614 mJSContext = nullptr;
1616 dom::WorkletThread::DeleteCycleCollectedJSContext();
1617 // Enter shutdown mode when this iteration is completed.
1618 // No need to Destroy tracks here. The main-thread owner of each
1619 // track is responsible for calling Destroy on them.
1620 return false;
1623 auto MediaTrackGraphImpl::OneIteration(
1624 GraphTime aStateTime, GraphTime aIterationEnd,
1625 MixerCallbackReceiver* aMixerReceiver) -> IterationResult {
1626 if (mGraphRunner) {
1627 return mGraphRunner->OneIteration(aStateTime, aIterationEnd,
1628 aMixerReceiver);
1631 return OneIterationImpl(aStateTime, aIterationEnd, aMixerReceiver);
1634 auto MediaTrackGraphImpl::OneIterationImpl(
1635 GraphTime aStateTime, GraphTime aIterationEnd,
1636 MixerCallbackReceiver* aMixerReceiver) -> IterationResult {
1637 TRACE("MTG::OneIterationImpl");
1639 if (SoftRealTimeLimitReached()) {
1640 TRACE("MTG::Demoting real-time thread!");
1641 DemoteThreadFromRealTime();
1644 // Changes to LIFECYCLE_RUNNING occur before starting or reviving the graph
1645 // thread, and so the monitor need not be held to check mLifecycleState.
1646 // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
1647 // graphs that have not started.
1649 // While changes occur on mainthread, this assert confirms that
1650 // this code shouldn't run if mainthread might be changing the state (to
1651 // > LIFECYCLE_RUNNING)
1653 // Ignore mutex warning: static during execution of the graph
1654 MOZ_PUSH_IGNORE_THREAD_SAFETY
1655 MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING);
1656 MOZ_POP_THREAD_SAFETY
1658 MOZ_ASSERT(OnGraphThread());
1660 WebCore::DenormalDisabler disabler;
1662 // Process graph message from the main thread for this iteration.
1663 RunMessagesInQueue();
1665 // Process MessagePort events.
1666 // These require a single thread, which has an nsThread with an event queue.
1667 if (mGraphRunner || !mRealtime) {
1668 TRACE("MTG::MessagePort events");
1669 NS_ProcessPendingEvents(nullptr);
1672 GraphTime stateTime = std::min(aStateTime, GraphTime(mEndTime));
1673 UpdateGraph(stateTime);
1675 mStateComputedTime = stateTime;
1677 GraphTime oldProcessedTime = mProcessedTime;
1678 Process(aMixerReceiver);
1679 MOZ_ASSERT(mProcessedTime == stateTime);
1681 UpdateCurrentTimeForTracks(oldProcessedTime);
1683 ProcessChunkMetadata(oldProcessedTime);
1685 // Process graph messages queued from RunMessageAfterProcessing() on this
1686 // thread during the iteration.
1687 RunMessagesInQueue();
1689 if (!UpdateMainThreadState()) {
1690 if (Switching()) {
1691 // We'll never get to do this switch. Clear mNextDriver to break the
1692 // ref-cycle graph->nextDriver->currentDriver->graph.
1693 SwitchAtNextIteration(nullptr);
1695 return IterationResult::CreateStop(
1696 NewRunnableMethod("MediaTrackGraphImpl::SignalMainThreadCleanup", this,
1697 &MediaTrackGraphImpl::SignalMainThreadCleanup));
1700 if (Switching()) {
1701 RefPtr<GraphDriver> nextDriver = std::move(mNextDriver);
1702 return IterationResult::CreateSwitchDriver(
1703 nextDriver, NewRunnableMethod<StoreRefPtrPassByPtr<GraphDriver>>(
1704 "MediaTrackGraphImpl::SetCurrentDriver", this,
1705 &MediaTrackGraphImpl::SetCurrentDriver, nextDriver));
1708 return IterationResult::CreateStillProcessing();
1711 void MediaTrackGraphImpl::ApplyTrackUpdate(TrackUpdate* aUpdate) {
1712 MOZ_ASSERT(NS_IsMainThread());
1713 mMonitor.AssertCurrentThreadOwns();
1715 MediaTrack* track = aUpdate->mTrack;
1716 if (!track) return;
1717 track->mMainThreadCurrentTime = aUpdate->mNextMainThreadCurrentTime;
1718 track->mMainThreadEnded = aUpdate->mNextMainThreadEnded;
1720 if (track->ShouldNotifyTrackEnded()) {
1721 track->NotifyMainThreadListeners();
1725 void MediaTrackGraphImpl::ForceShutDown() {
1726 MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
1727 LOG(LogLevel::Debug, ("%p: MediaTrackGraph::ForceShutdown", this));
1729 if (mShutdownBlocker) {
1730 // Avoid waiting forever for a graph to shut down
1731 // synchronously. Reports are that some 3rd-party audio drivers
1732 // occasionally hang in shutdown (both for us and Chrome).
1733 NS_NewTimerWithCallback(
1734 getter_AddRefs(mShutdownTimer), this,
1735 MediaTrackGraph::AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT,
1736 nsITimer::TYPE_ONE_SHOT);
1739 class Message final : public ControlMessage {
1740 public:
1741 explicit Message(MediaTrackGraphImpl* aGraph)
1742 : ControlMessage(nullptr), mGraph(aGraph) {}
1743 void Run() override {
1744 TRACE("MTG::ForceShutdown ControlMessage");
1745 mGraph->mForceShutDownReceived = true;
1747 // The graph owns this message.
1748 MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph;
1751 if (mMainThreadTrackCount > 0 || mMainThreadPortCount > 0) {
1752 // If both the track and port counts are zero, the regular shutdown
1753 // sequence will progress shortly to shutdown threads and destroy the graph.
1754 AppendMessage(MakeUnique<Message>(this));
1755 InterruptJS();
1759 NS_IMETHODIMP
1760 MediaTrackGraphImpl::Notify(nsITimer* aTimer) {
1761 MOZ_ASSERT(NS_IsMainThread());
1762 MOZ_ASSERT(!mShutdownBlocker, "MediaTrackGraph took too long to shut down!");
1763 // Sigh, graph took too long to shut down. Stop blocking system
1764 // shutdown and hope all is well.
1765 RemoveShutdownBlocker();
1766 return NS_OK;
1769 static nsCString GetDocumentTitle(uint64_t aWindowID) {
1770 MOZ_ASSERT(NS_IsMainThread());
1771 nsCString title;
1772 auto* win = nsGlobalWindowInner::GetInnerWindowWithId(aWindowID);
1773 if (!win) {
1774 return title;
1776 Document* doc = win->GetExtantDoc();
1777 if (!doc) {
1778 return title;
1780 nsAutoString titleUTF16;
1781 doc->GetTitle(titleUTF16);
1782 CopyUTF16toUTF8(titleUTF16, title);
1783 return title;
1786 NS_IMETHODIMP
1787 MediaTrackGraphImpl::Observe(nsISupports* aSubject, const char* aTopic,
1788 const char16_t* aData) {
1789 MOZ_ASSERT(NS_IsMainThread());
1790 MOZ_ASSERT(strcmp(aTopic, "document-title-changed") == 0);
1791 nsCString streamName = GetDocumentTitle(mWindowID);
1792 LOG(LogLevel::Debug, ("%p: document title: %s", this, streamName.get()));
1793 if (streamName.IsEmpty()) {
1794 return NS_OK;
1796 QueueControlMessageWithNoShutdown(
1797 [self = RefPtr{this}, this, streamName = std::move(streamName)] {
1798 CurrentDriver()->SetStreamName(streamName);
1800 return NS_OK;
1803 bool MediaTrackGraphImpl::AddShutdownBlocker() {
1804 MOZ_ASSERT(NS_IsMainThread());
1805 MOZ_ASSERT(!mShutdownBlocker);
1807 class Blocker : public media::ShutdownBlocker {
1808 const RefPtr<MediaTrackGraphImpl> mGraph;
1810 public:
1811 Blocker(MediaTrackGraphImpl* aGraph, const nsString& aName)
1812 : media::ShutdownBlocker(aName), mGraph(aGraph) {}
1814 NS_IMETHOD
1815 BlockShutdown(nsIAsyncShutdownClient* aProfileBeforeChange) override {
1816 mGraph->ForceShutDown();
1817 return NS_OK;
1821 nsCOMPtr<nsIAsyncShutdownClient> barrier = media::GetShutdownBarrier();
1822 if (!barrier) {
1823 // We're already shutting down, we won't be able to add a blocker, bail.
1824 LOG(LogLevel::Error,
1825 ("%p: Couldn't get shutdown barrier, won't add shutdown blocker",
1826 this));
1827 return false;
1830 // Blocker names must be distinct.
1831 nsString blockerName;
1832 blockerName.AppendPrintf("MediaTrackGraph %p shutdown", this);
1833 mShutdownBlocker = MakeAndAddRef<Blocker>(this, blockerName);
1834 nsresult rv = barrier->AddBlocker(mShutdownBlocker,
1835 NS_LITERAL_STRING_FROM_CSTRING(__FILE__),
1836 __LINE__, u"MediaTrackGraph shutdown"_ns);
1837 MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
1838 return true;
1841 void MediaTrackGraphImpl::RemoveShutdownBlocker() {
1842 if (!mShutdownBlocker) {
1843 return;
1845 media::MustGetShutdownBarrier()->RemoveBlocker(mShutdownBlocker);
1846 mShutdownBlocker = nullptr;
1849 NS_IMETHODIMP
1850 MediaTrackGraphImpl::GetName(nsACString& aName) {
1851 aName.AssignLiteral("MediaTrackGraphImpl");
1852 return NS_OK;
1855 namespace {
1857 class MediaTrackGraphShutDownRunnable : public Runnable {
1858 public:
1859 explicit MediaTrackGraphShutDownRunnable(MediaTrackGraphImpl* aGraph)
1860 : Runnable("MediaTrackGraphShutDownRunnable"), mGraph(aGraph) {}
1861 // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.
1862 // See bug 1535398.
1863 MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override {
1864 TRACE("MTG::MediaTrackGraphShutDownRunnable runnable");
1865 MOZ_ASSERT(NS_IsMainThread());
1866 MOZ_ASSERT(!mGraph->mGraphDriverRunning && mGraph->mDriver,
1867 "We should know the graph thread control loop isn't running!");
1869 LOG(LogLevel::Debug, ("%p: Shutting down graph", mGraph.get()));
1871 // We've asserted the graph isn't running. Use mDriver instead of
1872 // CurrentDriver to avoid thread-safety checks
1873 #if 0 // AudioCallbackDrivers are released asynchronously anyways
1874 // XXX a better test would be have setting mGraphDriverRunning make sure
1875 // any current callback has finished and block future ones -- or just
1876 // handle it all in Shutdown()!
1877 if (mGraph->mDriver->AsAudioCallbackDriver()) {
1878 MOZ_ASSERT(!mGraph->mDriver->AsAudioCallbackDriver()->InCallback());
1880 #endif
1882 for (MediaTrackGraphImpl::PendingResumeOperation& op :
1883 mGraph->mPendingResumeOperations) {
1884 op.Abort();
1887 if (mGraph->mGraphRunner) {
1888 RefPtr<GraphRunner>(mGraph->mGraphRunner)->Shutdown();
1891 RefPtr<GraphDriver>(mGraph->mDriver)->Shutdown();
1893 // Release the driver now so that an AudioCallbackDriver will release its
1894 // SharedThreadPool reference. Each SharedThreadPool reference must be
1895 // released before SharedThreadPool::SpinUntilEmpty() runs on
1896 // xpcom-shutdown-threads. Don't wait for GC/CC to release references to
1897 // objects owning tracks, or for expiration of mGraph->mShutdownTimer,
1898 // which won't otherwise release its reference on the graph until
1899 // nsTimerImpl::Shutdown(), which runs after xpcom-shutdown-threads.
1900 mGraph->SetCurrentDriver(nullptr);
1902 // Safe to access these without the monitor since the graph isn't running.
1903 // We may be one of several graphs. Drop ticket to eventually unblock
1904 // shutdown.
1905 if (mGraph->mShutdownTimer && !mGraph->mShutdownBlocker) {
1906 MOZ_ASSERT(
1907 false,
1908 "AudioCallbackDriver took too long to shut down and we let shutdown"
1909 " continue - freezing and leaking");
1911 // The timer fired, so we may be deeper in shutdown now. Block any
1912 // further teardown and just leak, for safety.
1913 return NS_OK;
1916 // mGraph's thread is not running so it's OK to do whatever here
1917 for (MediaTrack* track : mGraph->AllTracks()) {
1918 // Clean up all MediaSegments since we cannot release Images too
1919 // late during shutdown. Also notify listeners that they were removed
1920 // so they can clean up any gfx resources.
1921 track->RemoveAllResourcesAndListenersImpl();
1924 #ifdef DEBUG
1926 MonitorAutoLock lock(mGraph->mMonitor);
1927 MOZ_ASSERT(mGraph->mUpdateRunnables.IsEmpty());
1929 #endif
1930 mGraph->mPendingUpdateRunnables.Clear();
1932 mGraph->RemoveShutdownBlocker();
1934 // We can't block past the final LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION
1935 // stage, since completion of that stage requires all tracks to be freed,
1936 // which requires shutdown to proceed.
1938 if (mGraph->IsEmpty()) {
1939 // mGraph is no longer needed, so delete it.
1940 mGraph->Destroy();
1941 } else {
1942 // The graph is not empty. We must be in a forced shutdown.
1943 // Some later AppendMessage will detect that the graph has
1944 // been emptied, and delete it.
1945 NS_ASSERTION(mGraph->mForceShutDownReceived, "Not in forced shutdown?");
1946 mGraph->LifecycleStateRef() =
1947 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION;
1949 return NS_OK;
1952 private:
1953 RefPtr<MediaTrackGraphImpl> mGraph;
1956 class MediaTrackGraphStableStateRunnable : public Runnable {
1957 public:
1958 explicit MediaTrackGraphStableStateRunnable(MediaTrackGraphImpl* aGraph,
1959 bool aSourceIsMTG)
1960 : Runnable("MediaTrackGraphStableStateRunnable"),
1961 mGraph(aGraph),
1962 mSourceIsMTG(aSourceIsMTG) {}
1963 NS_IMETHOD Run() override {
1964 TRACE("MTG::MediaTrackGraphStableStateRunnable ControlMessage");
1965 if (mGraph) {
1966 mGraph->RunInStableState(mSourceIsMTG);
1968 return NS_OK;
1971 private:
1972 RefPtr<MediaTrackGraphImpl> mGraph;
1973 bool mSourceIsMTG;
1977 * Control messages forwarded from main thread to graph manager thread
1979 class CreateMessage : public ControlMessage {
1980 public:
1981 explicit CreateMessage(MediaTrack* aTrack) : ControlMessage(aTrack) {}
1982 void Run() override {
1983 TRACE("MTG::AddTrackGraphThread ControlMessage");
1984 mTrack->GraphImpl()->AddTrackGraphThread(mTrack);
1986 void RunDuringShutdown() override {
1987 // Make sure to run this message during shutdown too, to make sure
1988 // that we balance the number of tracks registered with the graph
1989 // as they're destroyed during shutdown.
1990 Run();
1994 } // namespace
1996 void MediaTrackGraphImpl::RunInStableState(bool aSourceIsMTG) {
1997 MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
1999 nsTArray<nsCOMPtr<nsIRunnable>> runnables;
2000 // When we're doing a forced shutdown, pending control messages may be
2001 // run on the main thread via RunDuringShutdown. Those messages must
2002 // run without the graph monitor being held. So, we collect them here.
2003 nsTArray<UniquePtr<ControlMessageInterface>>
2004 controlMessagesToRunDuringShutdown;
2007 MonitorAutoLock lock(mMonitor);
2008 if (aSourceIsMTG) {
2009 MOZ_ASSERT(mPostedRunInStableStateEvent);
2010 mPostedRunInStableStateEvent = false;
2013 // This should be kept in sync with the LifecycleState enum in
2014 // MediaTrackGraphImpl.h
2015 const char* LifecycleState_str[] = {
2016 "LIFECYCLE_THREAD_NOT_STARTED", "LIFECYCLE_RUNNING",
2017 "LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP",
2018 "LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN",
2019 "LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION"};
2021 if (LifecycleStateRef() != LIFECYCLE_RUNNING) {
2022 LOG(LogLevel::Debug,
2023 ("%p: Running stable state callback. Current state: %s", this,
2024 LifecycleState_str[LifecycleStateRef()]));
2027 runnables = std::move(mUpdateRunnables);
2028 for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) {
2029 TrackUpdate* update = &mTrackUpdates[i];
2030 if (update->mTrack) {
2031 ApplyTrackUpdate(update);
2034 mTrackUpdates.Clear();
2036 mMainThreadGraphTime = mNextMainThreadGraphTime;
2038 if (mCurrentTaskMessageQueue.IsEmpty()) {
2039 if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP &&
2040 IsEmpty()) {
2041 // Complete shutdown. First, ensure that this graph is no longer used.
2042 // A new graph graph will be created if one is needed.
2043 // Asynchronously clean up old graph. We don't want to do this
2044 // synchronously because it spins the event loop waiting for threads
2045 // to shut down, and we don't want to do that in a stable state handler.
2046 LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
2047 LOG(LogLevel::Debug,
2048 ("%p: Sending MediaTrackGraphShutDownRunnable", this));
2049 nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this);
2050 mMainThread->Dispatch(event.forget());
2052 } else {
2053 if (LifecycleStateRef() <= LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
2054 MessageBlock* block = mBackMessageQueue.AppendElement();
2055 block->mMessages = std::move(mCurrentTaskMessageQueue);
2056 EnsureNextIteration();
2059 // If this MediaTrackGraph has entered regular (non-forced) shutdown it
2060 // is not able to process any more messages. Those messages being added to
2061 // the graph in the first place is an error.
2062 MOZ_DIAGNOSTIC_ASSERT(LifecycleStateRef() <
2063 LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP ||
2064 mForceShutDownReceived);
2067 if (LifecycleStateRef() == LIFECYCLE_THREAD_NOT_STARTED) {
2068 // Start the driver now. We couldn't start it earlier because the graph
2069 // might exit immediately on finding it has no tracks. The first message
2070 // for a new graph must create a track. Ensure that his message runs on
2071 // the first iteration.
2072 MOZ_ASSERT(MessagesQueued());
2073 SwapMessageQueues();
2075 LOG(LogLevel::Debug,
2076 ("%p: Starting a graph with a %s", this,
2077 CurrentDriver()->AsAudioCallbackDriver() ? "AudioCallbackDriver"
2078 : "SystemClockDriver"));
2079 LifecycleStateRef() = LIFECYCLE_RUNNING;
2080 mGraphDriverRunning = true;
2081 RefPtr<GraphDriver> driver = CurrentDriver();
2082 driver->Start();
2083 // It's not safe to Shutdown() a thread from StableState, and
2084 // releasing this may shutdown a SystemClockDriver thread.
2085 // Proxy the release to outside of StableState.
2086 NS_ReleaseOnMainThread("MediaTrackGraphImpl::CurrentDriver",
2087 driver.forget(),
2088 true); // always proxy
2091 if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP &&
2092 mForceShutDownReceived) {
2093 // Defer calls to RunDuringShutdown() to happen while mMonitor is not
2094 // held.
2095 for (uint32_t i = 0; i < mBackMessageQueue.Length(); ++i) {
2096 MessageBlock& mb = mBackMessageQueue[i];
2097 controlMessagesToRunDuringShutdown.AppendElements(
2098 std::move(mb.mMessages));
2100 mBackMessageQueue.Clear();
2101 MOZ_ASSERT(mCurrentTaskMessageQueue.IsEmpty());
2102 // Stop MediaTrackGraph threads.
2103 LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
2104 nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this);
2105 mMainThread->Dispatch(event.forget());
2108 mGraphDriverRunning = LifecycleStateRef() == LIFECYCLE_RUNNING;
2111 // Make sure we get a new current time in the next event loop task
2112 if (!aSourceIsMTG) {
2113 MOZ_ASSERT(mPostedRunInStableState);
2114 mPostedRunInStableState = false;
2117 for (uint32_t i = 0; i < controlMessagesToRunDuringShutdown.Length(); ++i) {
2118 controlMessagesToRunDuringShutdown[i]->RunDuringShutdown();
2121 #ifdef DEBUG
2122 mCanRunMessagesSynchronously =
2123 !mGraphDriverRunning &&
2124 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
2125 #endif
2127 for (uint32_t i = 0; i < runnables.Length(); ++i) {
2128 runnables[i]->Run();
2132 void MediaTrackGraphImpl::EnsureRunInStableState() {
2133 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
2135 if (mPostedRunInStableState) return;
2136 mPostedRunInStableState = true;
2137 nsCOMPtr<nsIRunnable> event =
2138 new MediaTrackGraphStableStateRunnable(this, false);
2139 nsContentUtils::RunInStableState(event.forget());
2142 void MediaTrackGraphImpl::EnsureStableStateEventPosted() {
2143 MOZ_ASSERT(OnGraphThread());
2144 mMonitor.AssertCurrentThreadOwns();
2146 if (mPostedRunInStableStateEvent) return;
2147 mPostedRunInStableStateEvent = true;
2148 nsCOMPtr<nsIRunnable> event =
2149 new MediaTrackGraphStableStateRunnable(this, true);
2150 mMainThread->Dispatch(event.forget());
2153 void MediaTrackGraphImpl::SignalMainThreadCleanup() {
2154 MOZ_ASSERT(mDriver->OnThread());
2156 MonitorAutoLock lock(mMonitor);
2157 // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
2158 // graphs that have not started.
2159 MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING);
2160 LOG(LogLevel::Debug,
2161 ("%p: MediaTrackGraph waiting for main thread cleanup", this));
2162 LifecycleStateRef() =
2163 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP;
2164 EnsureStableStateEventPosted();
2167 void MediaTrackGraphImpl::AppendMessage(
2168 UniquePtr<ControlMessageInterface> aMessage) {
2169 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
2170 MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0 || mMainThreadPortCount > 0);
2172 if (!mGraphDriverRunning &&
2173 LifecycleStateRef() > LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
2174 // The graph control loop is not running and main thread cleanup has
2175 // happened. From now on we can't append messages to
2176 // mCurrentTaskMessageQueue, because that will never be processed again, so
2177 // just RunDuringShutdown this message. This should only happen during
2178 // forced shutdown, or after a non-realtime graph has finished processing.
2179 #ifdef DEBUG
2180 MOZ_ASSERT(mCanRunMessagesSynchronously);
2181 mCanRunMessagesSynchronously = false;
2182 #endif
2183 aMessage->RunDuringShutdown();
2184 #ifdef DEBUG
2185 mCanRunMessagesSynchronously = true;
2186 #endif
2187 if (IsEmpty() &&
2188 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION) {
2189 Destroy();
2191 return;
2194 mCurrentTaskMessageQueue.AppendElement(std::move(aMessage));
2195 EnsureRunInStableState();
2198 void MediaTrackGraphImpl::Dispatch(already_AddRefed<nsIRunnable>&& aRunnable) {
2199 mMainThread->Dispatch(std::move(aRunnable));
2202 MediaTrack::MediaTrack(TrackRate aSampleRate, MediaSegment::Type aType,
2203 MediaSegment* aSegment)
2204 : mSampleRate(aSampleRate),
2205 mType(aType),
2206 mSegment(aSegment),
2207 mStartTime(0),
2208 mForgottenTime(0),
2209 mEnded(false),
2210 mNotifiedEnded(false),
2211 mDisabledMode(DisabledTrackMode::ENABLED),
2212 mStartBlocking(GRAPH_TIME_MAX),
2213 mSuspendedCount(0),
2214 mMainThreadCurrentTime(0),
2215 mMainThreadEnded(false),
2216 mEndedNotificationSent(false),
2217 mMainThreadDestroyed(false),
2218 mGraph(nullptr) {
2219 MOZ_COUNT_CTOR(MediaTrack);
2220 MOZ_ASSERT_IF(mSegment, mSegment->GetType() == aType);
2223 MediaTrack::~MediaTrack() {
2224 MOZ_COUNT_DTOR(MediaTrack);
2225 NS_ASSERTION(mMainThreadDestroyed, "Should have been destroyed already");
2226 NS_ASSERTION(mMainThreadListeners.IsEmpty(),
2227 "All main thread listeners should have been removed");
2230 size_t MediaTrack::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const {
2231 size_t amount = 0;
2233 // Not owned:
2234 // - mGraph - Not reported here
2235 // - mConsumers - elements
2236 // Future:
2237 // - mLastPlayedVideoFrame
2238 // - mTrackListeners - elements
2240 amount += mTrackListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
2241 amount += mMainThreadListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
2242 amount += mConsumers.ShallowSizeOfExcludingThis(aMallocSizeOf);
2244 return amount;
2247 size_t MediaTrack::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
2248 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
2251 void MediaTrack::IncrementSuspendCount() {
2252 ++mSuspendedCount;
2253 if (mSuspendedCount != 1 || !mGraph) {
2254 MOZ_ASSERT(mGraph || mConsumers.IsEmpty());
2255 return;
2257 AssertOnGraphThreadOrNotRunning();
2258 auto* graph = GraphImpl();
2259 for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
2260 mConsumers[i]->Suspended();
2262 MOZ_ASSERT(graph->mTracks.Contains(this));
2263 graph->mTracks.RemoveElement(this);
2264 graph->mSuspendedTracks.AppendElement(this);
2265 graph->SetTrackOrderDirty();
2268 void MediaTrack::DecrementSuspendCount() {
2269 MOZ_ASSERT(mSuspendedCount > 0, "Suspend count underrun");
2270 --mSuspendedCount;
2271 if (mSuspendedCount != 0 || !mGraph) {
2272 MOZ_ASSERT(mGraph || mConsumers.IsEmpty());
2273 return;
2275 AssertOnGraphThreadOrNotRunning();
2276 auto* graph = GraphImpl();
2277 for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
2278 mConsumers[i]->Resumed();
2280 MOZ_ASSERT(graph->mSuspendedTracks.Contains(this));
2281 graph->mSuspendedTracks.RemoveElement(this);
2282 graph->mTracks.AppendElement(this);
2283 graph->SetTrackOrderDirty();
2286 void ProcessedMediaTrack::DecrementSuspendCount() {
2287 mCycleMarker = NOT_VISITED;
2288 MediaTrack::DecrementSuspendCount();
2291 MediaTrackGraphImpl* MediaTrack::GraphImpl() {
2292 return static_cast<MediaTrackGraphImpl*>(mGraph);
2295 const MediaTrackGraphImpl* MediaTrack::GraphImpl() const {
2296 return static_cast<MediaTrackGraphImpl*>(mGraph);
2299 void MediaTrack::SetGraphImpl(MediaTrackGraphImpl* aGraph) {
2300 MOZ_ASSERT(!mGraph, "Should only be called once");
2301 MOZ_ASSERT(mSampleRate == aGraph->GraphRate());
2302 mGraph = aGraph;
2305 void MediaTrack::SetGraphImpl(MediaTrackGraph* aGraph) {
2306 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(aGraph);
2307 SetGraphImpl(graph);
2310 TrackTime MediaTrack::GraphTimeToTrackTime(GraphTime aTime) const {
2311 NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
2312 aTime <= mStartBlocking,
2313 "Incorrectly ignoring blocking!");
2314 return aTime - mStartTime;
2317 GraphTime MediaTrack::TrackTimeToGraphTime(TrackTime aTime) const {
2318 NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
2319 aTime + mStartTime <= mStartBlocking,
2320 "Incorrectly ignoring blocking!");
2321 return aTime + mStartTime;
2324 TrackTime MediaTrack::GraphTimeToTrackTimeWithBlocking(GraphTime aTime) const {
2325 return GraphImpl()->GraphTimeToTrackTimeWithBlocking(this, aTime);
2328 void MediaTrack::RemoveAllResourcesAndListenersImpl() {
2329 GraphImpl()->AssertOnGraphThreadOrNotRunning();
2331 for (auto& l : mTrackListeners.Clone()) {
2332 l->NotifyRemoved(Graph());
2334 mTrackListeners.Clear();
2336 RemoveAllDirectListenersImpl();
2338 if (mSegment) {
2339 mSegment->Clear();
2343 void MediaTrack::DestroyImpl() {
2344 for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
2345 mConsumers[i]->Disconnect();
2347 if (mSegment) {
2348 mSegment->Clear();
2350 mGraph = nullptr;
2353 void MediaTrack::Destroy() {
2354 // Keep this track alive until we leave this method
2355 RefPtr<MediaTrack> kungFuDeathGrip = this;
2356 // Keep a reference to the graph, since Message might RunDuringShutdown()
2357 // synchronously and make GraphImpl() invalid.
2358 RefPtr<MediaTrackGraphImpl> graph = GraphImpl();
2360 QueueControlOrShutdownMessage(
2361 [self = RefPtr{this}, this](IsInShutdown aInShutdown) {
2362 if (aInShutdown == IsInShutdown::No) {
2363 OnGraphThreadDone();
2365 TRACE("MediaTrack::Destroy ControlMessage");
2366 RemoveAllResourcesAndListenersImpl();
2367 auto* graph = GraphImpl();
2368 DestroyImpl();
2369 graph->RemoveTrackGraphThread(this);
2371 graph->RemoveTrack(this);
2372 // Message::RunDuringShutdown may have removed this track from the graph,
2373 // but our kungFuDeathGrip above will have kept this track alive if
2374 // necessary.
2375 mMainThreadDestroyed = true;
2378 TrackTime MediaTrack::GetEnd() const {
2379 return mSegment ? mSegment->GetDuration() : 0;
2382 void MediaTrack::AddAudioOutput(void* aKey, const AudioDeviceInfo* aSink) {
2383 MOZ_ASSERT(NS_IsMainThread());
2384 AudioDeviceID deviceID = nullptr;
2385 TrackRate preferredSampleRate = 0;
2386 if (aSink) {
2387 deviceID = aSink->DeviceID();
2388 preferredSampleRate = static_cast<TrackRate>(aSink->DefaultRate());
2390 AddAudioOutput(aKey, deviceID, preferredSampleRate);
2393 void MediaTrack::AddAudioOutput(void* aKey, CubebUtils::AudioDeviceID aDeviceID,
2394 TrackRate aPreferredSampleRate) {
2395 MOZ_ASSERT(NS_IsMainThread());
2396 if (mMainThreadDestroyed) {
2397 return;
2399 LOG(LogLevel::Info, ("MediaTrack %p adding AudioOutput", this));
2400 GraphImpl()->RegisterAudioOutput(this, aKey, aDeviceID, aPreferredSampleRate);
2403 void MediaTrackGraphImpl::SetAudioOutputVolume(MediaTrack* aTrack, void* aKey,
2404 float aVolume) {
2405 MOZ_ASSERT(NS_IsMainThread());
2406 for (auto& params : mAudioOutputParams) {
2407 if (params.mKey == aKey && aTrack == params.mTrack) {
2408 params.mVolume = aVolume;
2409 UpdateAudioOutput(aTrack, params.mDeviceID);
2410 return;
2413 MOZ_CRASH("Audio output key not found when setting the volume.");
2416 void MediaTrack::SetAudioOutputVolume(void* aKey, float aVolume) {
2417 if (mMainThreadDestroyed) {
2418 return;
2420 GraphImpl()->SetAudioOutputVolume(this, aKey, aVolume);
2423 void MediaTrack::RemoveAudioOutput(void* aKey) {
2424 MOZ_ASSERT(NS_IsMainThread());
2425 if (mMainThreadDestroyed) {
2426 return;
2428 LOG(LogLevel::Info, ("MediaTrack %p removing AudioOutput", this));
2429 GraphImpl()->UnregisterAudioOutput(this, aKey);
2432 void MediaTrackGraphImpl::RegisterAudioOutput(
2433 MediaTrack* aTrack, void* aKey, CubebUtils::AudioDeviceID aDeviceID,
2434 TrackRate aPreferredSampleRate) {
2435 MOZ_ASSERT(NS_IsMainThread());
2436 MOZ_ASSERT(!mAudioOutputParams.Contains(TrackAndKey{aTrack, aKey}));
2438 IncrementOutputDeviceRefCnt(aDeviceID, aPreferredSampleRate);
2440 mAudioOutputParams.EmplaceBack(
2441 TrackKeyDeviceAndVolume{aTrack, aKey, aDeviceID, 1.f});
2443 UpdateAudioOutput(aTrack, aDeviceID);
2446 void MediaTrackGraphImpl::UnregisterAudioOutput(MediaTrack* aTrack,
2447 void* aKey) {
2448 MOZ_ASSERT(NS_IsMainThread());
2450 size_t index = mAudioOutputParams.IndexOf(TrackAndKey{aTrack, aKey});
2451 MOZ_ASSERT(index != mAudioOutputParams.NoIndex);
2452 AudioDeviceID deviceID = mAudioOutputParams[index].mDeviceID;
2453 mAudioOutputParams.UnorderedRemoveElementAt(index);
2455 UpdateAudioOutput(aTrack, deviceID);
2457 DecrementOutputDeviceRefCnt(deviceID);
2460 void MediaTrackGraphImpl::UpdateAudioOutput(MediaTrack* aTrack,
2461 AudioDeviceID aDeviceID) {
2462 MOZ_ASSERT(NS_IsMainThread());
2463 MOZ_ASSERT(!aTrack->IsDestroyed());
2465 float volume = 0.f;
2466 bool found = false;
2467 for (const auto& params : mAudioOutputParams) {
2468 if (params.mTrack == aTrack && params.mDeviceID == aDeviceID) {
2469 volume += params.mVolume;
2470 found = true;
2474 QueueControlMessageWithNoShutdown(
2475 // track has a strong reference to this.
2476 [track = RefPtr{aTrack}, aDeviceID, volume, found] {
2477 TRACE("MediaTrack::UpdateAudioOutput ControlMessage");
2478 MediaTrackGraphImpl* graph = track->GraphImpl();
2479 auto& outputDevicesRef = graph->mOutputDevices;
2480 size_t deviceIndex = outputDevicesRef.IndexOf(aDeviceID);
2481 MOZ_ASSERT(deviceIndex != outputDevicesRef.NoIndex);
2482 auto& deviceOutputsRef = outputDevicesRef[deviceIndex].mTrackOutputs;
2483 if (found) {
2484 for (auto& outputRef : deviceOutputsRef) {
2485 if (outputRef.mTrack == track) {
2486 outputRef.mVolume = volume;
2487 return;
2490 deviceOutputsRef.EmplaceBack(TrackAndVolume{track, volume});
2491 } else {
2492 DebugOnly<bool> removed = deviceOutputsRef.RemoveElement(track);
2493 MOZ_ASSERT(removed);
2494 // mOutputDevices[0] is retained for AudioCallbackDriver output even
2495 // when no tracks have audio outputs.
2496 if (deviceIndex != 0 && deviceOutputsRef.IsEmpty()) {
2497 // The device is no longer in use.
2498 outputDevicesRef.UnorderedRemoveElementAt(deviceIndex);
2504 void MediaTrackGraphImpl::IncrementOutputDeviceRefCnt(
2505 AudioDeviceID aDeviceID, TrackRate aPreferredSampleRate) {
2506 MOZ_ASSERT(NS_IsMainThread());
2508 for (auto& elementRef : mOutputDeviceRefCnts) {
2509 if (elementRef.mDeviceID == aDeviceID) {
2510 ++elementRef.mRefCnt;
2511 return;
2514 MOZ_ASSERT(aDeviceID != mPrimaryOutputDeviceID,
2515 "mOutputDeviceRefCnts should always have the primary device");
2516 // Need to add an output device.
2517 // Output via another graph for this device.
2518 // This sample rate is not exposed to content.
2519 TrackRate sampleRate =
2520 aPreferredSampleRate != 0
2521 ? aPreferredSampleRate
2522 : static_cast<TrackRate>(CubebUtils::PreferredSampleRate(
2523 /*aShouldResistFingerprinting*/ false));
2524 MediaTrackGraph* newGraph = MediaTrackGraphImpl::GetInstance(
2525 MediaTrackGraph::AUDIO_THREAD_DRIVER, mWindowID, sampleRate, aDeviceID,
2526 GetMainThreadSerialEventTarget());
2527 // CreateCrossGraphReceiver wants the sample rate of this graph.
2528 RefPtr receiver = newGraph->CreateCrossGraphReceiver(mSampleRate);
2529 receiver->AddAudioOutput(nullptr, aDeviceID, sampleRate);
2530 mOutputDeviceRefCnts.EmplaceBack(
2531 DeviceReceiverAndCount{aDeviceID, receiver, 1});
2533 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this, aDeviceID,
2534 receiver = std::move(receiver)]() mutable {
2535 TRACE("MediaTrackGraph add output device ControlMessage");
2536 MOZ_ASSERT(!mOutputDevices.Contains(aDeviceID));
2537 mOutputDevices.EmplaceBack(
2538 OutputDeviceEntry{aDeviceID, std::move(receiver)});
2542 void MediaTrackGraphImpl::DecrementOutputDeviceRefCnt(AudioDeviceID aDeviceID) {
2543 MOZ_ASSERT(NS_IsMainThread());
2545 size_t index = mOutputDeviceRefCnts.IndexOf(aDeviceID);
2546 MOZ_ASSERT(index != mOutputDeviceRefCnts.NoIndex);
2547 // mOutputDeviceRefCnts[0] is retained for consistency with
2548 // mOutputDevices[0], which is retained for AudioCallbackDriver output even
2549 // when no tracks have audio outputs.
2550 if (--mOutputDeviceRefCnts[index].mRefCnt == 0 && index != 0) {
2551 mOutputDeviceRefCnts[index].mReceiver->Destroy();
2552 mOutputDeviceRefCnts.UnorderedRemoveElementAt(index);
2556 void MediaTrack::Suspend() {
2557 // This can happen if this method has been called asynchronously, and the
2558 // track has been destroyed since then.
2559 if (mMainThreadDestroyed) {
2560 return;
2562 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this] {
2563 TRACE("MediaTrack::IncrementSuspendCount ControlMessage");
2564 IncrementSuspendCount();
2568 void MediaTrack::Resume() {
2569 // This can happen if this method has been called asynchronously, and the
2570 // track has been destroyed since then.
2571 if (mMainThreadDestroyed) {
2572 return;
2574 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this] {
2575 TRACE("MediaTrack::DecrementSuspendCount ControlMessage");
2576 DecrementSuspendCount();
2580 void MediaTrack::AddListenerImpl(
2581 already_AddRefed<MediaTrackListener> aListener) {
2582 RefPtr<MediaTrackListener> l(aListener);
2583 mTrackListeners.AppendElement(std::move(l));
2585 PrincipalHandle lastPrincipalHandle = mSegment->GetLastPrincipalHandle();
2586 mTrackListeners.LastElement()->NotifyPrincipalHandleChanged(
2587 Graph(), lastPrincipalHandle);
2588 if (mNotifiedEnded) {
2589 mTrackListeners.LastElement()->NotifyEnded(Graph());
2591 if (CombinedDisabledMode() == DisabledTrackMode::SILENCE_BLACK) {
2592 mTrackListeners.LastElement()->NotifyEnabledStateChanged(Graph(), false);
2596 void MediaTrack::AddListener(MediaTrackListener* aListener) {
2597 MOZ_ASSERT(mSegment, "Segment-less tracks do not support listeners");
2598 if (mMainThreadDestroyed) {
2599 return;
2601 QueueControlMessageWithNoShutdown(
2602 [self = RefPtr{this}, this, listener = RefPtr{aListener}]() mutable {
2603 TRACE("MediaTrack::AddListenerImpl ControlMessage");
2604 AddListenerImpl(listener.forget());
2608 void MediaTrack::RemoveListenerImpl(MediaTrackListener* aListener) {
2609 for (size_t i = 0; i < mTrackListeners.Length(); ++i) {
2610 if (mTrackListeners[i] == aListener) {
2611 mTrackListeners[i]->NotifyRemoved(Graph());
2612 mTrackListeners.RemoveElementAt(i);
2613 return;
2618 RefPtr<GenericPromise> MediaTrack::RemoveListener(
2619 MediaTrackListener* aListener) {
2620 MozPromiseHolder<GenericPromise> promiseHolder;
2621 RefPtr<GenericPromise> p = promiseHolder.Ensure(__func__);
2622 if (mMainThreadDestroyed) {
2623 promiseHolder.Reject(NS_ERROR_FAILURE, __func__);
2624 return p;
2626 QueueControlOrShutdownMessage(
2627 [self = RefPtr{this}, this, listener = RefPtr{aListener},
2628 promiseHolder = std::move(promiseHolder)](IsInShutdown) mutable {
2629 TRACE("MediaTrack::RemoveListenerImpl ControlMessage");
2630 // During shutdown we still want the listener's NotifyRemoved to be
2631 // called, since not doing that might block shutdown of other modules.
2632 RemoveListenerImpl(listener);
2633 promiseHolder.Resolve(true, __func__);
2635 return p;
2638 void MediaTrack::AddDirectListenerImpl(
2639 already_AddRefed<DirectMediaTrackListener> aListener) {
2640 AssertOnGraphThread();
2641 // Base implementation, for tracks that don't support direct track listeners.
2642 RefPtr<DirectMediaTrackListener> listener = aListener;
2643 listener->NotifyDirectListenerInstalled(
2644 DirectMediaTrackListener::InstallationResult::TRACK_NOT_SUPPORTED);
2647 void MediaTrack::AddDirectListener(DirectMediaTrackListener* aListener) {
2648 if (mMainThreadDestroyed) {
2649 return;
2651 QueueControlMessageWithNoShutdown(
2652 [self = RefPtr{this}, this, listener = RefPtr{aListener}]() mutable {
2653 TRACE("MediaTrack::AddDirectListenerImpl ControlMessage");
2654 AddDirectListenerImpl(listener.forget());
2658 void MediaTrack::RemoveDirectListenerImpl(DirectMediaTrackListener* aListener) {
2659 // Base implementation, the listener was never added so nothing to do.
2662 void MediaTrack::RemoveDirectListener(DirectMediaTrackListener* aListener) {
2663 if (mMainThreadDestroyed) {
2664 return;
2666 QueueControlOrShutdownMessage(
2667 [self = RefPtr{this}, this, listener = RefPtr{aListener}](IsInShutdown) {
2668 TRACE("MediaTrack::RemoveDirectListenerImpl ControlMessage");
2669 // During shutdown we still want the listener's
2670 // NotifyDirectListenerUninstalled to be called, since not doing that
2671 // might block shutdown of other modules.
2672 RemoveDirectListenerImpl(listener);
2676 void MediaTrack::RunAfterPendingUpdates(
2677 already_AddRefed<nsIRunnable> aRunnable) {
2678 MOZ_ASSERT(NS_IsMainThread());
2679 if (mMainThreadDestroyed) {
2680 return;
2682 QueueControlOrShutdownMessage(
2683 [self = RefPtr{this}, this,
2684 runnable = nsCOMPtr{aRunnable}](IsInShutdown aInShutdown) mutable {
2685 TRACE("MediaTrack::DispatchToMainThreadStableState ControlMessage");
2686 if (aInShutdown == IsInShutdown::No) {
2687 Graph()->DispatchToMainThreadStableState(runnable.forget());
2688 } else {
2689 // Don't run mRunnable now as it may call AppendMessage() which would
2690 // assume that there are no remaining
2691 // controlMessagesToRunDuringShutdown.
2692 MOZ_ASSERT(NS_IsMainThread());
2693 GraphImpl()->Dispatch(runnable.forget());
2698 void MediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) {
2699 AssertOnGraphThread();
2700 MOZ_DIAGNOSTIC_ASSERT(
2701 aMode == DisabledTrackMode::ENABLED ||
2702 mDisabledMode == DisabledTrackMode::ENABLED,
2703 "Changing disabled track mode for a track is not allowed");
2704 DisabledTrackMode oldMode = CombinedDisabledMode();
2705 mDisabledMode = aMode;
2706 NotifyIfDisabledModeChangedFrom(oldMode);
2709 void MediaTrack::SetDisabledTrackMode(DisabledTrackMode aMode) {
2710 if (mMainThreadDestroyed) {
2711 return;
2713 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this, aMode]() {
2714 TRACE("MediaTrack::SetDisabledTrackModeImpl ControlMessage");
2715 SetDisabledTrackModeImpl(aMode);
2719 void MediaTrack::ApplyTrackDisabling(MediaSegment* aSegment,
2720 MediaSegment* aRawSegment) {
2721 AssertOnGraphThread();
2722 mozilla::ApplyTrackDisabling(mDisabledMode, aSegment, aRawSegment);
2725 void MediaTrack::AddMainThreadListener(
2726 MainThreadMediaTrackListener* aListener) {
2727 MOZ_ASSERT(NS_IsMainThread());
2728 MOZ_ASSERT(aListener);
2729 MOZ_ASSERT(!mMainThreadListeners.Contains(aListener));
2731 mMainThreadListeners.AppendElement(aListener);
2733 // If it is not yet time to send the notification, then exit here.
2734 if (!mEndedNotificationSent) {
2735 return;
2738 class NotifyRunnable final : public Runnable {
2739 public:
2740 explicit NotifyRunnable(MediaTrack* aTrack)
2741 : Runnable("MediaTrack::NotifyRunnable"), mTrack(aTrack) {}
2743 NS_IMETHOD Run() override {
2744 TRACE("MediaTrack::NotifyMainThreadListeners Runnable");
2745 MOZ_ASSERT(NS_IsMainThread());
2746 mTrack->NotifyMainThreadListeners();
2747 return NS_OK;
2750 private:
2751 ~NotifyRunnable() = default;
2753 RefPtr<MediaTrack> mTrack;
2756 nsCOMPtr<nsIRunnable> runnable = new NotifyRunnable(this);
2757 GraphImpl()->Dispatch(runnable.forget());
2760 void MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
2761 GraphTime aBlockedTime) {
2762 mStartTime += aBlockedTime;
2764 if (!mSegment) {
2765 // No data to be forgotten.
2766 return;
2769 TrackTime time = aCurrentTime - mStartTime;
2770 // Only prune if there is a reasonable chunk (50ms) to forget, so we don't
2771 // spend too much time pruning segments.
2772 const TrackTime minChunkSize = mSampleRate * 50 / 1000;
2773 if (time < mForgottenTime + minChunkSize) {
2774 return;
2777 mForgottenTime = std::min(GetEnd() - 1, time);
2778 mSegment->ForgetUpTo(mForgottenTime);
2781 void MediaTrack::NotifyIfDisabledModeChangedFrom(DisabledTrackMode aOldMode) {
2782 DisabledTrackMode mode = CombinedDisabledMode();
2783 if (aOldMode == mode) {
2784 return;
2787 for (const auto& listener : mTrackListeners) {
2788 listener->NotifyEnabledStateChanged(
2789 Graph(), mode != DisabledTrackMode::SILENCE_BLACK);
2792 for (const auto& c : mConsumers) {
2793 if (c->GetDestination()) {
2794 c->GetDestination()->OnInputDisabledModeChanged(mode);
2799 void MediaTrack::QueueMessage(UniquePtr<ControlMessageInterface> aMessage) {
2800 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
2801 MOZ_RELEASE_ASSERT(!IsDestroyed());
2802 GraphImpl()->AppendMessage(std::move(aMessage));
2805 void MediaTrack::RunMessageAfterProcessing(
2806 UniquePtr<ControlMessageInterface> aMessage) {
2807 AssertOnGraphThread();
2808 GraphImpl()->RunMessageAfterProcessing(std::move(aMessage));
2811 SourceMediaTrack::SourceMediaTrack(MediaSegment::Type aType,
2812 TrackRate aSampleRate)
2813 : MediaTrack(aSampleRate, aType,
2814 aType == MediaSegment::AUDIO
2815 ? static_cast<MediaSegment*>(new AudioSegment())
2816 : static_cast<MediaSegment*>(new VideoSegment())),
2817 mMutex("mozilla::media::SourceMediaTrack") {
2818 mUpdateTrack = MakeUnique<TrackData>();
2819 mUpdateTrack->mInputRate = aSampleRate;
2820 mUpdateTrack->mResamplerChannelCount = 0;
2821 mUpdateTrack->mData = UniquePtr<MediaSegment>(mSegment->CreateEmptyClone());
2822 mUpdateTrack->mEnded = false;
2823 mUpdateTrack->mPullingEnabled = false;
2824 mUpdateTrack->mGraphThreadDone = false;
2827 void SourceMediaTrack::DestroyImpl() {
2828 GraphImpl()->AssertOnGraphThreadOrNotRunning();
2829 for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
2830 // Disconnect before we come under mMutex's lock since it can call back
2831 // through RemoveDirectListenerImpl() and deadlock.
2832 mConsumers[i]->Disconnect();
2835 // Hold mMutex while mGraph is reset so that other threads holding mMutex
2836 // can null-check know that the graph will not destroyed.
2837 MutexAutoLock lock(mMutex);
2838 mUpdateTrack = nullptr;
2839 MediaTrack::DestroyImpl();
2842 void SourceMediaTrack::SetPullingEnabled(bool aEnabled) {
2843 class Message : public ControlMessage {
2844 public:
2845 Message(SourceMediaTrack* aTrack, bool aEnabled)
2846 : ControlMessage(nullptr), mTrack(aTrack), mEnabled(aEnabled) {}
2847 void Run() override {
2848 TRACE("SourceMediaTrack::SetPullingEnabled ControlMessage");
2849 MutexAutoLock lock(mTrack->mMutex);
2850 if (!mTrack->mUpdateTrack) {
2851 // We can't enable pulling for a track that has ended. We ignore
2852 // this if we're disabling pulling, since shutdown sequences are
2853 // complex. If there's truly an issue we'll have issues enabling anyway.
2854 MOZ_ASSERT_IF(mEnabled, mTrack->mEnded);
2855 return;
2857 MOZ_ASSERT(mTrack->mType == MediaSegment::AUDIO,
2858 "Pulling is not allowed for video");
2859 mTrack->mUpdateTrack->mPullingEnabled = mEnabled;
2861 SourceMediaTrack* mTrack;
2862 bool mEnabled;
2864 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aEnabled));
2867 bool SourceMediaTrack::PullNewData(GraphTime aDesiredUpToTime) {
2868 TRACE_COMMENT("SourceMediaTrack::PullNewData", "%p", this);
2869 TrackTime t;
2870 TrackTime current;
2872 if (mEnded) {
2873 return false;
2875 MutexAutoLock lock(mMutex);
2876 if (mUpdateTrack->mEnded) {
2877 return false;
2879 if (!mUpdateTrack->mPullingEnabled) {
2880 return false;
2882 // Compute how much track time we'll need assuming we don't block
2883 // the track at all.
2884 t = GraphTimeToTrackTime(aDesiredUpToTime);
2885 current = GetEnd() + mUpdateTrack->mData->GetDuration();
2887 if (t <= current) {
2888 return false;
2890 LOG(LogLevel::Verbose, ("%p: Calling NotifyPull track=%p t=%f current end=%f",
2891 GraphImpl(), this, GraphImpl()->MediaTimeToSeconds(t),
2892 GraphImpl()->MediaTimeToSeconds(current)));
2893 for (auto& l : mTrackListeners) {
2894 l->NotifyPull(Graph(), current, t);
2896 return true;
2900 * This moves chunks from aIn to aOut. For audio this is simple. For video
2901 * we carry durations over if present, or extend up to aDesiredUpToTime if not.
2903 * We also handle "resetters" from captured media elements. This type of source
2904 * pushes future frames into the track, and should it need to remove some, e.g.,
2905 * because of a seek or pause, it tells us by letting time go backwards. Without
2906 * this, tracks would be live for too long after a seek or pause.
2908 static void MoveToSegment(SourceMediaTrack* aTrack, MediaSegment* aIn,
2909 MediaSegment* aOut, TrackTime aCurrentTime,
2910 TrackTime aDesiredUpToTime)
2911 MOZ_REQUIRES(aTrack->GetMutex()) {
2912 MOZ_ASSERT(aIn->GetType() == aOut->GetType());
2913 MOZ_ASSERT(aOut->GetDuration() >= aCurrentTime);
2914 MOZ_ASSERT(aDesiredUpToTime >= aCurrentTime);
2915 if (aIn->GetType() == MediaSegment::AUDIO) {
2916 AudioSegment* in = static_cast<AudioSegment*>(aIn);
2917 AudioSegment* out = static_cast<AudioSegment*>(aOut);
2918 TrackTime desiredDurationToMove = aDesiredUpToTime - aCurrentTime;
2919 TrackTime end = std::min(in->GetDuration(), desiredDurationToMove);
2921 out->AppendSlice(*in, 0, end);
2922 in->RemoveLeading(end);
2924 aTrack->GetMutex().AssertCurrentThreadOwns();
2925 out->ApplyVolume(aTrack->GetVolumeLocked());
2926 } else {
2927 VideoSegment* in = static_cast<VideoSegment*>(aIn);
2928 VideoSegment* out = static_cast<VideoSegment*>(aOut);
2929 for (VideoSegment::ConstChunkIterator c(*in); !c.IsEnded(); c.Next()) {
2930 MOZ_ASSERT(!c->mTimeStamp.IsNull());
2931 VideoChunk* last = out->GetLastChunk();
2932 if (!last || last->mTimeStamp.IsNull()) {
2933 // This is the first frame, or the last frame pushed to `out` has been
2934 // all consumed. Just append and we deal with its duration later.
2935 out->AppendFrame(*c);
2936 if (c->GetDuration() > 0) {
2937 out->ExtendLastFrameBy(c->GetDuration());
2939 continue;
2942 // We now know when this frame starts, aka when the last frame ends.
2944 if (c->mTimeStamp < last->mTimeStamp) {
2945 // Time is going backwards. This is a resetting frame from
2946 // DecodedStream. Clear everything up to currentTime.
2947 out->Clear();
2948 out->AppendNullData(aCurrentTime);
2951 // Append the current frame (will have duration 0).
2952 out->AppendFrame(*c);
2953 if (c->GetDuration() > 0) {
2954 out->ExtendLastFrameBy(c->GetDuration());
2957 if (out->GetDuration() < aDesiredUpToTime) {
2958 out->ExtendLastFrameBy(aDesiredUpToTime - out->GetDuration());
2960 in->Clear();
2961 MOZ_ASSERT(aIn->GetDuration() == 0, "aIn must be consumed");
2965 void SourceMediaTrack::ExtractPendingInput(GraphTime aCurrentTime,
2966 GraphTime aDesiredUpToTime) {
2967 MutexAutoLock lock(mMutex);
2969 if (!mUpdateTrack) {
2970 MOZ_ASSERT(mEnded);
2971 return;
2974 TrackTime trackCurrentTime = GraphTimeToTrackTime(aCurrentTime);
2976 ApplyTrackDisabling(mUpdateTrack->mData.get());
2978 if (!mUpdateTrack->mData->IsEmpty()) {
2979 for (const auto& l : mTrackListeners) {
2980 l->NotifyQueuedChanges(GraphImpl(), GetEnd(), *mUpdateTrack->mData);
2983 TrackTime trackDesiredUpToTime = GraphTimeToTrackTime(aDesiredUpToTime);
2984 TrackTime endTime = trackDesiredUpToTime;
2985 if (mUpdateTrack->mEnded) {
2986 endTime = std::min(trackDesiredUpToTime,
2987 GetEnd() + mUpdateTrack->mData->GetDuration());
2989 LOG(LogLevel::Verbose,
2990 ("%p: SourceMediaTrack %p advancing end from %" PRId64 " to %" PRId64,
2991 GraphImpl(), this, int64_t(trackCurrentTime), int64_t(endTime)));
2992 MoveToSegment(this, mUpdateTrack->mData.get(), mSegment.get(),
2993 trackCurrentTime, endTime);
2994 if (mUpdateTrack->mEnded && GetEnd() < trackDesiredUpToTime) {
2995 mEnded = true;
2996 mUpdateTrack = nullptr;
3000 void SourceMediaTrack::ResampleAudioToGraphSampleRate(MediaSegment* aSegment) {
3001 mMutex.AssertCurrentThreadOwns();
3002 if (aSegment->GetType() != MediaSegment::AUDIO ||
3003 mUpdateTrack->mInputRate == GraphImpl()->GraphRate()) {
3004 return;
3006 AudioSegment* segment = static_cast<AudioSegment*>(aSegment);
3007 segment->ResampleChunks(mUpdateTrack->mResampler,
3008 &mUpdateTrack->mResamplerChannelCount,
3009 mUpdateTrack->mInputRate, GraphImpl()->GraphRate());
3012 void SourceMediaTrack::AdvanceTimeVaryingValuesToCurrentTime(
3013 GraphTime aCurrentTime, GraphTime aBlockedTime) {
3014 MutexAutoLock lock(mMutex);
3015 MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(aCurrentTime, aBlockedTime);
3018 void SourceMediaTrack::SetAppendDataSourceRate(TrackRate aRate) {
3019 MutexAutoLock lock(mMutex);
3020 if (!mUpdateTrack) {
3021 return;
3023 MOZ_DIAGNOSTIC_ASSERT(mSegment->GetType() == MediaSegment::AUDIO);
3024 // Set the new input rate and reset the resampler.
3025 mUpdateTrack->mInputRate = aRate;
3026 mUpdateTrack->mResampler.own(nullptr);
3027 mUpdateTrack->mResamplerChannelCount = 0;
3030 TrackTime SourceMediaTrack::AppendData(MediaSegment* aSegment,
3031 MediaSegment* aRawSegment) {
3032 MutexAutoLock lock(mMutex);
3033 MOZ_DIAGNOSTIC_ASSERT(aSegment->GetType() == mType);
3034 TrackTime appended = 0;
3035 if (!mUpdateTrack || mUpdateTrack->mEnded || mUpdateTrack->mGraphThreadDone) {
3036 aSegment->Clear();
3037 return appended;
3040 // Data goes into mData, and on the next iteration of the MTG moves
3041 // into the track's segment after NotifyQueuedTrackChanges(). This adds
3042 // 0-10ms of delay before data gets to direct listeners.
3043 // Indirect listeners (via subsequent TrackUnion nodes) are synced to
3044 // playout time, and so can be delayed by buffering.
3046 // Apply track disabling before notifying any consumers directly
3047 // or inserting into the graph
3048 mozilla::ApplyTrackDisabling(mDirectDisabledMode, aSegment, aRawSegment);
3050 ResampleAudioToGraphSampleRate(aSegment);
3052 // Must notify first, since AppendFrom() will empty out aSegment
3053 NotifyDirectConsumers(aRawSegment ? aRawSegment : aSegment);
3054 appended = aSegment->GetDuration();
3055 mUpdateTrack->mData->AppendFrom(aSegment); // note: aSegment is now dead
3057 auto graph = GraphImpl();
3058 MonitorAutoLock lock(graph->GetMonitor());
3059 if (graph->CurrentDriver()) { // graph has not completed forced shutdown
3060 graph->EnsureNextIteration();
3064 return appended;
3067 TrackTime SourceMediaTrack::ClearFutureData() {
3068 MutexAutoLock lock(mMutex);
3069 auto graph = GraphImpl();
3070 if (!mUpdateTrack || !graph) {
3071 return 0;
3074 TrackTime duration = mUpdateTrack->mData->GetDuration();
3075 mUpdateTrack->mData->Clear();
3076 return duration;
3079 void SourceMediaTrack::NotifyDirectConsumers(MediaSegment* aSegment) {
3080 mMutex.AssertCurrentThreadOwns();
3082 for (const auto& l : mDirectTrackListeners) {
3083 TrackTime offset = 0; // FIX! need a separate TrackTime.... or the end of
3084 // the internal buffer
3085 l->NotifyRealtimeTrackDataAndApplyTrackDisabling(Graph(), offset,
3086 *aSegment);
3090 void SourceMediaTrack::AddDirectListenerImpl(
3091 already_AddRefed<DirectMediaTrackListener> aListener) {
3092 AssertOnGraphThread();
3093 MutexAutoLock lock(mMutex);
3095 RefPtr<DirectMediaTrackListener> listener = aListener;
3096 LOG(LogLevel::Debug,
3097 ("%p: Adding direct track listener %p to source track %p", GraphImpl(),
3098 listener.get(), this));
3100 MOZ_ASSERT(mType == MediaSegment::VIDEO);
3101 for (const auto& l : mDirectTrackListeners) {
3102 if (l == listener) {
3103 listener->NotifyDirectListenerInstalled(
3104 DirectMediaTrackListener::InstallationResult::ALREADY_EXISTS);
3105 return;
3109 mDirectTrackListeners.AppendElement(listener);
3111 LOG(LogLevel::Debug,
3112 ("%p: Added direct track listener %p", GraphImpl(), listener.get()));
3113 listener->NotifyDirectListenerInstalled(
3114 DirectMediaTrackListener::InstallationResult::SUCCESS);
3116 if (mDisabledMode != DisabledTrackMode::ENABLED) {
3117 listener->IncreaseDisabled(mDisabledMode);
3120 if (mEnded) {
3121 return;
3124 // Pass buffered data to the listener
3125 VideoSegment bufferedData;
3126 size_t videoFrames = 0;
3127 VideoSegment& segment = *GetData<VideoSegment>();
3128 for (VideoSegment::ConstChunkIterator iter(segment); !iter.IsEnded();
3129 iter.Next()) {
3130 if (iter->mTimeStamp.IsNull()) {
3131 // No timestamp means this is only for the graph's internal book-keeping,
3132 // denoting a late start of the track.
3133 continue;
3135 ++videoFrames;
3136 bufferedData.AppendFrame(*iter);
3139 VideoSegment& video = static_cast<VideoSegment&>(*mUpdateTrack->mData);
3140 for (VideoSegment::ConstChunkIterator iter(video); !iter.IsEnded();
3141 iter.Next()) {
3142 ++videoFrames;
3143 MOZ_ASSERT(!iter->mTimeStamp.IsNull());
3144 bufferedData.AppendFrame(*iter);
3147 LOG(LogLevel::Info,
3148 ("%p: Notifying direct listener %p of %zu video frames and duration "
3149 "%" PRId64,
3150 GraphImpl(), listener.get(), videoFrames, bufferedData.GetDuration()));
3151 listener->NotifyRealtimeTrackData(Graph(), 0, bufferedData);
3154 void SourceMediaTrack::RemoveDirectListenerImpl(
3155 DirectMediaTrackListener* aListener) {
3156 mGraph->AssertOnGraphThreadOrNotRunning();
3157 MutexAutoLock lock(mMutex);
3158 for (int32_t i = mDirectTrackListeners.Length() - 1; i >= 0; --i) {
3159 const RefPtr<DirectMediaTrackListener>& l = mDirectTrackListeners[i];
3160 if (l == aListener) {
3161 if (mDisabledMode != DisabledTrackMode::ENABLED) {
3162 aListener->DecreaseDisabled(mDisabledMode);
3164 aListener->NotifyDirectListenerUninstalled();
3165 mDirectTrackListeners.RemoveElementAt(i);
3170 void SourceMediaTrack::End() {
3171 MutexAutoLock lock(mMutex);
3172 if (!mUpdateTrack) {
3173 // Already ended
3174 return;
3176 mUpdateTrack->mEnded = true;
3177 if (auto graph = GraphImpl()) {
3178 MonitorAutoLock lock(graph->GetMonitor());
3179 if (graph->CurrentDriver()) { // graph has not completed forced shutdown
3180 graph->EnsureNextIteration();
3185 void SourceMediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) {
3186 AssertOnGraphThread();
3188 MutexAutoLock lock(mMutex);
3189 const DisabledTrackMode oldMode = mDirectDisabledMode;
3190 const bool oldEnabled = oldMode == DisabledTrackMode::ENABLED;
3191 const bool enabled = aMode == DisabledTrackMode::ENABLED;
3192 mDirectDisabledMode = aMode;
3193 for (const auto& l : mDirectTrackListeners) {
3194 if (!oldEnabled && enabled) {
3195 LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting "
3196 "direct listener enabled",
3197 GraphImpl(), this));
3198 l->DecreaseDisabled(oldMode);
3199 } else if (oldEnabled && !enabled) {
3200 LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting "
3201 "direct listener disabled",
3202 GraphImpl(), this));
3203 l->IncreaseDisabled(aMode);
3207 MediaTrack::SetDisabledTrackModeImpl(aMode);
3210 uint32_t SourceMediaTrack::NumberOfChannels() const {
3211 AudioSegment* audio = GetData<AudioSegment>();
3212 MOZ_DIAGNOSTIC_ASSERT(audio);
3213 if (!audio) {
3214 return 0;
3216 return audio->MaxChannelCount();
3219 void SourceMediaTrack::RemoveAllDirectListenersImpl() {
3220 GraphImpl()->AssertOnGraphThreadOrNotRunning();
3221 MutexAutoLock lock(mMutex);
3223 for (auto& l : mDirectTrackListeners.Clone()) {
3224 l->NotifyDirectListenerUninstalled();
3226 mDirectTrackListeners.Clear();
3229 void SourceMediaTrack::SetVolume(float aVolume) {
3230 MutexAutoLock lock(mMutex);
3231 mVolume = aVolume;
3234 float SourceMediaTrack::GetVolumeLocked() {
3235 mMutex.AssertCurrentThreadOwns();
3236 return mVolume;
3239 SourceMediaTrack::~SourceMediaTrack() = default;
3241 void MediaInputPort::Init() {
3242 mGraph->AssertOnGraphThreadOrNotRunning();
3243 LOG(LogLevel::Debug, ("%p: Adding MediaInputPort %p (from %p to %p)", mGraph,
3244 this, mSource, mDest));
3245 // Only connect the port if it wasn't disconnected on allocation.
3246 if (mSource) {
3247 mSource->AddConsumer(this);
3248 mDest->AddInput(this);
3250 // mPortCount decremented via MediaInputPort::Destroy's message
3251 ++mGraph->mPortCount;
3254 void MediaInputPort::Disconnect() {
3255 mGraph->AssertOnGraphThreadOrNotRunning();
3256 NS_ASSERTION(!mSource == !mDest,
3257 "mSource and mDest must either both be null or both non-null");
3259 if (!mSource) {
3260 return;
3263 mSource->RemoveConsumer(this);
3264 mDest->RemoveInput(this);
3265 mSource = nullptr;
3266 mDest = nullptr;
3268 mGraph->SetTrackOrderDirty();
3271 MediaTrack* MediaInputPort::GetSource() const {
3272 mGraph->AssertOnGraphThreadOrNotRunning();
3273 return mSource;
3276 ProcessedMediaTrack* MediaInputPort::GetDestination() const {
3277 mGraph->AssertOnGraphThreadOrNotRunning();
3278 return mDest;
3281 MediaInputPort::InputInterval MediaInputPort::GetNextInputInterval(
3282 MediaInputPort const* aPort, GraphTime aTime) {
3283 InputInterval result = {GRAPH_TIME_MAX, GRAPH_TIME_MAX, false};
3284 if (!aPort) {
3285 result.mStart = aTime;
3286 result.mInputIsBlocked = true;
3287 return result;
3289 aPort->mGraph->AssertOnGraphThreadOrNotRunning();
3290 if (aTime >= aPort->mDest->mStartBlocking) {
3291 return result;
3293 result.mStart = aTime;
3294 result.mEnd = aPort->mDest->mStartBlocking;
3295 result.mInputIsBlocked = aTime >= aPort->mSource->mStartBlocking;
3296 if (!result.mInputIsBlocked) {
3297 result.mEnd = std::min(result.mEnd, aPort->mSource->mStartBlocking);
3299 return result;
3302 void MediaInputPort::Suspended() {
3303 mGraph->AssertOnGraphThreadOrNotRunning();
3304 mDest->InputSuspended(this);
3307 void MediaInputPort::Resumed() {
3308 mGraph->AssertOnGraphThreadOrNotRunning();
3309 mDest->InputResumed(this);
3312 void MediaInputPort::Destroy() {
3313 class Message : public ControlMessage {
3314 public:
3315 explicit Message(MediaInputPort* aPort)
3316 : ControlMessage(nullptr), mPort(aPort) {}
3317 void Run() override {
3318 TRACE("MediaInputPort::Destroy ControlMessage");
3319 mPort->Disconnect();
3320 --mPort->GraphImpl()->mPortCount;
3321 mPort->SetGraphImpl(nullptr);
3322 NS_RELEASE(mPort);
3324 void RunDuringShutdown() override { Run(); }
3325 MediaInputPort* mPort;
3327 // Keep a reference to the graph, since Message might RunDuringShutdown()
3328 // synchronously and make GraphImpl() invalid.
3329 RefPtr<MediaTrackGraphImpl> graph = mGraph;
3330 graph->AppendMessage(MakeUnique<Message>(this));
3331 --graph->mMainThreadPortCount;
3334 MediaTrackGraphImpl* MediaInputPort::GraphImpl() const {
3335 mGraph->AssertOnGraphThreadOrNotRunning();
3336 return mGraph;
3339 MediaTrackGraph* MediaInputPort::Graph() const {
3340 mGraph->AssertOnGraphThreadOrNotRunning();
3341 return mGraph;
3344 void MediaInputPort::SetGraphImpl(MediaTrackGraphImpl* aGraph) {
3345 MOZ_ASSERT(!mGraph || !aGraph, "Should only be set once");
3346 DebugOnly<MediaTrackGraphImpl*> graph = mGraph ? mGraph : aGraph;
3347 MOZ_ASSERT(graph->OnGraphThreadOrNotRunning());
3348 mGraph = aGraph;
3351 already_AddRefed<MediaInputPort> ProcessedMediaTrack::AllocateInputPort(
3352 MediaTrack* aTrack, uint16_t aInputNumber, uint16_t aOutputNumber) {
3353 // This method creates two references to the MediaInputPort: one for
3354 // the main thread, and one for the MediaTrackGraph.
3355 class Message : public ControlMessage {
3356 public:
3357 explicit Message(MediaInputPort* aPort)
3358 : ControlMessage(aPort->mDest), mPort(aPort) {}
3359 void Run() override {
3360 TRACE("ProcessedMediaTrack::AllocateInputPort ControlMessage");
3361 mPort->Init();
3362 // The graph holds its reference implicitly
3363 mPort->GraphImpl()->SetTrackOrderDirty();
3364 Unused << mPort.forget();
3366 void RunDuringShutdown() override { Run(); }
3367 RefPtr<MediaInputPort> mPort;
3370 MOZ_DIAGNOSTIC_ASSERT(aTrack->mType == mType);
3371 RefPtr<MediaInputPort> port;
3372 if (aTrack->IsDestroyed()) {
3373 // Create a port that's disconnected, which is what it'd be after its source
3374 // track is Destroy()ed normally. Disconnect() is idempotent so destroying
3375 // this later is fine.
3376 port = new MediaInputPort(GraphImpl(), nullptr, nullptr, aInputNumber,
3377 aOutputNumber);
3378 } else {
3379 MOZ_ASSERT(aTrack->GraphImpl() == GraphImpl());
3380 port = new MediaInputPort(GraphImpl(), aTrack, this, aInputNumber,
3381 aOutputNumber);
3383 ++GraphImpl()->mMainThreadPortCount;
3384 GraphImpl()->AppendMessage(MakeUnique<Message>(port));
3385 return port.forget();
3388 void ProcessedMediaTrack::QueueSetAutoend(bool aAutoend) {
3389 class Message : public ControlMessage {
3390 public:
3391 Message(ProcessedMediaTrack* aTrack, bool aAutoend)
3392 : ControlMessage(aTrack), mAutoend(aAutoend) {}
3393 void Run() override {
3394 TRACE("ProcessedMediaTrack::SetAutoendImpl ControlMessage");
3395 static_cast<ProcessedMediaTrack*>(mTrack)->SetAutoendImpl(mAutoend);
3397 bool mAutoend;
3399 if (mMainThreadDestroyed) {
3400 return;
3402 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aAutoend));
3405 void ProcessedMediaTrack::DestroyImpl() {
3406 for (int32_t i = mInputs.Length() - 1; i >= 0; --i) {
3407 mInputs[i]->Disconnect();
3410 for (int32_t i = mSuspendedInputs.Length() - 1; i >= 0; --i) {
3411 mSuspendedInputs[i]->Disconnect();
3414 MediaTrack::DestroyImpl();
3415 // The track order is only important if there are connections, in which
3416 // case MediaInputPort::Disconnect() called SetTrackOrderDirty().
3417 // MediaTrackGraphImpl::RemoveTrackGraphThread() will also call
3418 // SetTrackOrderDirty(), for other reasons.
3421 MediaTrackGraphImpl::MediaTrackGraphImpl(uint64_t aWindowID,
3422 TrackRate aSampleRate,
3423 AudioDeviceID aPrimaryOutputDeviceID,
3424 nsISerialEventTarget* aMainThread)
3425 : MediaTrackGraph(aSampleRate, aPrimaryOutputDeviceID),
3426 mWindowID(aWindowID),
3427 mFirstCycleBreaker(0)
3428 // An offline graph is not initially processing.
3430 mPortCount(0),
3431 mMonitor("MediaTrackGraphImpl"),
3432 mLifecycleState(LIFECYCLE_THREAD_NOT_STARTED),
3433 mPostedRunInStableStateEvent(false),
3434 mGraphDriverRunning(false),
3435 mPostedRunInStableState(false),
3436 mTrackOrderDirty(false),
3437 mMainThread(aMainThread),
3438 mGlobalVolume(CubebUtils::GetVolumeScale())
3439 #ifdef DEBUG
3441 mCanRunMessagesSynchronously(false)
3442 #endif
3444 mMainThreadGraphTime(0, "MediaTrackGraphImpl::mMainThreadGraphTime"),
3445 mAudioOutputLatency(0.0),
3446 mMaxOutputChannelCount(CubebUtils::MaxNumberOfChannels()) {
3449 void MediaTrackGraphImpl::Init(GraphDriverType aDriverRequested,
3450 GraphRunType aRunTypeRequested,
3451 uint32_t aChannelCount) {
3452 mSelfRef = this;
3453 mEndTime = aDriverRequested == OFFLINE_THREAD_DRIVER ? 0 : GRAPH_TIME_MAX;
3454 mRealtime = aDriverRequested != OFFLINE_THREAD_DRIVER;
3455 // The primary output device always exists because an AudioCallbackDriver
3456 // may exist, and want to be fed data, even when no tracks have audio
3457 // outputs.
3458 mOutputDeviceRefCnts.EmplaceBack(
3459 DeviceReceiverAndCount{mPrimaryOutputDeviceID, nullptr, 0});
3460 mOutputDevices.EmplaceBack(OutputDeviceEntry{mPrimaryOutputDeviceID});
3462 bool failedToGetShutdownBlocker = false;
3463 if (!IsNonRealtime()) {
3464 failedToGetShutdownBlocker = !AddShutdownBlocker();
3467 mGraphRunner = aRunTypeRequested == SINGLE_THREAD
3468 ? GraphRunner::Create(this)
3469 : already_AddRefed<GraphRunner>(nullptr);
3471 if ((aRunTypeRequested == SINGLE_THREAD && !mGraphRunner) ||
3472 failedToGetShutdownBlocker) {
3473 MonitorAutoLock lock(mMonitor);
3474 // At least one of the following happened
3475 // - Failed to create thread.
3476 // - Failed to install a shutdown blocker when one is needed.
3477 // Because we have a fail state, jump to last phase of the lifecycle.
3478 mLifecycleState = LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION;
3479 RemoveShutdownBlocker(); // No-op if blocker wasn't added.
3480 #ifdef DEBUG
3481 mCanRunMessagesSynchronously = true;
3482 #endif
3483 return;
3485 if (mRealtime) {
3486 if (aDriverRequested == AUDIO_THREAD_DRIVER) {
3487 // Always start with zero input channels, and no particular preferences
3488 // for the input channel.
3489 mDriver = new AudioCallbackDriver(
3490 this, nullptr, mSampleRate, aChannelCount, 0, PrimaryOutputDeviceID(),
3491 nullptr, AudioInputType::Unknown, Nothing());
3492 } else {
3493 mDriver = new SystemClockDriver(this, nullptr, mSampleRate);
3495 nsCString streamName = GetDocumentTitle(mWindowID);
3496 LOG(LogLevel::Debug, ("%p: document title: %s", this, streamName.get()));
3497 mDriver->SetStreamName(streamName);
3498 } else {
3499 mDriver =
3500 new OfflineClockDriver(this, mSampleRate, MEDIA_GRAPH_TARGET_PERIOD_MS);
3503 mLastMainThreadUpdate = TimeStamp::Now();
3505 RegisterWeakAsyncMemoryReporter(this);
3508 #ifdef DEBUG
3509 bool MediaTrackGraphImpl::InDriverIteration(const GraphDriver* aDriver) const {
3510 return aDriver->OnThread() ||
3511 (mGraphRunner && mGraphRunner->InDriverIteration(aDriver));
3513 #endif
3515 void MediaTrackGraphImpl::Destroy() {
3516 // First unregister from memory reporting.
3517 UnregisterWeakMemoryReporter(this);
3519 // Clear the self reference which will destroy this instance if all
3520 // associated GraphDrivers are destroyed.
3521 mSelfRef = nullptr;
3524 // Internal method has a Window ID parameter so that TestAudioTrackGraph
3525 // GTests can create a graph without a window.
3526 /* static */
3527 MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstanceIfExists(
3528 uint64_t aWindowID, TrackRate aSampleRate,
3529 AudioDeviceID aPrimaryOutputDeviceID) {
3530 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3531 MOZ_ASSERT(aSampleRate > 0);
3533 GraphHashSet::Ptr p =
3534 Graphs()->lookup({aWindowID, aSampleRate, aPrimaryOutputDeviceID});
3535 return p ? *p : nullptr;
3538 // Public method has an nsPIDOMWindowInner* parameter to ensure that the
3539 // window is a real inner Window, not a WindowProxy.
3540 /* static */
3541 MediaTrackGraph* MediaTrackGraph::GetInstanceIfExists(
3542 nsPIDOMWindowInner* aWindow, TrackRate aSampleRate,
3543 AudioDeviceID aPrimaryOutputDeviceID) {
3544 TrackRate sampleRate =
3545 aSampleRate ? aSampleRate
3546 : CubebUtils::PreferredSampleRate(
3547 aWindow->AsGlobal()->ShouldResistFingerprinting(
3548 RFPTarget::AudioSampleRate));
3549 return MediaTrackGraphImpl::GetInstanceIfExists(
3550 aWindow->WindowID(), sampleRate, aPrimaryOutputDeviceID);
3553 /* static */
3554 MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstance(
3555 GraphDriverType aGraphDriverRequested, uint64_t aWindowID,
3556 TrackRate aSampleRate, AudioDeviceID aPrimaryOutputDeviceID,
3557 nsISerialEventTarget* aMainThread) {
3558 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3559 MOZ_ASSERT(aSampleRate > 0);
3560 MOZ_ASSERT(aGraphDriverRequested != OFFLINE_THREAD_DRIVER,
3561 "Use CreateNonRealtimeInstance() for offline graphs");
3563 GraphHashSet* graphs = Graphs();
3564 GraphHashSet::AddPtr addPtr =
3565 graphs->lookupForAdd({aWindowID, aSampleRate, aPrimaryOutputDeviceID});
3566 if (addPtr) { // graph already exists
3567 return *addPtr;
3570 GraphRunType runType = DIRECT_DRIVER;
3571 if (Preferences::GetBool("media.audiograph.single_thread.enabled", true)) {
3572 runType = SINGLE_THREAD;
3575 // In a real time graph, the number of output channels is determined by
3576 // the underlying number of channel of the default audio output device.
3577 uint32_t channelCount = CubebUtils::MaxNumberOfChannels();
3578 MediaTrackGraphImpl* graph = new MediaTrackGraphImpl(
3579 aWindowID, aSampleRate, aPrimaryOutputDeviceID, aMainThread);
3580 graph->Init(aGraphDriverRequested, runType, channelCount);
3581 MOZ_ALWAYS_TRUE(graphs->add(addPtr, graph));
3583 LOG(LogLevel::Debug, ("Starting up MediaTrackGraph %p for window 0x%" PRIx64,
3584 graph, aWindowID));
3586 return graph;
3589 /* static */
3590 MediaTrackGraph* MediaTrackGraph::GetInstance(
3591 GraphDriverType aGraphDriverRequested, nsPIDOMWindowInner* aWindow,
3592 TrackRate aSampleRate, AudioDeviceID aPrimaryOutputDeviceID) {
3593 TrackRate sampleRate =
3594 aSampleRate ? aSampleRate
3595 : CubebUtils::PreferredSampleRate(
3596 aWindow->AsGlobal()->ShouldResistFingerprinting(
3597 RFPTarget::AudioSampleRate));
3598 return MediaTrackGraphImpl::GetInstance(
3599 aGraphDriverRequested, aWindow->WindowID(), sampleRate,
3600 aPrimaryOutputDeviceID, GetMainThreadSerialEventTarget());
3603 MediaTrackGraph* MediaTrackGraphImpl::CreateNonRealtimeInstance(
3604 TrackRate aSampleRate) {
3605 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3607 nsISerialEventTarget* mainThread = GetMainThreadSerialEventTarget();
3608 // Offline graphs have 0 output channel count: they write the output to a
3609 // buffer, not an audio output track.
3610 MediaTrackGraphImpl* graph = new MediaTrackGraphImpl(
3611 0, aSampleRate, DEFAULT_OUTPUT_DEVICE, mainThread);
3612 graph->Init(OFFLINE_THREAD_DRIVER, DIRECT_DRIVER, 0);
3614 LOG(LogLevel::Debug, ("Starting up Offline MediaTrackGraph %p", graph));
3616 return graph;
3619 MediaTrackGraph* MediaTrackGraph::CreateNonRealtimeInstance(
3620 TrackRate aSampleRate) {
3621 return MediaTrackGraphImpl::CreateNonRealtimeInstance(aSampleRate);
3624 void MediaTrackGraph::ForceShutDown() {
3625 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3627 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
3629 graph->ForceShutDown();
3632 NS_IMPL_ISUPPORTS(MediaTrackGraphImpl, nsIMemoryReporter, nsIObserver,
3633 nsIThreadObserver, nsITimerCallback, nsINamed)
3635 NS_IMETHODIMP
3636 MediaTrackGraphImpl::CollectReports(nsIHandleReportCallback* aHandleReport,
3637 nsISupports* aData, bool aAnonymize) {
3638 MOZ_ASSERT(NS_IsMainThread());
3639 if (mMainThreadTrackCount == 0) {
3640 // No tracks to report.
3641 FinishCollectReports(aHandleReport, aData, nsTArray<AudioNodeSizes>());
3642 return NS_OK;
3645 class Message final : public ControlMessage {
3646 public:
3647 Message(MediaTrackGraphImpl* aGraph, nsIHandleReportCallback* aHandleReport,
3648 nsISupports* aHandlerData)
3649 : ControlMessage(nullptr),
3650 mGraph(aGraph),
3651 mHandleReport(aHandleReport),
3652 mHandlerData(aHandlerData) {}
3653 void Run() override {
3654 TRACE("MTG::CollectSizesForMemoryReport ControlMessage");
3655 mGraph->CollectSizesForMemoryReport(mHandleReport.forget(),
3656 mHandlerData.forget());
3658 void RunDuringShutdown() override {
3659 // Run this message during shutdown too, so that endReports is called.
3660 Run();
3662 MediaTrackGraphImpl* mGraph;
3663 // nsMemoryReporterManager keeps the callback and data alive only if it
3664 // does not time out.
3665 nsCOMPtr<nsIHandleReportCallback> mHandleReport;
3666 nsCOMPtr<nsISupports> mHandlerData;
3669 AppendMessage(MakeUnique<Message>(this, aHandleReport, aData));
3671 return NS_OK;
3674 void MediaTrackGraphImpl::CollectSizesForMemoryReport(
3675 already_AddRefed<nsIHandleReportCallback> aHandleReport,
3676 already_AddRefed<nsISupports> aHandlerData) {
3677 class FinishCollectRunnable final : public Runnable {
3678 public:
3679 explicit FinishCollectRunnable(
3680 already_AddRefed<nsIHandleReportCallback> aHandleReport,
3681 already_AddRefed<nsISupports> aHandlerData)
3682 : mozilla::Runnable("FinishCollectRunnable"),
3683 mHandleReport(aHandleReport),
3684 mHandlerData(aHandlerData) {}
3686 NS_IMETHOD Run() override {
3687 TRACE("MTG::FinishCollectReports ControlMessage");
3688 MediaTrackGraphImpl::FinishCollectReports(mHandleReport, mHandlerData,
3689 std::move(mAudioTrackSizes));
3690 return NS_OK;
3693 nsTArray<AudioNodeSizes> mAudioTrackSizes;
3695 private:
3696 ~FinishCollectRunnable() = default;
3698 // Avoiding nsCOMPtr because NSCAP_ASSERT_NO_QUERY_NEEDED in its
3699 // constructor modifies the ref-count, which cannot be done off main
3700 // thread.
3701 RefPtr<nsIHandleReportCallback> mHandleReport;
3702 RefPtr<nsISupports> mHandlerData;
3705 RefPtr<FinishCollectRunnable> runnable = new FinishCollectRunnable(
3706 std::move(aHandleReport), std::move(aHandlerData));
3708 auto audioTrackSizes = &runnable->mAudioTrackSizes;
3710 for (MediaTrack* t : AllTracks()) {
3711 AudioNodeTrack* track = t->AsAudioNodeTrack();
3712 if (track) {
3713 AudioNodeSizes* usage = audioTrackSizes->AppendElement();
3714 track->SizeOfAudioNodesIncludingThis(MallocSizeOf, *usage);
3718 mMainThread->Dispatch(runnable.forget());
3721 void MediaTrackGraphImpl::FinishCollectReports(
3722 nsIHandleReportCallback* aHandleReport, nsISupports* aData,
3723 const nsTArray<AudioNodeSizes>& aAudioTrackSizes) {
3724 MOZ_ASSERT(NS_IsMainThread());
3726 nsCOMPtr<nsIMemoryReporterManager> manager =
3727 do_GetService("@mozilla.org/memory-reporter-manager;1");
3729 if (!manager) return;
3731 #define REPORT(_path, _amount, _desc) \
3732 aHandleReport->Callback(""_ns, _path, KIND_HEAP, UNITS_BYTES, _amount, \
3733 nsLiteralCString(_desc), aData);
3735 for (size_t i = 0; i < aAudioTrackSizes.Length(); i++) {
3736 const AudioNodeSizes& usage = aAudioTrackSizes[i];
3737 const char* const nodeType =
3738 usage.mNodeType ? usage.mNodeType : "<unknown>";
3740 nsPrintfCString enginePath("explicit/webaudio/audio-node/%s/engine-objects",
3741 nodeType);
3742 REPORT(enginePath, usage.mEngine,
3743 "Memory used by AudioNode engine objects (Web Audio).");
3745 nsPrintfCString trackPath("explicit/webaudio/audio-node/%s/track-objects",
3746 nodeType);
3747 REPORT(trackPath, usage.mTrack,
3748 "Memory used by AudioNode track objects (Web Audio).");
3751 size_t hrtfLoaders = WebCore::HRTFDatabaseLoader::sizeOfLoaders(MallocSizeOf);
3752 if (hrtfLoaders) {
3753 REPORT(nsLiteralCString(
3754 "explicit/webaudio/audio-node/PannerNode/hrtf-databases"),
3755 hrtfLoaders, "Memory used by PannerNode databases (Web Audio).");
3758 #undef REPORT
3760 manager->EndReport();
3763 SourceMediaTrack* MediaTrackGraph::CreateSourceTrack(MediaSegment::Type aType) {
3764 SourceMediaTrack* track = new SourceMediaTrack(aType, GraphRate());
3765 AddTrack(track);
3766 return track;
3769 ProcessedMediaTrack* MediaTrackGraph::CreateForwardedInputTrack(
3770 MediaSegment::Type aType) {
3771 ForwardedInputTrack* track = new ForwardedInputTrack(GraphRate(), aType);
3772 AddTrack(track);
3773 return track;
3776 AudioCaptureTrack* MediaTrackGraph::CreateAudioCaptureTrack() {
3777 AudioCaptureTrack* track = new AudioCaptureTrack(GraphRate());
3778 AddTrack(track);
3779 return track;
3782 CrossGraphTransmitter* MediaTrackGraph::CreateCrossGraphTransmitter(
3783 CrossGraphReceiver* aReceiver) {
3784 CrossGraphTransmitter* track =
3785 new CrossGraphTransmitter(GraphRate(), aReceiver);
3786 AddTrack(track);
3787 return track;
3790 CrossGraphReceiver* MediaTrackGraph::CreateCrossGraphReceiver(
3791 TrackRate aTransmitterRate) {
3792 CrossGraphReceiver* track =
3793 new CrossGraphReceiver(GraphRate(), aTransmitterRate);
3794 AddTrack(track);
3795 return track;
3798 void MediaTrackGraph::AddTrack(MediaTrack* aTrack) {
3799 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
3800 MOZ_ASSERT(NS_IsMainThread());
3801 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
3802 if (graph->mRealtime) {
3803 GraphHashSet::Ptr p = Graphs()->lookup(*graph);
3804 MOZ_DIAGNOSTIC_ASSERT(p, "Graph must not be shutting down");
3806 #endif
3807 if (graph->mMainThreadTrackCount == 0) {
3808 nsCOMPtr<nsIObserverService> observerService =
3809 mozilla::services::GetObserverService();
3810 if (observerService) {
3811 observerService->AddObserver(graph, "document-title-changed", false);
3815 NS_ADDREF(aTrack);
3816 aTrack->SetGraphImpl(graph);
3817 ++graph->mMainThreadTrackCount;
3818 graph->AppendMessage(MakeUnique<CreateMessage>(aTrack));
3821 void MediaTrackGraphImpl::RemoveTrack(MediaTrack* aTrack) {
3822 MOZ_ASSERT(NS_IsMainThread());
3823 MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0);
3825 mAudioOutputParams.RemoveElementsBy(
3826 [&](const TrackKeyDeviceAndVolume& aElement) {
3827 if (aElement.mTrack != aTrack) {
3828 return false;
3830 DecrementOutputDeviceRefCnt(aElement.mDeviceID);
3831 return true;
3834 if (--mMainThreadTrackCount == 0) {
3835 LOG(LogLevel::Info, ("MediaTrackGraph %p, last track %p removed from "
3836 "main thread. Graph will shut down.",
3837 this, aTrack));
3838 if (mRealtime) {
3839 // Find the graph in the hash table and remove it.
3840 GraphHashSet* graphs = Graphs();
3841 GraphHashSet::Ptr p = graphs->lookup(*this);
3842 MOZ_ASSERT(*p == this);
3843 graphs->remove(p);
3845 nsCOMPtr<nsIObserverService> observerService =
3846 mozilla::services::GetObserverService();
3847 if (observerService) {
3848 observerService->RemoveObserver(this, "document-title-changed");
3851 // The graph thread will shut itself down soon, but won't be able to do
3852 // that if JS continues to run.
3853 InterruptJS();
3857 auto MediaTrackGraphImpl::NotifyWhenDeviceStarted(AudioDeviceID aDeviceID)
3858 -> RefPtr<GraphStartedPromise> {
3859 MOZ_ASSERT(NS_IsMainThread());
3861 size_t index = mOutputDeviceRefCnts.IndexOf(aDeviceID);
3862 if (index == decltype(mOutputDeviceRefCnts)::NoIndex) {
3863 return GraphStartedPromise::CreateAndReject(NS_ERROR_INVALID_ARG, __func__);
3866 MozPromiseHolder<GraphStartedPromise> h;
3867 RefPtr<GraphStartedPromise> p = h.Ensure(__func__);
3869 if (CrossGraphReceiver* receiver = mOutputDeviceRefCnts[index].mReceiver) {
3870 receiver->GraphImpl()->NotifyWhenPrimaryDeviceStarted(std::move(h));
3871 return p;
3874 // aSink corresponds to the primary audio output device of this graph.
3875 NotifyWhenPrimaryDeviceStarted(std::move(h));
3876 return p;
3879 void MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted(
3880 MozPromiseHolder<GraphStartedPromise>&& aHolder) {
3881 MOZ_ASSERT(NS_IsMainThread());
3882 if (mOutputDeviceRefCnts[0].mRefCnt == 0) {
3883 // There are no track outputs that require the device, so the creator of
3884 // this promise no longer needs to know when the graph is running. Don't
3885 // keep the graph alive with another message.
3886 aHolder.Reject(NS_ERROR_NOT_AVAILABLE, __func__);
3887 return;
3890 QueueControlOrShutdownMessage(
3891 [self = RefPtr{this}, this,
3892 holder = std::move(aHolder)](IsInShutdown aInShutdown) mutable {
3893 if (aInShutdown == IsInShutdown::Yes) {
3894 holder.Reject(NS_ERROR_ILLEGAL_DURING_SHUTDOWN, __func__);
3895 return;
3898 TRACE("MTG::NotifyWhenPrimaryDeviceStarted ControlMessage");
3899 // This runs on the graph thread, so when this runs, and the current
3900 // driver is an AudioCallbackDriver, we know the audio hardware is
3901 // started. If not, we are going to switch soon, keep reposting this
3902 // ControlMessage.
3903 if (CurrentDriver()->AsAudioCallbackDriver() &&
3904 CurrentDriver()->ThreadRunning() &&
3905 !CurrentDriver()->AsAudioCallbackDriver()->OnFallback()) {
3906 // Avoid Resolve's locking on the graph thread by doing it on main.
3907 Dispatch(NS_NewRunnableFunction(
3908 "MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted::Resolver",
3909 [holder = std::move(holder)]() mutable {
3910 holder.Resolve(true, __func__);
3911 }));
3912 } else {
3913 DispatchToMainThreadStableState(
3914 NewRunnableMethod<
3915 StoreCopyPassByRRef<MozPromiseHolder<GraphStartedPromise>>>(
3916 "MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted", this,
3917 &MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted,
3918 std::move(holder)));
3923 class AudioContextOperationControlMessage : public ControlMessage {
3924 using AudioContextOperationPromise =
3925 MediaTrackGraph::AudioContextOperationPromise;
3927 public:
3928 AudioContextOperationControlMessage(
3929 MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks,
3930 AudioContextOperation aOperation,
3931 MozPromiseHolder<AudioContextOperationPromise>&& aHolder)
3932 : ControlMessage(aDestinationTrack),
3933 mTracks(std::move(aTracks)),
3934 mAudioContextOperation(aOperation),
3935 mHolder(std::move(aHolder)) {}
3936 void Run() override {
3937 TRACE_COMMENT("MTG::ApplyAudioContextOperationImpl ControlMessage",
3938 kAudioContextOptionsStrings[static_cast<uint8_t>(
3939 mAudioContextOperation)]);
3940 mTrack->GraphImpl()->ApplyAudioContextOperationImpl(this);
3942 void RunDuringShutdown() override {
3943 MOZ_ASSERT(mAudioContextOperation == AudioContextOperation::Close,
3944 "We should be reviving the graph?");
3945 mHolder.Reject(false, __func__);
3948 nsTArray<RefPtr<MediaTrack>> mTracks;
3949 AudioContextOperation mAudioContextOperation;
3950 MozPromiseHolder<AudioContextOperationPromise> mHolder;
3953 void MediaTrackGraphImpl::ApplyAudioContextOperationImpl(
3954 AudioContextOperationControlMessage* aMessage) {
3955 MOZ_ASSERT(OnGraphThread());
3956 // Initialize state to zero. This silences a GCC warning about uninitialized
3957 // values, because although the switch below initializes state for all valid
3958 // enum values, the actual value could be any integer that fits in the enum.
3959 AudioContextState state{0};
3960 switch (aMessage->mAudioContextOperation) {
3961 // Suspend and Close operations may be performed immediately because no
3962 // specific kind of GraphDriver is required. CheckDriver() will schedule
3963 // a change to a SystemCallbackDriver if all tracks are suspended.
3964 case AudioContextOperation::Suspend:
3965 state = AudioContextState::Suspended;
3966 break;
3967 case AudioContextOperation::Close:
3968 state = AudioContextState::Closed;
3969 break;
3970 case AudioContextOperation::Resume:
3971 // Resume operations require an AudioCallbackDriver. CheckDriver() will
3972 // schedule an AudioCallbackDriver if necessary and process pending
3973 // operations if and when an AudioCallbackDriver is running.
3974 mPendingResumeOperations.EmplaceBack(aMessage);
3975 return;
3977 // First resolve any pending Resume promises for the same AudioContext so as
3978 // to resolve its associated promises in the same order as they were
3979 // created. These Resume operations are considered complete and immediately
3980 // canceled by the Suspend or Close.
3981 MediaTrack* destinationTrack = aMessage->GetTrack();
3982 bool shrinking = false;
3983 auto moveDest = mPendingResumeOperations.begin();
3984 for (PendingResumeOperation& op : mPendingResumeOperations) {
3985 if (op.DestinationTrack() == destinationTrack) {
3986 op.Apply(this);
3987 shrinking = true;
3988 continue;
3990 if (shrinking) { // Fill-in gaps in the array.
3991 *moveDest = std::move(op);
3993 ++moveDest;
3995 mPendingResumeOperations.TruncateLength(moveDest -
3996 mPendingResumeOperations.begin());
3998 for (MediaTrack* track : aMessage->mTracks) {
3999 track->IncrementSuspendCount();
4001 // Resolve after main thread state is up to date with completed processing.
4002 DispatchToMainThreadStableState(NS_NewRunnableFunction(
4003 "MediaTrackGraphImpl::ApplyAudioContextOperationImpl",
4004 [holder = std::move(aMessage->mHolder), state]() mutable {
4005 holder.Resolve(state, __func__);
4006 }));
4009 MediaTrackGraphImpl::PendingResumeOperation::PendingResumeOperation(
4010 AudioContextOperationControlMessage* aMessage)
4011 : mDestinationTrack(aMessage->GetTrack()),
4012 mTracks(std::move(aMessage->mTracks)),
4013 mHolder(std::move(aMessage->mHolder)) {
4014 MOZ_ASSERT(aMessage->mAudioContextOperation == AudioContextOperation::Resume);
4017 void MediaTrackGraphImpl::PendingResumeOperation::Apply(
4018 MediaTrackGraphImpl* aGraph) {
4019 MOZ_ASSERT(aGraph->OnGraphThread());
4020 for (MediaTrack* track : mTracks) {
4021 track->DecrementSuspendCount();
4023 // The graph is provided through the parameter so that it is available even
4024 // when the track is destroyed.
4025 aGraph->DispatchToMainThreadStableState(NS_NewRunnableFunction(
4026 "PendingResumeOperation::Apply", [holder = std::move(mHolder)]() mutable {
4027 holder.Resolve(AudioContextState::Running, __func__);
4028 }));
4031 void MediaTrackGraphImpl::PendingResumeOperation::Abort() {
4032 // The graph is shutting down before the operation completed.
4033 MOZ_ASSERT(!mDestinationTrack->GraphImpl() ||
4034 mDestinationTrack->GraphImpl()->LifecycleStateRef() ==
4035 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN);
4036 mHolder.Reject(false, __func__);
4039 auto MediaTrackGraph::ApplyAudioContextOperation(
4040 MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks,
4041 AudioContextOperation aOperation) -> RefPtr<AudioContextOperationPromise> {
4042 MozPromiseHolder<AudioContextOperationPromise> holder;
4043 RefPtr<AudioContextOperationPromise> p = holder.Ensure(__func__);
4044 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
4045 graphImpl->AppendMessage(MakeUnique<AudioContextOperationControlMessage>(
4046 aDestinationTrack, std::move(aTracks), aOperation, std::move(holder)));
4047 return p;
4050 uint32_t MediaTrackGraphImpl::PrimaryOutputChannelCount() const {
4051 MOZ_ASSERT(!mOutputDevices[0].mReceiver);
4052 return AudioOutputChannelCount(mOutputDevices[0]);
4055 uint32_t MediaTrackGraphImpl::AudioOutputChannelCount(
4056 const OutputDeviceEntry& aDevice) const {
4057 MOZ_ASSERT(OnGraphThread());
4058 // The audio output channel count for a graph is the maximum of the output
4059 // channel count of all the tracks with outputs to this device, or the max
4060 // audio output channel count the machine can do, whichever is smaller.
4061 uint32_t channelCount = 0;
4062 for (const auto& output : aDevice.mTrackOutputs) {
4063 channelCount = std::max(channelCount, output.mTrack->NumberOfChannels());
4065 channelCount = std::min(channelCount, mMaxOutputChannelCount);
4066 if (channelCount) {
4067 return channelCount;
4068 } else {
4069 // null aDevice.mReceiver indicates the primary graph output device.
4070 if (!aDevice.mReceiver && CurrentDriver()->AsAudioCallbackDriver()) {
4071 return CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
4073 return 2;
4077 double MediaTrackGraph::AudioOutputLatency() {
4078 return static_cast<MediaTrackGraphImpl*>(this)->AudioOutputLatency();
4081 double MediaTrackGraphImpl::AudioOutputLatency() {
4082 MOZ_ASSERT(NS_IsMainThread());
4083 if (mAudioOutputLatency != 0.0) {
4084 return mAudioOutputLatency;
4086 MonitorAutoLock lock(mMonitor);
4087 if (CurrentDriver()->AsAudioCallbackDriver()) {
4088 mAudioOutputLatency = CurrentDriver()
4089 ->AsAudioCallbackDriver()
4090 ->AudioOutputLatency()
4091 .ToSeconds();
4092 } else {
4093 // Failure mode: return 0.0 if running on a normal thread.
4094 mAudioOutputLatency = 0.0;
4097 return mAudioOutputLatency;
4100 bool MediaTrackGraph::OutputForAECMightDrift() {
4101 return static_cast<MediaTrackGraphImpl*>(this)->OutputForAECMightDrift();
4103 bool MediaTrackGraph::IsNonRealtime() const {
4104 return !static_cast<const MediaTrackGraphImpl*>(this)->mRealtime;
4107 void MediaTrackGraph::StartNonRealtimeProcessing(uint32_t aTicksToProcess) {
4108 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
4110 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
4111 NS_ASSERTION(!graph->mRealtime, "non-realtime only");
4113 class Message : public ControlMessage {
4114 public:
4115 explicit Message(MediaTrackGraphImpl* aGraph, uint32_t aTicksToProcess)
4116 : ControlMessage(nullptr),
4117 mGraph(aGraph),
4118 mTicksToProcess(aTicksToProcess) {}
4119 void Run() override {
4120 TRACE("MTG::StartNonRealtimeProcessing ControlMessage");
4121 MOZ_ASSERT(mGraph->mEndTime == 0,
4122 "StartNonRealtimeProcessing should be called only once");
4123 mGraph->mEndTime = mGraph->RoundUpToEndOfAudioBlock(
4124 mGraph->mStateComputedTime + mTicksToProcess);
4126 // The graph owns this message.
4127 MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph;
4128 uint32_t mTicksToProcess;
4131 graph->AppendMessage(MakeUnique<Message>(graph, aTicksToProcess));
4134 void MediaTrackGraphImpl::InterruptJS() {
4135 MonitorAutoLock lock(mMonitor);
4136 mInterruptJSCalled = true;
4137 if (mJSContext) {
4138 JS_RequestInterruptCallback(mJSContext);
4142 static bool InterruptCallback(JSContext* aCx) {
4143 // Interrupt future calls also.
4144 JS_RequestInterruptCallback(aCx);
4145 // Stop execution.
4146 return false;
4149 void MediaTrackGraph::NotifyJSContext(JSContext* aCx) {
4150 MOZ_ASSERT(OnGraphThread());
4151 MOZ_ASSERT(aCx);
4153 auto* impl = static_cast<MediaTrackGraphImpl*>(this);
4154 MonitorAutoLock lock(impl->mMonitor);
4155 if (impl->mJSContext) {
4156 MOZ_ASSERT(impl->mJSContext == aCx);
4157 return;
4159 JS_AddInterruptCallback(aCx, InterruptCallback);
4160 impl->mJSContext = aCx;
4161 if (impl->mInterruptJSCalled) {
4162 JS_RequestInterruptCallback(aCx);
4166 void ProcessedMediaTrack::AddInput(MediaInputPort* aPort) {
4167 MediaTrack* t = aPort->GetSource();
4168 if (!t->IsSuspended()) {
4169 mInputs.AppendElement(aPort);
4170 } else {
4171 mSuspendedInputs.AppendElement(aPort);
4173 GraphImpl()->SetTrackOrderDirty();
4176 void ProcessedMediaTrack::InputSuspended(MediaInputPort* aPort) {
4177 GraphImpl()->AssertOnGraphThreadOrNotRunning();
4178 mInputs.RemoveElement(aPort);
4179 mSuspendedInputs.AppendElement(aPort);
4180 GraphImpl()->SetTrackOrderDirty();
4183 void ProcessedMediaTrack::InputResumed(MediaInputPort* aPort) {
4184 GraphImpl()->AssertOnGraphThreadOrNotRunning();
4185 mSuspendedInputs.RemoveElement(aPort);
4186 mInputs.AppendElement(aPort);
4187 GraphImpl()->SetTrackOrderDirty();
4190 void MediaTrackGraphImpl::SwitchAtNextIteration(GraphDriver* aNextDriver) {
4191 MOZ_ASSERT(OnGraphThread());
4192 LOG(LogLevel::Debug, ("%p: Switching to new driver: %p", this, aNextDriver));
4193 if (GraphDriver* nextDriver = NextDriver()) {
4194 if (nextDriver != CurrentDriver()) {
4195 LOG(LogLevel::Debug,
4196 ("%p: Discarding previous next driver: %p", this, nextDriver));
4199 mNextDriver = aNextDriver;
4202 void MediaTrackGraph::RegisterCaptureTrackForWindow(
4203 uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) {
4204 MOZ_ASSERT(NS_IsMainThread());
4205 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
4206 graphImpl->RegisterCaptureTrackForWindow(aWindowId, aCaptureTrack);
4209 void MediaTrackGraphImpl::RegisterCaptureTrackForWindow(
4210 uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) {
4211 MOZ_ASSERT(NS_IsMainThread());
4212 WindowAndTrack winAndTrack;
4213 winAndTrack.mWindowId = aWindowId;
4214 winAndTrack.mCaptureTrackSink = aCaptureTrack;
4215 mWindowCaptureTracks.AppendElement(winAndTrack);
4218 void MediaTrackGraph::UnregisterCaptureTrackForWindow(uint64_t aWindowId) {
4219 MOZ_ASSERT(NS_IsMainThread());
4220 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
4221 graphImpl->UnregisterCaptureTrackForWindow(aWindowId);
4224 void MediaTrackGraphImpl::UnregisterCaptureTrackForWindow(uint64_t aWindowId) {
4225 MOZ_ASSERT(NS_IsMainThread());
4226 mWindowCaptureTracks.RemoveElementsBy(
4227 [aWindowId](const auto& track) { return track.mWindowId == aWindowId; });
4230 already_AddRefed<MediaInputPort> MediaTrackGraph::ConnectToCaptureTrack(
4231 uint64_t aWindowId, MediaTrack* aMediaTrack) {
4232 return aMediaTrack->GraphImpl()->ConnectToCaptureTrack(aWindowId,
4233 aMediaTrack);
4236 already_AddRefed<MediaInputPort> MediaTrackGraphImpl::ConnectToCaptureTrack(
4237 uint64_t aWindowId, MediaTrack* aMediaTrack) {
4238 MOZ_ASSERT(NS_IsMainThread());
4239 for (uint32_t i = 0; i < mWindowCaptureTracks.Length(); i++) {
4240 if (mWindowCaptureTracks[i].mWindowId == aWindowId) {
4241 ProcessedMediaTrack* sink = mWindowCaptureTracks[i].mCaptureTrackSink;
4242 return sink->AllocateInputPort(aMediaTrack);
4245 return nullptr;
4248 void MediaTrackGraph::DispatchToMainThreadStableState(
4249 already_AddRefed<nsIRunnable> aRunnable) {
4250 AssertOnGraphThreadOrNotRunning();
4251 static_cast<MediaTrackGraphImpl*>(this)
4252 ->mPendingUpdateRunnables.AppendElement(std::move(aRunnable));
4255 Watchable<mozilla::GraphTime>& MediaTrackGraphImpl::CurrentTime() {
4256 MOZ_ASSERT(NS_IsMainThread());
4257 return mMainThreadGraphTime;
4260 GraphTime MediaTrackGraph::ProcessedTime() const {
4261 AssertOnGraphThreadOrNotRunning();
4262 return static_cast<const MediaTrackGraphImpl*>(this)->mProcessedTime;
4265 void* MediaTrackGraph::CurrentDriver() const {
4266 AssertOnGraphThreadOrNotRunning();
4267 return static_cast<const MediaTrackGraphImpl*>(this)->mDriver;
4270 uint32_t MediaTrackGraphImpl::AudioInputChannelCount(
4271 CubebUtils::AudioDeviceID aID) {
4272 MOZ_ASSERT(OnGraphThreadOrNotRunning());
4273 DeviceInputTrack* t =
4274 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
4275 return t ? t->MaxRequestedInputChannels() : 0;
4278 AudioInputType MediaTrackGraphImpl::AudioInputDevicePreference(
4279 CubebUtils::AudioDeviceID aID) {
4280 MOZ_ASSERT(OnGraphThreadOrNotRunning());
4281 DeviceInputTrack* t =
4282 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
4283 return t && t->HasVoiceInput() ? AudioInputType::Voice
4284 : AudioInputType::Unknown;
4287 void MediaTrackGraphImpl::SetNewNativeInput() {
4288 MOZ_ASSERT(NS_IsMainThread());
4289 MOZ_ASSERT(!mDeviceInputTrackManagerMainThread.GetNativeInputTrack());
4291 LOG(LogLevel::Debug, ("%p SetNewNativeInput", this));
4293 NonNativeInputTrack* track =
4294 mDeviceInputTrackManagerMainThread.GetFirstNonNativeInputTrack();
4295 if (!track) {
4296 LOG(LogLevel::Debug, ("%p No other devices opened. Do nothing", this));
4297 return;
4300 const CubebUtils::AudioDeviceID deviceId = track->mDeviceId;
4301 const PrincipalHandle principal = track->mPrincipalHandle;
4303 LOG(LogLevel::Debug,
4304 ("%p Select device %p as the new native input device", this, deviceId));
4306 struct TrackListener {
4307 DeviceInputConsumerTrack* track;
4308 // Keep its reference so it won't be dropped when after
4309 // DisconnectDeviceInput().
4310 RefPtr<AudioDataListener> listener;
4312 nsTArray<TrackListener> pairs;
4314 for (const auto& t : track->GetConsumerTracks()) {
4315 pairs.AppendElement(
4316 TrackListener{t.get(), t->GetAudioDataListener().get()});
4319 for (TrackListener& pair : pairs) {
4320 pair.track->DisconnectDeviceInput();
4323 for (TrackListener& pair : pairs) {
4324 pair.track->ConnectDeviceInput(deviceId, pair.listener.get(), principal);
4325 LOG(LogLevel::Debug,
4326 ("%p: Reinitialize AudioProcessingTrack %p for device %p", this,
4327 pair.track, deviceId));
4330 LOG(LogLevel::Debug,
4331 ("%p Native input device is set to device %p now", this, deviceId));
4333 MOZ_ASSERT(mDeviceInputTrackManagerMainThread.GetNativeInputTrack());
4336 // nsIThreadObserver methods
4338 NS_IMETHODIMP
4339 MediaTrackGraphImpl::OnDispatchedEvent() {
4340 MonitorAutoLock lock(mMonitor);
4341 EnsureNextIteration();
4342 return NS_OK;
4345 NS_IMETHODIMP
4346 MediaTrackGraphImpl::OnProcessNextEvent(nsIThreadInternal*, bool) {
4347 return NS_OK;
4350 NS_IMETHODIMP
4351 MediaTrackGraphImpl::AfterProcessNextEvent(nsIThreadInternal*, bool) {
4352 return NS_OK;
4354 } // namespace mozilla