1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "ScriptPreloader-inl.h"
8 #include "mozilla/URLPreloader.h"
9 #include "mozilla/loader/AutoMemMap.h"
11 #include "mozilla/ArrayUtils.h"
12 #include "mozilla/ClearOnShutdown.h"
13 #include "mozilla/FileUtils.h"
14 #include "mozilla/IOBuffers.h"
15 #include "mozilla/Logging.h"
16 #include "mozilla/ScopeExit.h"
17 #include "mozilla/Services.h"
18 #include "mozilla/Try.h"
19 #include "mozilla/Unused.h"
20 #include "mozilla/Vector.h"
21 #include "mozilla/scache/StartupCache.h"
24 #include "MainThreadUtils.h"
25 #include "nsPrintfCString.h"
28 #include "nsIFileURL.h"
29 #include "nsNetUtil.h"
30 #include "nsPromiseFlatString.h"
31 #include "nsProxyRelease.h"
32 #include "nsThreadUtils.h"
33 #include "nsXULAppAPI.h"
34 #include "nsZipArchive.h"
35 #include "xpcpublic.h"
39 static LazyLogModule
gURLLog("URLPreloader");
41 #define LOG(level, ...) MOZ_LOG(gURLLog, LogLevel::level, (__VA_ARGS__))
44 bool StartsWith(const T
& haystack
, const T
& needle
) {
45 return StringHead(haystack
, needle
.Length()) == needle
;
47 } // anonymous namespace
49 using namespace mozilla::loader
;
50 using mozilla::scache::StartupCache
;
52 nsresult
URLPreloader::CollectReports(nsIHandleReportCallback
* aHandleReport
,
53 nsISupports
* aData
, bool aAnonymize
) {
54 MOZ_COLLECT_REPORT("explicit/url-preloader/other", KIND_HEAP
, UNITS_BYTES
,
55 ShallowSizeOfIncludingThis(MallocSizeOf
),
56 "Memory used by the URL preloader service itself.");
58 for (const auto& elem
: mCachedURLs
.Values()) {
59 nsAutoCString pathName
;
60 pathName
.Append(elem
->mPath
);
61 // The backslashes will automatically be replaced with slashes in
62 // about:memory, without splitting each path component into a separate
63 // branch in the memory report tree.
64 pathName
.ReplaceChar('/', '\\');
66 nsPrintfCString
path("explicit/url-preloader/cached-urls/%s/[%s]",
67 elem
->TypeString(), pathName
.get());
69 aHandleReport
->Callback(
70 ""_ns
, path
, KIND_HEAP
, UNITS_BYTES
,
71 elem
->SizeOfIncludingThis(MallocSizeOf
),
72 nsLiteralCString("Memory used to hold cache data for files which "
73 "have been read or pre-loaded during this session."),
81 already_AddRefed
<URLPreloader
> URLPreloader::Create(bool* aInitialized
) {
82 // The static APIs like URLPreloader::Read work in the child process because
83 // they fall back to a synchronous read. The actual preloader must be
84 // explicitly initialized, and this should only be done in the parent.
85 MOZ_RELEASE_ASSERT(XRE_IsParentProcess());
87 RefPtr
<URLPreloader
> preloader
= new URLPreloader();
88 if (preloader
->InitInternal().isOk()) {
90 RegisterWeakMemoryReporter(preloader
);
92 *aInitialized
= false;
95 return preloader
.forget();
98 URLPreloader
& URLPreloader::GetSingleton() {
100 sSingleton
= Create(&sInitialized
);
101 ClearOnShutdown(&sSingleton
);
107 bool URLPreloader::sInitialized
= false;
109 StaticRefPtr
<URLPreloader
> URLPreloader::sSingleton
;
111 URLPreloader::~URLPreloader() {
113 UnregisterWeakMemoryReporter(this);
114 sInitialized
= false;
118 Result
<Ok
, nsresult
> URLPreloader::InitInternal() {
119 MOZ_RELEASE_ASSERT(NS_IsMainThread());
121 if (Omnijar::HasOmnijar(Omnijar::GRE
)) {
122 MOZ_TRY(Omnijar::GetURIString(Omnijar::GRE
, mGREPrefix
));
124 if (Omnijar::HasOmnijar(Omnijar::APP
)) {
125 MOZ_TRY(Omnijar::GetURIString(Omnijar::APP
, mAppPrefix
));
129 nsCOMPtr
<nsIIOService
> ios
= do_GetIOService(&rv
);
132 nsCOMPtr
<nsIProtocolHandler
> ph
;
133 MOZ_TRY(ios
->GetProtocolHandler("resource", getter_AddRefs(ph
)));
135 mResProto
= do_QueryInterface(ph
, &rv
);
138 mChromeReg
= services::GetChromeRegistry();
140 return Err(NS_ERROR_UNEXPECTED
);
143 MOZ_TRY(NS_GetSpecialDirectory("ProfLDS", getter_AddRefs(mProfD
)));
148 URLPreloader
& URLPreloader::ReInitialize() {
149 MOZ_ASSERT(sSingleton
);
150 sSingleton
= nullptr;
151 sSingleton
= Create(&sInitialized
);
155 Result
<nsCOMPtr
<nsIFile
>, nsresult
> URLPreloader::GetCacheFile(
156 const nsAString
& suffix
) {
158 return Err(NS_ERROR_NOT_INITIALIZED
);
161 nsCOMPtr
<nsIFile
> cacheFile
;
162 MOZ_TRY(mProfD
->Clone(getter_AddRefs(cacheFile
)));
164 MOZ_TRY(cacheFile
->AppendNative("startupCache"_ns
));
165 Unused
<< cacheFile
->Create(nsIFile::DIRECTORY_TYPE
, 0777);
167 MOZ_TRY(cacheFile
->Append(u
"urlCache"_ns
+ suffix
));
169 return std::move(cacheFile
);
172 static const uint8_t URL_MAGIC
[] = "mozURLcachev003";
174 Result
<nsCOMPtr
<nsIFile
>, nsresult
> URLPreloader::FindCacheFile() {
175 if (StartupCache::GetIgnoreDiskCache()) {
176 return Err(NS_ERROR_ABORT
);
179 nsCOMPtr
<nsIFile
> cacheFile
;
180 MOZ_TRY_VAR(cacheFile
, GetCacheFile(u
".bin"_ns
));
183 MOZ_TRY(cacheFile
->Exists(&exists
));
185 MOZ_TRY(cacheFile
->MoveTo(nullptr, u
"urlCache-current.bin"_ns
));
187 MOZ_TRY(cacheFile
->SetLeafName(u
"urlCache-current.bin"_ns
));
188 MOZ_TRY(cacheFile
->Exists(&exists
));
190 return Err(NS_ERROR_FILE_NOT_FOUND
);
194 return std::move(cacheFile
);
197 Result
<Ok
, nsresult
> URLPreloader::WriteCache() {
198 MOZ_ASSERT(!NS_IsMainThread());
199 MOZ_DIAGNOSTIC_ASSERT(mStartupFinished
);
201 // The script preloader might call us a second time, if it has to re-write
202 // its cache after a cache flush. We don't care about cache flushes, since
203 // our cache doesn't store any file data, only paths. And we currently clear
204 // our cached file list after the first write, which means that a second
205 // write would (aside from breaking the invariant that we never touch
206 // mCachedURLs off-main-thread after the first write, and trigger a data
207 // race) mean we get no pre-loading on the next startup.
211 mCacheWritten
= true;
213 LOG(Debug
, "Writing cache...");
215 nsCOMPtr
<nsIFile
> cacheFile
;
216 MOZ_TRY_VAR(cacheFile
, GetCacheFile(u
"-new.bin"_ns
));
219 MOZ_TRY(cacheFile
->Exists(&exists
));
221 MOZ_TRY(cacheFile
->Remove(false));
226 MOZ_TRY(cacheFile
->OpenNSPRFileDesc(PR_WRONLY
| PR_CREATE_FILE
, 0644,
227 getter_Transfers(raiiFd
)));
228 const auto fd
= raiiFd
.get();
230 nsTArray
<URLEntry
*> entries
;
231 for (const auto& entry
: mCachedURLs
.Values()) {
232 if (entry
->mReadTime
) {
233 entries
.AppendElement(entry
.get());
237 entries
.Sort(URLEntry::Comparator());
240 for (auto entry
: entries
) {
244 uint8_t headerSize
[4];
245 LittleEndian::writeUint32(headerSize
, buf
.cursor());
248 LittleEndian::writeUint32(crc
, ComputeCrc32c(~0, buf
.Get(), buf
.cursor()));
250 MOZ_TRY(Write(fd
, URL_MAGIC
, sizeof(URL_MAGIC
)));
251 MOZ_TRY(Write(fd
, headerSize
, sizeof(headerSize
)));
252 MOZ_TRY(Write(fd
, crc
, sizeof(crc
)));
253 MOZ_TRY(Write(fd
, buf
.Get(), buf
.cursor()));
256 MOZ_TRY(cacheFile
->MoveTo(nullptr, u
"urlCache.bin"_ns
));
258 NS_DispatchToMainThread(
259 NewRunnableMethod("URLPreloader::Cleanup", this, &URLPreloader::Cleanup
));
264 void URLPreloader::Cleanup() { mCachedURLs
.Clear(); }
266 Result
<Ok
, nsresult
> URLPreloader::ReadCache(
267 LinkedList
<URLEntry
>& pendingURLs
) {
268 LOG(Debug
, "Reading cache...");
270 nsCOMPtr
<nsIFile
> cacheFile
;
271 MOZ_TRY_VAR(cacheFile
, FindCacheFile());
274 MOZ_TRY(cache
.init(cacheFile
));
276 auto size
= cache
.size();
280 if (size
< sizeof(URL_MAGIC
) + sizeof(headerSize
) + sizeof(crc
)) {
281 return Err(NS_ERROR_UNEXPECTED
);
284 auto data
= cache
.get
<uint8_t>();
285 auto end
= data
+ size
;
287 if (memcmp(URL_MAGIC
, data
.get(), sizeof(URL_MAGIC
))) {
288 return Err(NS_ERROR_UNEXPECTED
);
290 data
+= sizeof(URL_MAGIC
);
292 headerSize
= LittleEndian::readUint32(data
.get());
293 data
+= sizeof(headerSize
);
295 crc
= LittleEndian::readUint32(data
.get());
298 if (data
+ headerSize
> end
) {
299 return Err(NS_ERROR_UNEXPECTED
);
302 if (crc
!= ComputeCrc32c(~0, data
.get(), headerSize
)) {
303 return Err(NS_ERROR_UNEXPECTED
);
307 mMonitor
.AssertCurrentThreadOwns();
309 auto cleanup
= MakeScopeExit([&]() {
310 while (auto* elem
= pendingURLs
.getFirst()) {
316 Range
<uint8_t> header(data
, data
+ headerSize
);
319 InputBuffer
buf(header
);
320 while (!buf
.finished()) {
323 LOG(Debug
, "Cached file: %s %s", key
.TypeString(), key
.mPath
.get());
325 // Don't bother doing anything else if the key didn't load correctly.
326 // We're going to throw it out right away, and it is possible that this
327 // leads to pendingURLs getting into a weird state.
329 return Err(NS_ERROR_UNEXPECTED
);
332 auto entry
= mCachedURLs
.GetOrInsertNew(key
, key
);
333 entry
->mResultCode
= NS_ERROR_NOT_INITIALIZED
;
335 if (entry
->isInList()) {
337 MOZ_DIAGNOSTIC_ASSERT(pendingURLs
.contains(entry
),
338 "Entry should be in pendingURLs");
339 MOZ_DIAGNOSTIC_ASSERT(key
.mPath
.Length() > 0,
340 "Path should be non-empty");
341 MOZ_DIAGNOSTIC_CRASH("Entry should be new and not in any list");
343 return Err(NS_ERROR_UNEXPECTED
);
346 pendingURLs
.insertBack(entry
);
349 MOZ_RELEASE_ASSERT(!buf
.error(),
350 "We should have already bailed on an error");
358 void URLPreloader::BackgroundReadFiles() {
359 auto cleanup
= MakeScopeExit([&]() {
360 auto lock
= mReaderThread
.Lock();
361 auto& readerThread
= lock
.ref();
362 NS_DispatchToMainThread(NewRunnableMethod(
363 "nsIThread::AsyncShutdown", readerThread
, &nsIThread::AsyncShutdown
));
365 readerThread
= nullptr;
368 Vector
<nsZipCursor
> cursors
;
369 LinkedList
<URLEntry
> pendingURLs
;
371 MonitorAutoLock
mal(mMonitor
);
373 if (ReadCache(pendingURLs
).isErr()) {
374 mReaderInitialized
= true;
379 int numZipEntries
= 0;
380 for (auto entry
: pendingURLs
) {
381 if (entry
->mType
!= entry
->TypeFile
) {
385 MOZ_RELEASE_ASSERT(cursors
.reserve(numZipEntries
));
387 // Initialize the zip cursors for all files in Omnijar while the monitor
388 // is locked. Omnijar is not threadsafe, so the caller of
389 // AutoBeginReading guard must ensure that no code accesses Omnijar
390 // until this segment is done. Once the cursors have been initialized,
391 // the actual reading and decompression can safely be done off-thread,
392 // as is the case for thread-retargeted jar: channels.
393 for (auto entry
: pendingURLs
) {
394 if (entry
->mType
== entry
->TypeFile
) {
398 RefPtr
<nsZipArchive
> zip
= entry
->Archive();
400 MOZ_CRASH_UNSAFE_PRINTF(
401 "Failed to get Omnijar %s archive for entry (path: \"%s\")",
402 entry
->TypeString(), entry
->mPath
.get());
405 auto item
= zip
->GetItem(entry
->mPath
.get());
407 entry
->mResultCode
= NS_ERROR_FILE_NOT_FOUND
;
411 size_t size
= item
->RealSize();
413 entry
->mData
.SetLength(size
);
414 auto data
= entry
->mData
.BeginWriting();
416 cursors
.infallibleEmplaceBack(item
, zip
, reinterpret_cast<uint8_t*>(data
),
420 mReaderInitialized
= true;
424 // Loop over the entries, read the file's contents, store them in the
425 // entry's mData pointer, and notify any waiting threads to check for
428 for (auto entry
: pendingURLs
) {
429 // If there is any other error code, the entry has already failed at
430 // this point, so don't bother trying to read it again.
431 if (entry
->mResultCode
!= NS_ERROR_NOT_INITIALIZED
) {
437 LOG(Debug
, "Background reading %s file %s", entry
->TypeString(),
440 if (entry
->mType
== entry
->TypeFile
) {
441 auto result
= entry
->Read();
442 if (result
.isErr()) {
443 rv
= result
.unwrapErr();
446 auto& cursor
= cursors
[i
++];
450 if (len
!= entry
->mData
.Length()) {
451 entry
->mData
.Truncate();
452 rv
= NS_ERROR_FAILURE
;
456 entry
->mResultCode
= rv
;
457 mMonitor
.NotifyAll();
460 // We're done reading pending entries, so clear the list.
464 void URLPreloader::BeginBackgroundRead() {
465 auto lock
= mReaderThread
.Lock();
466 auto& readerThread
= lock
.ref();
467 if (!readerThread
&& !mReaderInitialized
&& sInitialized
) {
469 rv
= NS_NewNamedThread("BGReadURLs", getter_AddRefs(readerThread
));
470 if (NS_WARN_IF(NS_FAILED(rv
))) {
474 nsCOMPtr
<nsIRunnable
> runnable
=
475 NewRunnableMethod("URLPreloader::BackgroundReadFiles", this,
476 &URLPreloader::BackgroundReadFiles
);
477 rv
= readerThread
->Dispatch(runnable
.forget(), NS_DISPATCH_NORMAL
);
478 if (NS_WARN_IF(NS_FAILED(rv
))) {
479 // If we can't launch the task, just destroy the thread
480 readerThread
= nullptr;
486 Result
<nsCString
, nsresult
> URLPreloader::ReadInternal(const CacheKey
& key
,
488 if (mStartupFinished
|| !mReaderInitialized
) {
494 auto entry
= mCachedURLs
.GetOrInsertNew(key
, key
);
496 entry
->UpdateUsedTime();
498 return entry
->ReadOrWait(readType
);
501 Result
<nsCString
, nsresult
> URLPreloader::ReadURIInternal(nsIURI
* uri
,
504 MOZ_TRY_VAR(key
, ResolveURI(uri
));
506 return ReadInternal(key
, readType
);
509 /* static */ Result
<nsCString
, nsresult
> URLPreloader::Read(const CacheKey
& key
,
511 // If we're being called before the preloader has been initialized (i.e.,
512 // before the profile has been initialized), just fall back to a synchronous
513 // read. This happens when we're reading .ini and preference files that are
514 // needed to locate and initialize the profile.
516 return URLEntry(key
).Read();
519 return GetSingleton().ReadInternal(key
, readType
);
522 /* static */ Result
<nsCString
, nsresult
> URLPreloader::ReadURI(
523 nsIURI
* uri
, ReadType readType
) {
525 return Err(NS_ERROR_NOT_INITIALIZED
);
528 return GetSingleton().ReadURIInternal(uri
, readType
);
531 /* static */ Result
<nsCString
, nsresult
> URLPreloader::ReadFile(
532 nsIFile
* file
, ReadType readType
) {
533 return Read(CacheKey(file
), readType
);
536 /* static */ Result
<nsCString
, nsresult
> URLPreloader::Read(
537 FileLocation
& location
, ReadType readType
) {
538 if (location
.IsZip()) {
539 if (location
.GetBaseZip()) {
541 location
.GetPath(path
);
542 return ReadZip(location
.GetBaseZip(), path
);
544 return URLEntry::ReadLocation(location
);
547 nsCOMPtr
<nsIFile
> file
= location
.GetBaseFile();
548 return ReadFile(file
, readType
);
551 /* static */ Result
<nsCString
, nsresult
> URLPreloader::ReadZip(
552 nsZipArchive
* zip
, const nsACString
& path
, ReadType readType
) {
553 // If the zip archive belongs to an Omnijar location, map it to a cache
554 // entry, and cache it as normal. Otherwise, simply read the entry
555 // synchronously, since other JAR archives are currently unsupported by the
557 RefPtr
<nsZipArchive
> reader
= Omnijar::GetReader(Omnijar::GRE
);
559 CacheKey
key(CacheKey::TypeGREJar
, path
);
560 return Read(key
, readType
);
563 reader
= Omnijar::GetReader(Omnijar::APP
);
565 CacheKey
key(CacheKey::TypeAppJar
, path
);
566 return Read(key
, readType
);
569 // Not an Omnijar archive, so just read it directly.
570 FileLocation
location(zip
, PromiseFlatCString(path
).BeginReading());
571 return URLEntry::ReadLocation(location
);
574 Result
<URLPreloader::CacheKey
, nsresult
> URLPreloader::ResolveURI(nsIURI
* uri
) {
577 MOZ_TRY(uri
->GetSpec(spec
));
578 MOZ_TRY(uri
->GetScheme(scheme
));
580 nsCOMPtr
<nsIURI
> resolved
;
582 // If the URI is a resource: or chrome: URI, first resolve it to the
583 // underlying URI that it wraps.
584 if (scheme
.EqualsLiteral("resource")) {
585 MOZ_TRY(mResProto
->ResolveURI(uri
, spec
));
586 MOZ_TRY(NS_NewURI(getter_AddRefs(resolved
), spec
));
587 } else if (scheme
.EqualsLiteral("chrome")) {
588 MOZ_TRY(mChromeReg
->ConvertChromeURL(uri
, getter_AddRefs(resolved
)));
589 MOZ_TRY(resolved
->GetSpec(spec
));
593 MOZ_TRY(resolved
->GetScheme(scheme
));
595 // Try the GRE and App Omnijar prefixes.
596 if (mGREPrefix
.Length() && StartsWith(spec
, mGREPrefix
)) {
597 return CacheKey(CacheKey::TypeGREJar
, Substring(spec
, mGREPrefix
.Length()));
600 if (mAppPrefix
.Length() && StartsWith(spec
, mAppPrefix
)) {
601 return CacheKey(CacheKey::TypeAppJar
, Substring(spec
, mAppPrefix
.Length()));
604 // Try for a file URI.
605 if (scheme
.EqualsLiteral("file")) {
606 nsCOMPtr
<nsIFileURL
> fileURL
= do_QueryInterface(resolved
);
609 nsCOMPtr
<nsIFile
> file
;
610 MOZ_TRY(fileURL
->GetFile(getter_AddRefs(file
)));
613 MOZ_TRY(file
->GetPath(path
));
615 return CacheKey(CacheKey::TypeFile
, NS_ConvertUTF16toUTF8(path
));
618 // Not a file or Omnijar URI, so currently unsupported.
619 return Err(NS_ERROR_INVALID_ARG
);
622 size_t URLPreloader::ShallowSizeOfIncludingThis(
623 mozilla::MallocSizeOf mallocSizeOf
) {
624 return (mallocSizeOf(this) +
625 mAppPrefix
.SizeOfExcludingThisEvenIfShared(mallocSizeOf
) +
626 mGREPrefix
.SizeOfExcludingThisEvenIfShared(mallocSizeOf
) +
627 mCachedURLs
.ShallowSizeOfExcludingThis(mallocSizeOf
));
630 Result
<FileLocation
, nsresult
> URLPreloader::CacheKey::ToFileLocation() {
631 if (mType
== TypeFile
) {
632 nsCOMPtr
<nsIFile
> file
;
634 NS_NewLocalFile(NS_ConvertUTF8toUTF16(mPath
), getter_AddRefs(file
)));
635 return FileLocation(file
);
638 RefPtr
<nsZipArchive
> zip
= Archive();
639 return FileLocation(zip
, mPath
.get());
642 Result
<nsCString
, nsresult
> URLPreloader::URLEntry::Read() {
643 FileLocation location
;
644 MOZ_TRY_VAR(location
, ToFileLocation());
646 MOZ_TRY_VAR(mData
, ReadLocation(location
));
650 /* static */ Result
<nsCString
, nsresult
> URLPreloader::URLEntry::ReadLocation(
651 FileLocation
& location
) {
652 FileLocation::Data data
;
653 MOZ_TRY(location
.GetData(data
));
656 MOZ_TRY(data
.GetSize(&size
));
659 result
.SetLength(size
);
660 MOZ_TRY(data
.Copy(result
.BeginWriting(), size
));
662 return std::move(result
);
665 Result
<nsCString
, nsresult
> URLPreloader::URLEntry::ReadOrWait(
667 auto now
= TimeStamp::Now();
668 LOG(Info
, "Reading %s\n", mPath
.get());
669 auto cleanup
= MakeScopeExit([&]() {
670 LOG(Info
, "Read in %fms\n", (TimeStamp::Now() - now
).ToMilliseconds());
673 if (mResultCode
== NS_ERROR_NOT_INITIALIZED
) {
674 MonitorAutoLock
mal(GetSingleton().mMonitor
);
676 while (mResultCode
== NS_ERROR_NOT_INITIALIZED
) {
681 if (mResultCode
== NS_OK
&& mData
.IsVoid()) {
682 LOG(Info
, "Reading synchronously...\n");
686 if (NS_FAILED(mResultCode
)) {
687 return Err(mResultCode
);
690 nsCString res
= mData
;
692 if (readType
== Forget
) {
693 mData
.SetIsVoid(true);
698 inline URLPreloader::CacheKey::CacheKey(InputBuffer
& buffer
) {
700 MOZ_DIAGNOSTIC_ASSERT(
701 mType
== TypeAppJar
|| mType
== TypeGREJar
|| mType
== TypeFile
,
702 "mType should be valid");
705 NS_IMPL_ISUPPORTS(URLPreloader
, nsIMemoryReporter
)
709 } // namespace mozilla