third_party/WebKit/Source/core/loader/DocumentLoader.cpp

   1 /*
   2  * Copyright (C) 2006, 2007, 2008 Apple Inc. All rights reserved.
   3  * Copyright (C) 2011 Google Inc. All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  *
   9  * 1.  Redistributions of source code must retain the above copyright
  10  *     notice, this list of conditions and the following disclaimer.
  11  * 2.  Redistributions in binary form must reproduce the above copyright
  12  *     notice, this list of conditions and the following disclaimer in the
  13  *     documentation and/or other materials provided with the distribution.
  14  * 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of
  15  *     its contributors may be used to endorse or promote products derived
  16  *     from this software without specific prior written permission.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
  19  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  20  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  21  * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
  22  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  23  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  25  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  27  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  */
  29
  30 #include "config.h"
  31 #include "core/loader/DocumentLoader.h"
  32
  33 #include "core/dom/Document.h"
  34 #include "core/dom/DocumentParser.h"
  35 #include "core/dom/WeakIdentifierMap.h"
  36 #include "core/events/Event.h"
  37 #include "core/fetch/CSSStyleSheetResource.h"
  38 #include "core/fetch/FetchInitiatorTypeNames.h"
  39 #include "core/fetch/FetchRequest.h"
  40 #include "core/fetch/ImageResource.h"
  41 #include "core/fetch/MemoryCache.h"
  42 #include "core/fetch/ResourceFetcher.h"
  43 #include "core/fetch/ResourceLoader.h"
  44 #include "core/fetch/ScriptResource.h"
  45 #include "core/frame/FrameHost.h"
  46 #include "core/frame/LocalDOMWindow.h"
  47 #include "core/frame/LocalFrame.h"
  48 #include "core/frame/Settings.h"
  49 #include "core/frame/csp/ContentSecurityPolicy.h"
  50 #include "core/html/HTMLFrameOwnerElement.h"
  51 #include "core/html/parser/HTMLDocumentParser.h"
  52 #include "core/html/parser/TextResourceDecoder.h"
  53 #include "core/inspector/ConsoleMessage.h"
  54 #include "core/inspector/InspectorInstrumentation.h"
  55 #include "core/loader/FrameFetchContext.h"
  56 #include "core/loader/FrameLoader.h"
  57 #include "core/loader/FrameLoaderClient.h"
  58 #include "core/loader/LinkLoader.h"
  59 #include "core/loader/appcache/ApplicationCacheHost.h"
  60 #include "core/page/FrameTree.h"
  61 #include "core/page/Page.h"
  62 #include "platform/Logging.h"
  63 #include "platform/ThreadedDataReceiver.h"
  64 #include "platform/UserGestureIndicator.h"
  65 #include "platform/mhtml/ArchiveResource.h"
  66 #include "platform/mhtml/ArchiveResourceCollection.h"
  67 #include "platform/mhtml/MHTMLArchive.h"
  68 #include "platform/network/ContentSecurityPolicyResponseHeaders.h"
  69 #include "platform/plugins/PluginData.h"
  70 #include "platform/weborigin/SchemeRegistry.h"
  71 #include "platform/weborigin/SecurityPolicy.h"
  72 #include "public/platform/Platform.h"
  73 #include "public/platform/WebMimeRegistry.h"
  74 #include "wtf/Assertions.h"
  75 #include "wtf/TemporaryChange.h"
  76 #include "wtf/text/WTFString.h"
  77
  78 namespace blink {
  79
  80 static bool isArchiveMIMEType(const String& mimeType)
  81 {
  82     return equalIgnoringCase("multipart/related", mimeType);
  83 }
  84
  85 DocumentLoader::DocumentLoader(LocalFrame* frame, const ResourceRequest& req, const SubstituteData& substituteData)
  86     : m_frame(frame)
  87     , m_fetcher(FrameFetchContext::createContextAndFetcher(this))
  88     , m_originalRequest(req)
  89     , m_substituteData(substituteData)
  90     , m_request(req)
  91     , m_isClientRedirect(false)
  92     , m_replacesCurrentHistoryItem(false)
  93     , m_navigationType(NavigationTypeOther)
  94     , m_documentLoadTiming(*this)
  95     , m_timeOfLastDataReceived(0.0)
  96     , m_applicationCacheHost(ApplicationCacheHost::create(this))
  97     , m_state(NotStarted)
  98     , m_inDataReceived(false)
  99     , m_dataBuffer(SharedBuffer::create())
 100 {
 101 }
 102
 103 FrameLoader* DocumentLoader::frameLoader() const
 104 {
 105     if (!m_frame)
 106         return nullptr;
 107     return &m_frame->loader();
 108 }
 109
 110 ResourceLoader* DocumentLoader::mainResourceLoader() const
 111 {
 112     return m_mainResource ? m_mainResource->loader() : nullptr;
 113 }
 114
 115 DocumentLoader::~DocumentLoader()
 116 {
 117     ASSERT(!m_frame || !isLoading());
 118     ASSERT(!m_mainResource);
 119     ASSERT(!m_applicationCacheHost);
 120 }
 121
 122 DEFINE_TRACE(DocumentLoader)
 123 {
 124     visitor->trace(m_frame);
 125     visitor->trace(m_fetcher);
 126     // TODO(sof): start tracing ResourcePtr<>s (and m_mainResource.)
 127     visitor->trace(m_writer);
 128     visitor->trace(m_archive);
 129     visitor->trace(m_documentLoadTiming);
 130     visitor->trace(m_applicationCacheHost);
 131     visitor->trace(m_contentSecurityPolicy);
 132 }
 133
 134 unsigned long DocumentLoader::mainResourceIdentifier() const
 135 {
 136     return m_mainResource ? m_mainResource->identifier() : 0;
 137 }
 138
 139 Document* DocumentLoader::document() const
 140 {
 141     if (m_frame && m_frame->loader().documentLoader() == this)
 142         return m_frame->document();
 143     return nullptr;
 144 }
 145
 146 const ResourceRequest& DocumentLoader::originalRequest() const
 147 {
 148     return m_originalRequest;
 149 }
 150
 151 const ResourceRequest& DocumentLoader::request() const
 152 {
 153     return m_request;
 154 }
 155
 156 const KURL& DocumentLoader::url() const
 157 {
 158     return m_request.url();
 159 }
 160
 161 void DocumentLoader::startPreload(Resource::Type type, FetchRequest& request)
 162 {
 163     ASSERT(type == Resource::Script || type == Resource::CSSStyleSheet || type == Resource::Image || type == Resource::ImportResource);
 164     ResourcePtr<Resource> resource;
 165     switch (type) {
 166     case Resource::Image:
 167         resource = ImageResource::fetch(request, fetcher());
 168         break;
 169     case Resource::Script:
 170         resource = ScriptResource::fetch(request, fetcher());
 171         break;
 172     case Resource::CSSStyleSheet:
 173         resource = CSSStyleSheetResource::fetch(request, fetcher());
 174         break;
 175     default: // Resource::ImportResource
 176         resource = RawResource::fetchImport(request, fetcher());
 177         break;
 178     }
 179
 180     if (resource)
 181         fetcher()->preloadStarted(resource.get());
 182 }
 183
 184 void DocumentLoader::didChangePerformanceTiming()
 185 {
 186     if (frameLoader())
 187         frameLoader()->client()->didChangePerformanceTiming();
 188 }
 189
 190 void DocumentLoader::updateForSameDocumentNavigation(const KURL& newURL, SameDocumentNavigationSource sameDocumentNavigationSource)
 191 {
 192     KURL oldURL = m_request.url();
 193     m_originalRequest.setURL(newURL);
 194     m_request.setURL(newURL);
 195     if (sameDocumentNavigationSource == SameDocumentNavigationHistoryApi) {
 196         m_request.setHTTPMethod("GET");
 197         m_request.setHTTPBody(nullptr);
 198     }
 199     clearRedirectChain();
 200     if (m_isClientRedirect)
 201         appendRedirect(oldURL);
 202     appendRedirect(newURL);
 203 }
 204
 205 const KURL& DocumentLoader::urlForHistory() const
 206 {
 207     return unreachableURL().isEmpty() ? url() : unreachableURL();
 208 }
 209
 210 void DocumentLoader::mainReceivedError(const ResourceError& error)
 211 {
 212     ASSERT(!error.isNull());
 213     ASSERT(!mainResourceLoader() || !mainResourceLoader()->defersLoading() || InspectorInstrumentation::isDebuggerPaused(m_frame));
 214     if (m_applicationCacheHost)
 215         m_applicationCacheHost->failedLoadingMainResource();
 216     if (!frameLoader())
 217         return;
 218     m_mainDocumentError = error;
 219     m_state = MainResourceDone;
 220     frameLoader()->receivedMainResourceError(this, error);
 221     clearMainResourceHandle();
 222 }
 223
 224 // Cancels the data source's pending loads.  Conceptually, a data source only loads
 225 // one document at a time, but one document may have many related resources.
 226 // stopLoading will stop all loads initiated by the data source,
 227 // but not loads initiated by child frames' data sources -- that's the WebFrame's job.
 228 void DocumentLoader::stopLoading()
 229 {
 230     RefPtrWillBeRawPtr<LocalFrame> protectFrame(m_frame.get());
 231     RefPtrWillBeRawPtr<DocumentLoader> protectLoader(this);
 232
 233     if (isLoading())
 234         cancelMainResourceLoad(ResourceError::cancelledError(m_request.url()));
 235     m_fetcher->stopFetching();
 236 }
 237
 238 void DocumentLoader::commitIfReady()
 239 {
 240     if (m_state < Committed) {
 241         m_state = Committed;
 242         frameLoader()->commitProvisionalLoad();
 243     }
 244 }
 245
 246 bool DocumentLoader::isLoading() const
 247 {
 248     if (document() && document()->hasActiveParser())
 249         return true;
 250
 251     return (m_state > NotStarted && m_state < MainResourceDone) || m_fetcher->isFetching();
 252 }
 253
 254 void DocumentLoader::notifyFinished(Resource* resource)
 255 {
 256     ASSERT_UNUSED(resource, m_mainResource == resource);
 257     ASSERT(m_mainResource);
 258
 259     RefPtrWillBeRawPtr<DocumentLoader> protect(this);
 260
 261     if (!m_mainResource->errorOccurred() && !m_mainResource->wasCanceled()) {
 262         finishedLoading(m_mainResource->loadFinishTime());
 263         return;
 264     }
 265
 266     mainReceivedError(m_mainResource->resourceError());
 267 }
 268
 269 void DocumentLoader::finishedLoading(double finishTime)
 270 {
 271     ASSERT(!mainResourceLoader() || !mainResourceLoader()->defersLoading() || InspectorInstrumentation::isDebuggerPaused(m_frame));
 272
 273     RefPtrWillBeRawPtr<DocumentLoader> protect(this);
 274
 275     double responseEndTime = finishTime;
 276     if (!responseEndTime)
 277         responseEndTime = m_timeOfLastDataReceived;
 278     if (!responseEndTime)
 279         responseEndTime = monotonicallyIncreasingTime();
 280     timing().setResponseEnd(responseEndTime);
 281
 282     commitIfReady();
 283     if (!frameLoader())
 284         return;
 285
 286     if (!maybeCreateArchive()) {
 287         // If this is an empty document, it will not have actually been created yet. Commit dummy data so that
 288         // DocumentWriter::begin() gets called and creates the Document.
 289         if (!m_writer)
 290             commitData(0, 0);
 291     }
 292
 293     endWriting(m_writer.get());
 294
 295     if (!m_mainDocumentError.isNull())
 296         return;
 297     m_state = MainResourceDone;
 298
 299     // If the document specified an application cache manifest, it violates the author's intent if we store it in the memory cache
 300     // and deny the appcache the chance to intercept it in the future, so remove from the memory cache.
 301     if (m_frame) {
 302         if (m_mainResource && m_frame->document()->hasAppCacheManifest())
 303             memoryCache()->remove(m_mainResource.get());
 304     }
 305     m_applicationCacheHost->finishedLoadingMainResource();
 306     clearMainResourceHandle();
 307 }
 308
 309 bool DocumentLoader::isRedirectAfterPost(const ResourceRequest& newRequest, const ResourceResponse& redirectResponse)
 310 {
 311     int status = redirectResponse.httpStatusCode();
 312     if (((status >= 301 && status <= 303) || status == 307)
 313         && m_originalRequest.httpMethod() == "POST")
 314         return true;
 315
 316     return false;
 317 }
 318
 319 bool DocumentLoader::shouldContinueForNavigationPolicy(const ResourceRequest& request, ContentSecurityPolicyDisposition shouldCheckMainWorldContentSecurityPolicy, NavigationPolicy policy)
 320 {
 321     // Don't ask if we are loading an empty URL.
 322     if (request.url().isEmpty() || m_substituteData.isValid())
 323         return true;
 324
 325     // If we're loading content into a subframe, check against the parent's Content Security Policy
 326     // and kill the load if that check fails, unless we should bypass the main world's CSP.
 327     // FIXME: CSP checks are broken for OOPI. For now, this policy always allows frames with a remote parent...
 328     if ((shouldCheckMainWorldContentSecurityPolicy == CheckContentSecurityPolicy) && (m_frame->deprecatedLocalOwner() && !m_frame->deprecatedLocalOwner()->document().contentSecurityPolicy()->allowChildFrameFromSource(request.url(), request.followedRedirect() ? ContentSecurityPolicy::DidRedirect : ContentSecurityPolicy::DidNotRedirect))) {
 329         // Fire a load event, as timing attacks would otherwise reveal that the
 330         // frame was blocked. This way, it looks like every other cross-origin
 331         // page load.
 332         m_frame->document()->enforceSandboxFlags(SandboxOrigin);
 333         m_frame->owner()->dispatchLoad();
 334         return false;
 335     }
 336
 337     policy = frameLoader()->client()->decidePolicyForNavigation(request, this, policy);
 338     if (policy == NavigationPolicyCurrentTab)
 339         return true;
 340     if (policy == NavigationPolicyIgnore)
 341         return false;
 342     if (!LocalDOMWindow::allowPopUp(*m_frame) && !UserGestureIndicator::processingUserGesture())
 343         return false;
 344     frameLoader()->client()->loadURLExternally(request, policy);
 345     return false;
 346 }
 347
 348 void DocumentLoader::redirectReceived(Resource* resource, ResourceRequest& request, const ResourceResponse& redirectResponse)
 349 {
 350     ASSERT_UNUSED(resource, resource == m_mainResource);
 351     willSendRequest(request, redirectResponse);
 352 }
 353
 354 void DocumentLoader::updateRequest(Resource* resource, const ResourceRequest& request)
 355 {
 356     ASSERT_UNUSED(resource, resource == m_mainResource);
 357     m_request = request;
 358 }
 359
 360 static bool isFormSubmission(NavigationType type)
 361 {
 362     return type == NavigationTypeFormSubmitted || type == NavigationTypeFormResubmitted;
 363 }
 364
 365 void DocumentLoader::willSendRequest(ResourceRequest& newRequest, const ResourceResponse& redirectResponse)
 366 {
 367     // Note that there are no asserts here as there are for the other callbacks. This is due to the
 368     // fact that this "callback" is sent when starting every load, and the state of callback
 369     // deferrals plays less of a part in this function in preventing the bad behavior deferring
 370     // callbacks is meant to prevent.
 371     ASSERT(!newRequest.isNull());
 372     if (isFormSubmission(m_navigationType) && !m_frame->document()->contentSecurityPolicy()->allowFormAction(newRequest.url())) {
 373         cancelMainResourceLoad(ResourceError::cancelledError(newRequest.url()));
 374         return;
 375     }
 376
 377     ASSERT(timing().fetchStart());
 378     if (!redirectResponse.isNull()) {
 379         // If the redirecting url is not allowed to display content from the target origin,
 380         // then block the redirect.
 381         RefPtr<SecurityOrigin> redirectingOrigin = SecurityOrigin::create(redirectResponse.url());
 382         if (!redirectingOrigin->canDisplay(newRequest.url())) {
 383             FrameLoader::reportLocalLoadFailed(m_frame, newRequest.url().string());
 384             cancelMainResourceLoad(ResourceError::cancelledError(newRequest.url()));
 385             return;
 386         }
 387         timing().addRedirect(redirectResponse.url(), newRequest.url());
 388     }
 389
 390     // If we're fielding a redirect in response to a POST, force a load from origin, since
 391     // this is a common site technique to return to a page viewing some data that the POST
 392     // just modified.
 393     if (newRequest.cachePolicy() == UseProtocolCachePolicy && isRedirectAfterPost(newRequest, redirectResponse))
 394         newRequest.setCachePolicy(ReloadBypassingCache);
 395
 396     m_request = newRequest;
 397
 398     if (redirectResponse.isNull())
 399         return;
 400
 401     appendRedirect(newRequest.url());
 402     frameLoader()->receivedMainResourceRedirect(m_request.url());
 403     if (!shouldContinueForNavigationPolicy(newRequest, CheckContentSecurityPolicy))
 404         cancelMainResourceLoad(ResourceError::cancelledError(m_request.url()));
 405 }
 406
 407 static bool canShowMIMEType(const String& mimeType, Page* page)
 408 {
 409     if (Platform::current()->mimeRegistry()->supportsMIMEType(mimeType) == WebMimeRegistry::IsSupported)
 410         return true;
 411     PluginData* pluginData = page->pluginData();
 412     return !mimeType.isEmpty() && pluginData && pluginData->supportsMimeType(mimeType);
 413 }
 414
 415 bool DocumentLoader::shouldContinueForResponse() const
 416 {
 417     if (m_substituteData.isValid())
 418         return true;
 419
 420     int statusCode = m_response.httpStatusCode();
 421     if (statusCode == 204 || statusCode == 205) {
 422         // The server does not want us to replace the page contents.
 423         return false;
 424     }
 425
 426     if (contentDispositionType(m_response.httpHeaderField("Content-Disposition")) == ContentDispositionAttachment) {
 427         // The server wants us to download instead of replacing the page contents.
 428         // Downloading is handled by the embedder, but we still get the initial
 429         // response so that we can ignore it and clean up properly.
 430         return false;
 431     }
 432
 433     if (!canShowMIMEType(m_response.mimeType(), m_frame->page()))
 434         return false;
 435
 436     // Prevent remote web archives from loading because they can claim to be from any domain and thus avoid cross-domain security checks.
 437     if (isArchiveMIMEType(m_response.mimeType()) && !SchemeRegistry::shouldTreatURLSchemeAsLocal(m_request.url().protocol()))
 438         return false;
 439
 440     return true;
 441 }
 442
 443 void DocumentLoader::cancelLoadAfterXFrameOptionsOrCSPDenied(const ResourceResponse& response)
 444 {
 445     InspectorInstrumentation::continueAfterXFrameOptionsDenied(m_frame, this, mainResourceIdentifier(), response);
 446
 447     frame()->document()->enforceSandboxFlags(SandboxOrigin);
 448     if (FrameOwner* owner = frame()->owner())
 449         owner->dispatchLoad();
 450
 451     // The load event might have detached this frame. In that case, the load will already have been cancelled during detach.
 452     if (frameLoader())
 453         cancelMainResourceLoad(ResourceError::cancelledError(m_request.url()));
 454     return;
 455 }
 456
 457 void DocumentLoader::responseReceived(Resource* resource, const ResourceResponse& response, PassOwnPtr<WebDataConsumerHandle> handle)
 458 {
 459     ASSERT_UNUSED(resource, m_mainResource == resource);
 460     ASSERT_UNUSED(handle, !handle);
 461     RefPtrWillBeRawPtr<DocumentLoader> protect(this);
 462     ASSERT(frame());
 463
 464     m_applicationCacheHost->didReceiveResponseForMainResource(response);
 465
 466     // The memory cache doesn't understand the application cache or its caching rules. So if a main resource is served
 467     // from the application cache, ensure we don't save the result for future use. All responses loaded
 468     // from appcache will have a non-zero appCacheID().
 469     if (response.appCacheID())
 470         memoryCache()->remove(m_mainResource.get());
 471
 472     m_contentSecurityPolicy = ContentSecurityPolicy::create();
 473     m_contentSecurityPolicy->setOverrideURLForSelf(response.url());
 474     m_contentSecurityPolicy->didReceiveHeaders(ContentSecurityPolicyResponseHeaders(response));
 475     if (!m_contentSecurityPolicy->allowAncestors(m_frame, response.url())) {
 476         cancelLoadAfterXFrameOptionsOrCSPDenied(response);
 477         return;
 478     }
 479
 480     DEFINE_STATIC_LOCAL(AtomicString, xFrameOptionHeader, ("x-frame-options", AtomicString::ConstructFromLiteral));
 481
 482     // 'frame-ancestors' obviates 'x-frame-options': https://w3c.github.io/webappsec/specs/content-security-policy/#frame-ancestors-and-frame-options
 483     if (!m_contentSecurityPolicy->isFrameAncestorsEnforced()) {
 484         HTTPHeaderMap::const_iterator it = response.httpHeaderFields().find(xFrameOptionHeader);
 485         if (it != response.httpHeaderFields().end()) {
 486             String content = it->value;
 487             if (frameLoader()->shouldInterruptLoadForXFrameOptions(content, response.url(), mainResourceIdentifier())) {
 488                 String message = "Refused to display '" + response.url().elidedString() + "' in a frame because it set 'X-Frame-Options' to '" + content + "'.";
 489                 RefPtrWillBeRawPtr<ConsoleMessage> consoleMessage = ConsoleMessage::create(SecurityMessageSource, ErrorMessageLevel, message);
 490                 consoleMessage->setRequestIdentifier(mainResourceIdentifier());
 491                 frame()->document()->addConsoleMessage(consoleMessage.release());
 492
 493                 cancelLoadAfterXFrameOptionsOrCSPDenied(response);
 494                 return;
 495             }
 496         }
 497     }
 498
 499     ASSERT(!mainResourceLoader() || !mainResourceLoader()->defersLoading());
 500
 501     m_response = response;
 502
 503     if (isArchiveMIMEType(m_response.mimeType()) && m_mainResource->dataBufferingPolicy() != BufferData)
 504         m_mainResource->setDataBufferingPolicy(BufferData);
 505
 506     if (!shouldContinueForResponse()) {
 507         InspectorInstrumentation::continueWithPolicyIgnore(m_frame, this, m_mainResource->identifier(), m_response);
 508         cancelMainResourceLoad(ResourceError::cancelledError(m_request.url()));
 509         return;
 510     }
 511
 512     if (m_response.isHTTP()) {
 513         int status = m_response.httpStatusCode();
 514         if ((status < 200 || status >= 300) && m_frame->owner())
 515             m_frame->owner()->renderFallbackContent();
 516     }
 517 }
 518
 519 void DocumentLoader::ensureWriter(const AtomicString& mimeType, const KURL& overridingURL)
 520 {
 521     if (m_writer)
 522         return;
 523
 524     const AtomicString& encoding = m_frame->host()->overrideEncoding().isNull() ? response().textEncodingName() : m_frame->host()->overrideEncoding();
 525
 526     // Prepare a DocumentInit before clearing the frame, because it may need to
 527     // inherit an aliased security context.
 528     DocumentInit init(url(), m_frame);
 529     init.withNewRegistrationContext();
 530     m_frame->loader().clear();
 531     ASSERT(m_frame->page());
 532
 533     ParserSynchronizationPolicy parsingPolicy = AllowAsynchronousParsing;
 534     if ((m_substituteData.isValid() && m_substituteData.forceSynchronousLoad()) || !Document::threadedParsingEnabledForTesting())
 535         parsingPolicy = ForceSynchronousParsing;
 536
 537     m_writer = createWriterFor(0, init, mimeType, encoding, false, parsingPolicy);
 538     m_writer->setDocumentWasLoadedAsPartOfNavigation();
 539
 540     // This should be set before receivedFirstData().
 541     if (!overridingURL.isEmpty())
 542         m_frame->document()->setBaseURLOverride(overridingURL);
 543
 544     // Call receivedFirstData() exactly once per load.
 545     frameLoader()->receivedFirstData();
 546     m_frame->document()->maybeHandleHttpRefresh(m_response.httpHeaderField("Refresh"), Document::HttpRefreshFromHeader);
 547 }
 548
 549 void DocumentLoader::commitData(const char* bytes, size_t length)
 550 {
 551     ASSERT(m_state < MainResourceDone);
 552     ensureWriter(m_response.mimeType());
 553
 554     // This can happen if document.close() is called by an event handler while
 555     // there's still pending incoming data.
 556     if (m_frame && !m_frame->document()->parsing()) {
 557         cancelMainResourceLoad(ResourceError::cancelledError(m_request.url()));
 558         return;
 559     }
 560
 561     if (length)
 562         m_state = DataReceived;
 563
 564     m_writer->addData(bytes, length);
 565 }
 566
 567 void DocumentLoader::dataReceived(Resource* resource, const char* data, unsigned length)
 568 {
 569     ASSERT(data);
 570     ASSERT(length);
 571     ASSERT_UNUSED(resource, resource == m_mainResource);
 572     ASSERT(!m_response.isNull());
 573     ASSERT(!mainResourceLoader() || !mainResourceLoader()->defersLoading());
 574
 575     if (m_inDataReceived) {
 576         // If this function is reentered, defer processing of the additional
 577         // data to the top-level invocation. Reentrant calls can occur because
 578         // of web platform (mis-)features that require running a nested message
 579         // loop:
 580         // - alert(), confirm(), prompt()
 581         // - Detach of plugin elements.
 582         // - Synchronous XMLHTTPRequest
 583         m_dataBuffer->append(data, length);
 584         return;
 585     }
 586
 587     // Both unloading the old page and parsing the new page may execute JavaScript which destroys the datasource
 588     // by starting a new load, so retain temporarily.
 589     RefPtrWillBeRawPtr<LocalFrame> protectFrame(m_frame.get());
 590     RefPtrWillBeRawPtr<DocumentLoader> protectLoader(this);
 591
 592     TemporaryChange<bool> reentrancyProtector(m_inDataReceived, true);
 593     processData(data, length);
 594
 595     // Process data received in reentrant invocations. Note that the
 596     // invocations of processData() may queue more data in reentrant
 597     // invocations, so iterate until it's empty.
 598     const char* segment;
 599     unsigned pos = 0;
 600     while (unsigned length = m_dataBuffer->getSomeData(segment, pos)) {
 601         processData(segment, length);
 602         pos += length;
 603     }
 604     // All data has been consumed, so flush the buffer.
 605     m_dataBuffer->clear();
 606 }
 607
 608 void DocumentLoader::processData(const char* data, unsigned length)
 609 {
 610     m_applicationCacheHost->mainResourceDataReceived(data, length);
 611     m_timeOfLastDataReceived = monotonicallyIncreasingTime();
 612
 613     if (isArchiveMIMEType(response().mimeType()))
 614         return;
 615     commitIfReady();
 616     if (!frameLoader())
 617         return;
 618     commitData(data, length);
 619
 620     // If we are sending data to MediaDocument, we should stop here
 621     // and cancel the request.
 622     if (m_frame && m_frame->document()->isMediaDocument())
 623         cancelMainResourceLoad(ResourceError::cancelledError(m_request.url()));
 624 }
 625
 626 void DocumentLoader::clearRedirectChain()
 627 {
 628     m_redirectChain.clear();
 629 }
 630
 631 void DocumentLoader::appendRedirect(const KURL& url)
 632 {
 633     m_redirectChain.append(url);
 634 }
 635
 636 bool DocumentLoader::loadingMultipartContent() const
 637 {
 638     return mainResourceLoader() ? mainResourceLoader()->loadingMultipartContent() : false;
 639 }
 640
 641 void DocumentLoader::detachFromFrame()
 642 {
 643     ASSERT(m_frame);
 644     RefPtrWillBeRawPtr<LocalFrame> protectFrame(m_frame.get());
 645     RefPtrWillBeRawPtr<DocumentLoader> protectLoader(this);
 646
 647     // It never makes sense to have a document loader that is detached from its
 648     // frame have any loads active, so go ahead and kill all the loads.
 649     stopLoading();
 650
 651     // If that load cancellation triggered another detach, leave.
 652     // (fast/frames/detach-frame-nested-no-crash.html is an example of this.)
 653     if (!m_frame)
 654         return;
 655
 656     m_fetcher->clearContext();
 657
 658     m_applicationCacheHost->detachFromDocumentLoader();
 659     m_applicationCacheHost.clear();
 660     WeakIdentifierMap<DocumentLoader>::notifyObjectDestroyed(this);
 661     clearMainResourceHandle();
 662     m_frame = nullptr;
 663 }
 664
 665 void DocumentLoader::clearMainResourceHandle()
 666 {
 667     if (!m_mainResource)
 668         return;
 669     m_mainResource->removeClient(this);
 670     m_mainResource = nullptr;
 671 }
 672
 673 bool DocumentLoader::maybeCreateArchive()
 674 {
 675     // Only the top-frame can load MHTML.
 676     if (m_frame->tree().parent())
 677         return false;
 678
 679     // Give the archive machinery a crack at this document. If the MIME type is not an archive type, it will return 0.
 680     if (!isArchiveMIMEType(m_response.mimeType()))
 681         return false;
 682
 683     ASSERT(m_mainResource);
 684     m_archive = MHTMLArchive::create(m_response.url(), m_mainResource->resourceBuffer());
 685     // Invalid MHTML.
 686     if (!m_archive || !m_archive->mainResource()) {
 687         m_archive.clear();
 688         return false;
 689     }
 690
 691     m_fetcher->addAllArchiveResources(m_archive.get());
 692     ArchiveResource* mainResource = m_archive->mainResource();
 693
 694     // The origin is the MHTML file, we need to set the base URL to the document encoded in the MHTML so
 695     // relative URLs are resolved properly.
 696     ensureWriter(mainResource->mimeType(), m_archive->mainResource()->url());
 697
 698     // The Document has now been created.
 699     document()->enforceSandboxFlags(SandboxAll);
 700
 701     commitData(mainResource->data()->data(), mainResource->data()->size());
 702     return true;
 703 }
 704
 705 void DocumentLoader::prepareSubframeArchiveLoadIfNeeded()
 706 {
 707     if (!m_frame->tree().parent() || !m_frame->tree().parent()->isLocalFrame())
 708         return;
 709
 710     ArchiveResourceCollection* parentCollection = toLocalFrame(m_frame->tree().parent())->loader().documentLoader()->fetcher()->archiveResourceCollection();
 711     if (!parentCollection)
 712         return;
 713
 714     m_archive = parentCollection->popSubframeArchive(m_frame->tree().uniqueName(), m_request.url());
 715
 716     if (!m_archive)
 717         return;
 718     m_fetcher->addAllArchiveResources(m_archive.get());
 719
 720     ArchiveResource* mainResource = m_archive->mainResource();
 721     m_substituteData = SubstituteData(mainResource->data(), mainResource->mimeType(), mainResource->textEncoding(), KURL());
 722 }
 723
 724 const AtomicString& DocumentLoader::responseMIMEType() const
 725 {
 726     return m_response.mimeType();
 727 }
 728
 729 const KURL& DocumentLoader::unreachableURL() const
 730 {
 731     return m_substituteData.failingURL();
 732 }
 733
 734 void DocumentLoader::setDefersLoading(bool defers)
 735 {
 736     // Multiple frames may be loading the same main resource simultaneously. If deferral state changes,
 737     // each frame's DocumentLoader will try to send a setDefersLoading() to the same underlying ResourceLoader. Ensure only
 738     // the "owning" DocumentLoader does so, as setDefersLoading() is not resilient to setting the same value repeatedly.
 739     if (mainResourceLoader() && mainResourceLoader()->isLoadedBy(m_fetcher.get()))
 740         mainResourceLoader()->setDefersLoading(defers);
 741
 742     m_fetcher->setDefersLoading(defers);
 743 }
 744
 745 bool DocumentLoader::maybeLoadEmpty()
 746 {
 747     bool shouldLoadEmpty = !m_substituteData.isValid() && (m_request.url().isEmpty() || SchemeRegistry::shouldLoadURLSchemeAsEmptyDocument(m_request.url().protocol()));
 748     if (!shouldLoadEmpty)
 749         return false;
 750
 751     if (m_request.url().isEmpty() && !frameLoader()->stateMachine()->creatingInitialEmptyDocument())
 752         m_request.setURL(blankURL());
 753     m_response = ResourceResponse(m_request.url(), "text/html", 0, nullAtom, String());
 754     finishedLoading(monotonicallyIncreasingTime());
 755     return true;
 756 }
 757
 758 void DocumentLoader::startLoadingMainResource()
 759 {
 760     RefPtrWillBeRawPtr<DocumentLoader> protect(this);
 761     m_mainDocumentError = ResourceError();
 762     timing().markNavigationStart();
 763     ASSERT(!m_mainResource);
 764     ASSERT(m_state == NotStarted);
 765     m_state = Provisional;
 766
 767     if (maybeLoadEmpty())
 768         return;
 769
 770     ASSERT(timing().navigationStart());
 771     ASSERT(!timing().fetchStart());
 772     timing().markFetchStart();
 773     willSendRequest(m_request, ResourceResponse());
 774
 775     // willSendRequest() may lead to our LocalFrame being detached or cancelling the load via nulling the ResourceRequest.
 776     if (!m_frame || m_request.isNull())
 777         return;
 778
 779     m_applicationCacheHost->willStartLoadingMainResource(m_request);
 780     prepareSubframeArchiveLoadIfNeeded();
 781
 782     ResourceRequest request(m_request);
 783     DEFINE_STATIC_LOCAL(ResourceLoaderOptions, mainResourceLoadOptions,
 784         (DoNotBufferData, AllowStoredCredentials, ClientRequestedCredentials, CheckContentSecurityPolicy, DocumentContext));
 785     FetchRequest cachedResourceRequest(request, FetchInitiatorTypeNames::document, mainResourceLoadOptions);
 786     m_mainResource = RawResource::fetchMainResource(cachedResourceRequest, fetcher(), m_substituteData);
 787     if (!m_mainResource) {
 788         m_request = ResourceRequest();
 789         // If the load was aborted by clearing m_request, it's possible the ApplicationCacheHost
 790         // is now in a state where starting an empty load will be inconsistent. Replace it with
 791         // a new ApplicationCacheHost.
 792         if (m_applicationCacheHost)
 793             m_applicationCacheHost->detachFromDocumentLoader();
 794         m_applicationCacheHost = ApplicationCacheHost::create(this);
 795         maybeLoadEmpty();
 796         return;
 797     }
 798     m_mainResource->addClient(this);
 799
 800     // A bunch of headers are set when the underlying ResourceLoader is created, and m_request needs to include those.
 801     if (mainResourceLoader())
 802         request = mainResourceLoader()->originalRequest();
 803     // If there was a fragment identifier on m_request, the cache will have stripped it. m_request should include
 804     // the fragment identifier, so add that back in.
 805     if (equalIgnoringFragmentIdentifier(m_request.url(), request.url()))
 806         request.setURL(m_request.url());
 807     m_request = request;
 808 }
 809
 810 void DocumentLoader::cancelMainResourceLoad(const ResourceError& resourceError)
 811 {
 812     RefPtrWillBeRawPtr<DocumentLoader> protect(this);
 813     ResourceError error = resourceError.isNull() ? ResourceError::cancelledError(m_request.url()) : resourceError;
 814
 815     if (mainResourceLoader())
 816         mainResourceLoader()->cancel(error);
 817
 818     mainReceivedError(error);
 819 }
 820
 821 void DocumentLoader::attachThreadedDataReceiver(PassRefPtrWillBeRawPtr<ThreadedDataReceiver> threadedDataReceiver)
 822 {
 823     if (mainResourceLoader())
 824         mainResourceLoader()->attachThreadedDataReceiver(threadedDataReceiver);
 825 }
 826
 827 void DocumentLoader::acceptDataFromThreadedReceiver(const char* data, int dataLength, int encodedDataLength)
 828 {
 829     m_fetcher->acceptDataFromThreadedReceiver(mainResourceIdentifier(), data, dataLength, encodedDataLength);
 830 }
 831
 832 void DocumentLoader::endWriting(DocumentWriter* writer)
 833 {
 834     ASSERT_UNUSED(writer, m_writer == writer);
 835     m_writer->end();
 836     m_writer.clear();
 837 }
 838
 839 PassRefPtrWillBeRawPtr<DocumentWriter> DocumentLoader::createWriterFor(const Document* ownerDocument, const DocumentInit& init, const AtomicString& mimeType, const AtomicString& encoding, bool dispatch, ParserSynchronizationPolicy parsingPolicy)
 840 {
 841     LocalFrame* frame = init.frame();
 842
 843     ASSERT(!frame->document() || !frame->document()->isActive());
 844     ASSERT(frame->tree().childCount() == 0);
 845
 846     if (!init.shouldReuseDefaultView())
 847         frame->setDOMWindow(LocalDOMWindow::create(*frame));
 848
 849     RefPtrWillBeRawPtr<Document> document = frame->localDOMWindow()->installNewDocument(mimeType, init);
 850     if (ownerDocument) {
 851         document->setCookieURL(ownerDocument->cookieURL());
 852         document->setSecurityOrigin(ownerDocument->securityOrigin());
 853     }
 854
 855     frame->loader().didBeginDocument(dispatch);
 856
 857     return DocumentWriter::create(document.get(), parsingPolicy, mimeType, encoding);
 858 }
 859
 860 const AtomicString& DocumentLoader::mimeType() const
 861 {
 862     if (m_writer)
 863         return m_writer->mimeType();
 864     return m_response.mimeType();
 865 }
 866
 867 // This is only called by FrameLoader::replaceDocumentWhileExecutingJavaScriptURL()
 868 void DocumentLoader::replaceDocumentWhileExecutingJavaScriptURL(const DocumentInit& init, const String& source, Document* ownerDocument)
 869 {
 870     m_writer = createWriterFor(ownerDocument, init, mimeType(), m_writer ? m_writer->encoding() : emptyAtom, true, ForceSynchronousParsing);
 871     if (!source.isNull())
 872         m_writer->appendReplacingData(source);
 873     endWriting(m_writer.get());
 874 }
 875
 876 DEFINE_WEAK_IDENTIFIER_MAP(DocumentLoader);
 877
 878 } // namespace blink