net/cert_net/cert_net_fetcher_impl.cc

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/cert_net/cert_net_fetcher_impl.h"
   6
   7 #include "base/callback_helpers.h"
   8 #include "base/containers/linked_list.h"
   9 #include "base/logging.h"
  10 #include "base/numerics/safe_math.h"
  11 #include "base/stl_util.h"
  12 #include "base/timer/timer.h"
  13 #include "net/base/load_flags.h"
  14 #include "net/url_request/redirect_info.h"
  15 #include "net/url_request/url_request_context.h"
  16
  17 // TODO(eroman): Add support for POST parameters.
  18 // TODO(eroman): Add controls for bypassing the cache.
  19 // TODO(eroman): Add a maximum number of in-flight jobs/requests.
  20 // TODO(eroman): Add NetLog integration.
  21
  22 namespace net {
  23
  24 namespace {
  25
  26 // The size of the buffer used for reading the response body of the URLRequest.
  27 const int kReadBufferSizeInBytes = 4096;
  28
  29 // The maximum size in bytes for the response body when fetching a CRL.
  30 const int kMaxResponseSizeInBytesForCrl = 5 * 1024 * 1024;
  31
  32 // The maximum size in bytes for the response body when fetching an AIA URL
  33 // (caIssuers/OCSP).
  34 const int kMaxResponseSizeInBytesForAia = 64 * 1024;
  35
  36 // The default timeout in seconds for fetch requests.
  37 const int kTimeoutSeconds = 15;
  38
  39 // Policy for which URLs are allowed to be fetched. This is called both for the
  40 // initial URL and for each redirect. Returns OK on success or a net error
  41 // code on failure.
  42 Error CanFetchUrl(const GURL& url) {
  43   if (!url.SchemeIs("http"))
  44     return ERR_DISALLOWED_URL_SCHEME;
  45   return OK;
  46 }
  47
  48 base::TimeDelta GetTimeout(int timeout_milliseconds) {
  49   if (timeout_milliseconds == CertNetFetcher::DEFAULT)
  50     return base::TimeDelta::FromSeconds(kTimeoutSeconds);
  51   return base::TimeDelta::FromMilliseconds(timeout_milliseconds);
  52 }
  53
  54 size_t GetMaxResponseBytes(int max_response_bytes,
  55                            size_t default_max_response_bytes) {
  56   if (max_response_bytes == CertNetFetcher::DEFAULT)
  57     return default_max_response_bytes;
  58
  59   // Ensure that the specified limit is not negative, and cannot result in an
  60   // overflow while reading.
  61   base::CheckedNumeric<size_t> check(max_response_bytes);
  62   check += kReadBufferSizeInBytes;
  63   DCHECK(check.IsValid());
  64
  65   return max_response_bytes;
  66 }
  67
  68 enum HttpMethod {
  69   HTTP_METHOD_GET,
  70   HTTP_METHOD_POST,
  71 };
  72
  73 }  // namespace
  74
  75 // CertNetFetcherImpl::RequestImpl tracks an outstanding call to Fetch().
  76 class CertNetFetcherImpl::RequestImpl : public CertNetFetcher::Request,
  77                                         public base::LinkNode<RequestImpl> {
  78  public:
  79   RequestImpl(Job* job, const FetchCallback& callback)
  80       : callback_(callback), job_(job) {
  81     DCHECK(!callback.is_null());
  82   }
  83
  84   // Deletion cancels the outstanding request.
  85   ~RequestImpl() override;
  86
  87   void OnJobCancelled(Job* job) {
  88     DCHECK_EQ(job_, job);
  89     job_ = nullptr;
  90     callback_.Reset();
  91   }
  92
  93   void OnJobCompleted(Job* job,
  94                       Error error,
  95                       const std::vector<uint8_t>& response_body) {
  96     DCHECK_EQ(job_, job);
  97     job_ = nullptr;
  98     base::ResetAndReturn(&callback_).Run(error, response_body);
  99   }
 100
 101  private:
 102   // The callback to invoke when the request has completed.
 103   FetchCallback callback_;
 104
 105   // A non-owned pointer to the job that is executing the request.
 106   Job* job_;
 107
 108  private:
 109   DISALLOW_COPY_AND_ASSIGN(RequestImpl);
 110 };
 111
 112 struct CertNetFetcherImpl::RequestParams {
 113   RequestParams();
 114
 115   bool operator<(const RequestParams& other) const;
 116
 117   GURL url;
 118   HttpMethod http_method;
 119   size_t max_response_bytes;
 120
 121   // If set to a value <= 0 then means "no timeout".
 122   base::TimeDelta timeout;
 123
 124   // IMPORTANT: When adding fields to this structure, update operator<().
 125
 126  private:
 127   DISALLOW_COPY_AND_ASSIGN(RequestParams);
 128 };
 129
 130 CertNetFetcherImpl::RequestParams::RequestParams()
 131     : http_method(HTTP_METHOD_GET), max_response_bytes(0) {
 132 }
 133
 134 bool CertNetFetcherImpl::RequestParams::operator<(
 135     const RequestParams& other) const {
 136   if (url != other.url)
 137     return url < other.url;
 138   if (http_method != other.http_method)
 139     return http_method < other.http_method;
 140   if (max_response_bytes != other.max_response_bytes)
 141     return max_response_bytes < other.max_response_bytes;
 142   return timeout < other.timeout;
 143 }
 144
 145 // CertNetFetcherImpl::Job tracks an outstanding URLRequest as well as all of
 146 // the pending requests for it.
 147 class CertNetFetcherImpl::Job : public URLRequest::Delegate {
 148  public:
 149   Job(scoped_ptr<RequestParams> request_params, CertNetFetcherImpl* parent);
 150   ~Job() override;
 151
 152   // Cancels the job and all requests attached to it. No callbacks will be
 153   // invoked following cancellation.
 154   void Cancel();
 155
 156   const RequestParams& request_params() const { return *request_params_; }
 157
 158   // Create a request and attaches it to the job. When the job completes it will
 159   // notify the request of completion through OnJobCompleted. Note that the Job
 160   // does NOT own the request.
 161   scoped_ptr<Request> CreateRequest(const FetchCallback& callback);
 162
 163   // Removes |request| from the job.
 164   void DetachRequest(RequestImpl* request);
 165
 166   // Creates and starts a URLRequest for the job. After the request has
 167   // completed, OnJobCompleted() will be invoked and all the registered requests
 168   // notified of completion.
 169   void StartURLRequest(URLRequestContext* context);
 170
 171  private:
 172   // The pointers in RequestList are not owned by the Job.
 173   using RequestList = base::LinkedList<RequestImpl>;
 174
 175   // Implementation of URLRequest::Delegate
 176   void OnReceivedRedirect(URLRequest* request,
 177                           const RedirectInfo& redirect_info,
 178                           bool* defer_redirect) override;
 179   void OnResponseStarted(URLRequest* request) override;
 180   void OnReadCompleted(URLRequest* request, int bytes_read) override;
 181
 182   // Clears the URLRequest and timer. Helper for doing work common to
 183   // cancellation and job completion.
 184   void Stop();
 185
 186   // Reads as much data as available from |request|.
 187   void ReadBody(URLRequest* request);
 188
 189   // Helper to copy the partial bytes read from the read IOBuffer to an
 190   // aggregated buffer.
 191   bool ConsumeBytesRead(URLRequest* request, int num_bytes);
 192
 193   // Called once the job has exceeded its deadline.
 194   void OnTimeout();
 195
 196   // Called when the URLRequest has completed (either success or failure).
 197   void OnUrlRequestCompleted(URLRequest* request);
 198
 199   // Called when the Job has completed. The job may finish in response to a
 200   // timeout, an invalid URL, or the URLRequest completing. By the time this
 201   // method is called, the response variables have been assigned
 202   // (result_net_error_ and response_body_).
 203   void OnJobCompleted();
 204
 205   // The requests attached to this job.
 206   RequestList requests_;
 207
 208   // The input parameters for starting a URLRequest.
 209   scoped_ptr<RequestParams> request_params_;
 210
 211   // The URLRequest response information.
 212   std::vector<uint8_t> response_body_;
 213   Error result_net_error_;
 214
 215   scoped_ptr<URLRequest> url_request_;
 216   scoped_refptr<IOBuffer> read_buffer_;
 217
 218   // Used to timeout the job when the URLRequest takes too long. This timer is
 219   // also used for notifying a failure to start the URLRequest.
 220   base::OneShotTimer<Job> timer_;
 221
 222   // Non-owned pointer to the CertNetFetcherImpl that created this job.
 223   CertNetFetcherImpl* parent_;
 224
 225   DISALLOW_COPY_AND_ASSIGN(Job);
 226 };
 227
 228 CertNetFetcherImpl::RequestImpl::~RequestImpl() {
 229   if (job_)
 230     job_->DetachRequest(this);
 231 }
 232
 233 CertNetFetcherImpl::Job::Job(scoped_ptr<RequestParams> request_params,
 234                              CertNetFetcherImpl* parent)
 235     : request_params_(request_params.Pass()),
 236       result_net_error_(ERR_IO_PENDING),
 237       parent_(parent) {
 238 }
 239
 240 CertNetFetcherImpl::Job::~Job() {
 241   Cancel();
 242 }
 243
 244 void CertNetFetcherImpl::Job::Cancel() {
 245   parent_ = nullptr;
 246
 247   // Notify each request of cancellation and remove it from the list.
 248   for (base::LinkNode<RequestImpl>* current = requests_.head();
 249        current != requests_.end();) {
 250     base::LinkNode<RequestImpl>* next = current->next();
 251     current->value()->OnJobCancelled(this);
 252     current->RemoveFromList();
 253     current = next;
 254   }
 255
 256   DCHECK(requests_.empty());
 257
 258   Stop();
 259 }
 260
 261 scoped_ptr<CertNetFetcher::Request> CertNetFetcherImpl::Job::CreateRequest(
 262     const FetchCallback& callback) {
 263   scoped_ptr<RequestImpl> request(new RequestImpl(this, callback));
 264   requests_.Append(request.get());
 265   return request.Pass();
 266 }
 267
 268 void CertNetFetcherImpl::Job::DetachRequest(RequestImpl* request) {
 269   scoped_ptr<Job> delete_this;
 270
 271   request->RemoveFromList();
 272
 273   // If there are no longer any requests attached to the job then
 274   // cancel and delete it.
 275   if (requests_.empty() && !parent_->IsCurrentlyCompletingJob(this))
 276     delete_this = parent_->RemoveJob(this);
 277 }
 278
 279 void CertNetFetcherImpl::Job::StartURLRequest(URLRequestContext* context) {
 280   Error error = CanFetchUrl(request_params_->url);
 281   if (error != OK) {
 282     result_net_error_ = error;
 283     // The CertNetFetcher's API contract is that requests always complete
 284     // asynchronously. Use the timer class so the task is easily cancelled.
 285     timer_.Start(FROM_HERE, base::TimeDelta(), this, &Job::OnJobCompleted);
 286     return;
 287   }
 288
 289   // Start the URLRequest.
 290   read_buffer_ = new IOBuffer(kReadBufferSizeInBytes);
 291   url_request_ =
 292       context->CreateRequest(request_params_->url, DEFAULT_PRIORITY, this);
 293   if (request_params_->http_method == HTTP_METHOD_POST)
 294     url_request_->set_method("POST");
 295   url_request_->SetLoadFlags(LOAD_DO_NOT_SAVE_COOKIES |
 296                              LOAD_DO_NOT_SEND_COOKIES);
 297   url_request_->Start();
 298
 299   // Start a timer to limit how long the job runs for.
 300   if (request_params_->timeout > base::TimeDelta())
 301     timer_.Start(FROM_HERE, request_params_->timeout, this, &Job::OnTimeout);
 302 }
 303
 304 void CertNetFetcherImpl::Job::OnReceivedRedirect(
 305     URLRequest* request,
 306     const RedirectInfo& redirect_info,
 307     bool* defer_redirect) {
 308   DCHECK_EQ(url_request_.get(), request);
 309
 310   // Ensure that the new URL matches the policy.
 311   Error error = CanFetchUrl(redirect_info.new_url);
 312   if (error != OK) {
 313     request->CancelWithError(error);
 314     OnUrlRequestCompleted(request);
 315     return;
 316   }
 317 }
 318
 319 void CertNetFetcherImpl::Job::OnResponseStarted(URLRequest* request) {
 320   DCHECK_EQ(url_request_.get(), request);
 321
 322   if (!request->status().is_success()) {
 323     OnUrlRequestCompleted(request);
 324     return;
 325   }
 326
 327   if (request->GetResponseCode() != 200) {
 328     // TODO(eroman): Use a more specific error code.
 329     request->CancelWithError(ERR_FAILED);
 330     OnUrlRequestCompleted(request);
 331     return;
 332   }
 333
 334   ReadBody(request);
 335 }
 336
 337 void CertNetFetcherImpl::Job::OnReadCompleted(URLRequest* request,
 338                                               int bytes_read) {
 339   DCHECK_EQ(url_request_.get(), request);
 340
 341   // Keep reading the response body.
 342   if (ConsumeBytesRead(request, bytes_read))
 343     ReadBody(request);
 344 }
 345
 346 void CertNetFetcherImpl::Job::Stop() {
 347   timer_.Stop();
 348   url_request_.reset();
 349 }
 350
 351 void CertNetFetcherImpl::Job::ReadBody(URLRequest* request) {
 352   // Read as many bytes as are available synchronously.
 353   int num_bytes;
 354   while (
 355       request->Read(read_buffer_.get(), kReadBufferSizeInBytes, &num_bytes)) {
 356     if (!ConsumeBytesRead(request, num_bytes))
 357       return;
 358   }
 359
 360   // Check whether the read failed synchronously.
 361   if (!request->status().is_io_pending())
 362     OnUrlRequestCompleted(request);
 363   return;
 364 }
 365
 366 bool CertNetFetcherImpl::Job::ConsumeBytesRead(URLRequest* request,
 367                                                int num_bytes) {
 368   if (num_bytes <= 0) {
 369     // Error while reading, or EOF.
 370     OnUrlRequestCompleted(request);
 371     return false;
 372   }
 373
 374   // Enforce maximum size bound.
 375   if (num_bytes + response_body_.size() > request_params_->max_response_bytes) {
 376     request->CancelWithError(ERR_FILE_TOO_BIG);
 377     OnUrlRequestCompleted(request);
 378     return false;
 379   }
 380
 381   // Append the data to |response_body_|.
 382   response_body_.reserve(num_bytes);
 383   response_body_.insert(response_body_.end(), read_buffer_->data(),
 384                         read_buffer_->data() + num_bytes);
 385   return true;
 386 }
 387
 388 void CertNetFetcherImpl::Job::OnTimeout() {
 389   result_net_error_ = ERR_TIMED_OUT;
 390   url_request_->CancelWithError(result_net_error_);
 391   OnJobCompleted();
 392 }
 393
 394 void CertNetFetcherImpl::Job::OnUrlRequestCompleted(URLRequest* request) {
 395   DCHECK_EQ(request, url_request_.get());
 396
 397   if (request->status().is_success())
 398     result_net_error_ = OK;
 399   else
 400     result_net_error_ = static_cast<Error>(request->status().error());
 401
 402   OnJobCompleted();
 403 }
 404
 405 void CertNetFetcherImpl::Job::OnJobCompleted() {
 406   // Stop the timer and clear the URLRequest.
 407   Stop();
 408
 409   // Invoking the callbacks is subtle as state may be mutated while iterating
 410   // through the callbacks:
 411   //
 412   //   * The parent CertNetFetcherImpl may be deleted
 413   //   * Requests in this job may be cancelled
 414
 415   scoped_ptr<Job> delete_this = parent_->RemoveJob(this);
 416   parent_->SetCurrentlyCompletingJob(this);
 417
 418   while (!requests_.empty()) {
 419     base::LinkNode<RequestImpl>* request = requests_.head();
 420     request->RemoveFromList();
 421     request->value()->OnJobCompleted(this, result_net_error_, response_body_);
 422   }
 423
 424   if (parent_)
 425     parent_->ClearCurrentlyCompletingJob(this);
 426 }
 427
 428 CertNetFetcherImpl::CertNetFetcherImpl(URLRequestContext* context)
 429     : currently_completing_job_(nullptr), context_(context) {
 430 }
 431
 432 CertNetFetcherImpl::~CertNetFetcherImpl() {
 433   STLDeleteElements(&jobs_);
 434
 435   // The CertNetFetcherImpl was destroyed in a FetchCallback. Detach all
 436   // remaining requests from the job so no further callbacks are called.
 437   if (currently_completing_job_)
 438     currently_completing_job_->Cancel();
 439 }
 440
 441 scoped_ptr<CertNetFetcher::Request> CertNetFetcherImpl::FetchCaIssuers(
 442     const GURL& url,
 443     int timeout_milliseconds,
 444     int max_response_bytes,
 445     const FetchCallback& callback) {
 446   scoped_ptr<RequestParams> request_params(new RequestParams);
 447
 448   request_params->url = url;
 449   request_params->http_method = HTTP_METHOD_GET;
 450   request_params->timeout = GetTimeout(timeout_milliseconds);
 451   request_params->max_response_bytes =
 452       GetMaxResponseBytes(max_response_bytes, kMaxResponseSizeInBytesForAia);
 453
 454   return Fetch(request_params.Pass(), callback);
 455 }
 456
 457 scoped_ptr<CertNetFetcher::Request> CertNetFetcherImpl::FetchCrl(
 458     const GURL& url,
 459     int timeout_milliseconds,
 460     int max_response_bytes,
 461     const FetchCallback& callback) {
 462   scoped_ptr<RequestParams> request_params(new RequestParams);
 463
 464   request_params->url = url;
 465   request_params->http_method = HTTP_METHOD_GET;
 466   request_params->timeout = GetTimeout(timeout_milliseconds);
 467   request_params->max_response_bytes =
 468       GetMaxResponseBytes(max_response_bytes, kMaxResponseSizeInBytesForCrl);
 469
 470   return Fetch(request_params.Pass(), callback);
 471 }
 472
 473 scoped_ptr<CertNetFetcher::Request> CertNetFetcherImpl::FetchOcsp(
 474     const GURL& url,
 475     int timeout_milliseconds,
 476     int max_response_bytes,
 477     const FetchCallback& callback) {
 478   scoped_ptr<RequestParams> request_params(new RequestParams);
 479
 480   request_params->url = url;
 481   request_params->http_method = HTTP_METHOD_GET;
 482   request_params->timeout = GetTimeout(timeout_milliseconds);
 483   request_params->max_response_bytes =
 484       GetMaxResponseBytes(max_response_bytes, kMaxResponseSizeInBytesForAia);
 485
 486   return Fetch(request_params.Pass(), callback);
 487 }
 488
 489 bool CertNetFetcherImpl::JobComparator::operator()(const Job* job1,
 490                                                    const Job* job2) const {
 491   return job1->request_params() < job2->request_params();
 492 }
 493
 494 scoped_ptr<CertNetFetcher::Request> CertNetFetcherImpl::Fetch(
 495     scoped_ptr<RequestParams> request_params,
 496     const FetchCallback& callback) {
 497   DCHECK(thread_checker_.CalledOnValidThread());
 498
 499   // If there is an in-progress job that matches the request parameters use it.
 500   // Otherwise start a new job.
 501   Job* job = FindJob(*request_params);
 502
 503   if (!job) {
 504     job = new Job(request_params.Pass(), this);
 505     jobs_.insert(job);
 506     job->StartURLRequest(context_);
 507   }
 508
 509   return job->CreateRequest(callback);
 510 }
 511
 512 struct CertNetFetcherImpl::JobToRequestParamsComparator {
 513   bool operator()(const Job* job,
 514                   const CertNetFetcherImpl::RequestParams& value) const {
 515     return job->request_params() < value;
 516   }
 517 };
 518
 519 CertNetFetcherImpl::Job* CertNetFetcherImpl::FindJob(
 520     const RequestParams& params) {
 521   DCHECK(thread_checker_.CalledOnValidThread());
 522
 523   // The JobSet is kept in sorted order so items can be found using binary
 524   // search.
 525   JobSet::iterator it = std::lower_bound(jobs_.begin(), jobs_.end(), params,
 526                                          JobToRequestParamsComparator());
 527   if (it != jobs_.end() && !(params < (*it)->request_params()))
 528     return *it;
 529   return nullptr;
 530 }
 531
 532 scoped_ptr<CertNetFetcherImpl::Job> CertNetFetcherImpl::RemoveJob(Job* job) {
 533   DCHECK(thread_checker_.CalledOnValidThread());
 534   bool erased_job = jobs_.erase(job) == 1;
 535   CHECK(erased_job);
 536   return make_scoped_ptr(job);
 537 }
 538
 539 void CertNetFetcherImpl::SetCurrentlyCompletingJob(Job* job) {
 540   DCHECK(!currently_completing_job_);
 541   DCHECK(job);
 542   currently_completing_job_ = job;
 543 }
 544
 545 void CertNetFetcherImpl::ClearCurrentlyCompletingJob(Job* job) {
 546   DCHECK_EQ(currently_completing_job_, job);
 547   currently_completing_job_ = nullptr;
 548 }
 549
 550 bool CertNetFetcherImpl::IsCurrentlyCompletingJob(Job* job) {
 551   return job == currently_completing_job_;
 552 }
 553
 554 }  // namespace net