net/base/sdch_manager.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/base/sdch_manager.h"
   6
   7 #include "base/base64.h"
   8 #include "base/logging.h"
   9 #include "base/metrics/histogram.h"
  10 #include "base/strings/string_number_conversions.h"
  11 #include "base/strings/string_util.h"
  12 #include "crypto/sha2.h"
  13 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
  14 #include "net/url_request/url_request_http_job.h"
  15
  16 namespace net {
  17
  18 //------------------------------------------------------------------------------
  19 // static
  20
  21 // Adjust SDCH limits downwards for mobile.
  22 #if defined(OS_ANDROID) || defined(OS_IOS)
  23 // static
  24 const size_t SdchManager::kMaxDictionaryCount = 1;
  25 const size_t SdchManager::kMaxDictionarySize = 500 * 1000;
  26 #else
  27 // static
  28 const size_t SdchManager::kMaxDictionaryCount = 20;
  29 const size_t SdchManager::kMaxDictionarySize = 1000 * 1000;
  30 #endif
  31
  32 // static
  33 bool SdchManager::g_sdch_enabled_ = true;
  34
  35 // static
  36 bool SdchManager::g_secure_scheme_supported_ = false;
  37
  38 //------------------------------------------------------------------------------
  39 SdchManager::Dictionary::Dictionary(const std::string& dictionary_text,
  40                                     size_t offset,
  41                                     const std::string& client_hash,
  42                                     const GURL& gurl,
  43                                     const std::string& domain,
  44                                     const std::string& path,
  45                                     const base::Time& expiration,
  46                                     const std::set<int>& ports)
  47     : text_(dictionary_text, offset),
  48       client_hash_(client_hash),
  49       url_(gurl),
  50       domain_(domain),
  51       path_(path),
  52       expiration_(expiration),
  53       ports_(ports) {
  54 }
  55
  56 SdchManager::Dictionary::~Dictionary() {
  57 }
  58
  59 bool SdchManager::Dictionary::CanAdvertise(const GURL& target_url) {
  60   /* The specific rules of when a dictionary should be advertised in an
  61      Avail-Dictionary header are modeled after the rules for cookie scoping. The
  62      terms "domain-match" and "pathmatch" are defined in RFC 2965 [6]. A
  63      dictionary may be advertised in the Avail-Dictionaries header exactly when
  64      all of the following are true:
  65       1. The server's effective host name domain-matches the Domain attribute of
  66          the dictionary.
  67       2. If the dictionary has a Port attribute, the request port is one of the
  68          ports listed in the Port attribute.
  69       3. The request URI path-matches the path header of the dictionary.
  70       4. The request is not an HTTPS request.
  71      We can override (ignore) item (4) only when we have explicitly enabled
  72      HTTPS support AND the dictionary acquisition scheme matches the target
  73      url scheme.
  74     */
  75   if (!DomainMatch(target_url, domain_))
  76     return false;
  77   if (!ports_.empty() && 0 == ports_.count(target_url.EffectiveIntPort()))
  78     return false;
  79   if (path_.size() && !PathMatch(target_url.path(), path_))
  80     return false;
  81   if (!SdchManager::secure_scheme_supported() && target_url.SchemeIsSecure())
  82     return false;
  83   if (target_url.SchemeIsSecure() != url_.SchemeIsSecure())
  84     return false;
  85   if (base::Time::Now() > expiration_)
  86     return false;
  87   return true;
  88 }
  89
  90 //------------------------------------------------------------------------------
  91 // Security functions restricting loads and use of dictionaries.
  92
  93 // static
  94 bool SdchManager::Dictionary::CanSet(const std::string& domain,
  95                                      const std::string& path,
  96                                      const std::set<int>& ports,
  97                                      const GURL& dictionary_url) {
  98   /*
  99   A dictionary is invalid and must not be stored if any of the following are
 100   true:
 101     1. The dictionary has no Domain attribute.
 102     2. The effective host name that derives from the referer URL host name does
 103       not domain-match the Domain attribute.
 104     3. The Domain attribute is a top level domain.
 105     4. The referer URL host is a host domain name (not IP address) and has the
 106       form HD, where D is the value of the Domain attribute, and H is a string
 107       that contains one or more dots.
 108     5. If the dictionary has a Port attribute and the referer URL's port was not
 109       in the list.
 110   */
 111
 112   // TODO(jar): Redirects in dictionary fetches might plausibly be problematic,
 113   // and hence the conservative approach is to not allow any redirects (if there
 114   // were any... then don't allow the dictionary to be set).
 115
 116   if (domain.empty()) {
 117     SdchErrorRecovery(DICTIONARY_MISSING_DOMAIN_SPECIFIER);
 118     return false;  // Domain is required.
 119   }
 120   if (registry_controlled_domains::GetDomainAndRegistry(
 121         domain,
 122         registry_controlled_domains::INCLUDE_PRIVATE_REGISTRIES).empty()) {
 123     SdchErrorRecovery(DICTIONARY_SPECIFIES_TOP_LEVEL_DOMAIN);
 124     return false;  // domain was a TLD.
 125   }
 126   if (!Dictionary::DomainMatch(dictionary_url, domain)) {
 127     SdchErrorRecovery(DICTIONARY_DOMAIN_NOT_MATCHING_SOURCE_URL);
 128     return false;
 129   }
 130
 131   std::string referrer_url_host = dictionary_url.host();
 132   size_t postfix_domain_index = referrer_url_host.rfind(domain);
 133   // See if it is indeed a postfix, or just an internal string.
 134   if (referrer_url_host.size() == postfix_domain_index + domain.size()) {
 135     // It is a postfix... so check to see if there's a dot in the prefix.
 136     size_t end_of_host_index = referrer_url_host.find_first_of('.');
 137     if (referrer_url_host.npos != end_of_host_index  &&
 138         end_of_host_index < postfix_domain_index) {
 139       SdchErrorRecovery(DICTIONARY_REFERER_URL_HAS_DOT_IN_PREFIX);
 140       return false;
 141     }
 142   }
 143
 144   if (!ports.empty()
 145       && 0 == ports.count(dictionary_url.EffectiveIntPort())) {
 146     SdchErrorRecovery(DICTIONARY_PORT_NOT_MATCHING_SOURCE_URL);
 147     return false;
 148   }
 149   return true;
 150 }
 151
 152 // static
 153 bool SdchManager::Dictionary::CanUse(const GURL& referring_url) {
 154   /*
 155     1. The request URL's host name domain-matches the Domain attribute of the
 156       dictionary.
 157     2. If the dictionary has a Port attribute, the request port is one of the
 158       ports listed in the Port attribute.
 159     3. The request URL path-matches the path attribute of the dictionary.
 160     4. The request is not an HTTPS request.
 161     We can override (ignore) item (4) only when we have explicitly enabled
 162     HTTPS support AND the dictionary acquisition scheme matches the target
 163      url scheme.
 164   */
 165   if (!DomainMatch(referring_url, domain_)) {
 166     SdchErrorRecovery(DICTIONARY_FOUND_HAS_WRONG_DOMAIN);
 167     return false;
 168   }
 169   if (!ports_.empty()
 170       && 0 == ports_.count(referring_url.EffectiveIntPort())) {
 171     SdchErrorRecovery(DICTIONARY_FOUND_HAS_WRONG_PORT_LIST);
 172     return false;
 173   }
 174   if (path_.size() && !PathMatch(referring_url.path(), path_)) {
 175     SdchErrorRecovery(DICTIONARY_FOUND_HAS_WRONG_PATH);
 176     return false;
 177   }
 178   if (!SdchManager::secure_scheme_supported() &&
 179       referring_url.SchemeIsSecure()) {
 180     SdchErrorRecovery(DICTIONARY_FOUND_HAS_WRONG_SCHEME);
 181     return false;
 182   }
 183   if (referring_url.SchemeIsSecure() != url_.SchemeIsSecure()) {
 184     SdchErrorRecovery(DICTIONARY_FOUND_HAS_WRONG_SCHEME);
 185     return false;
 186   }
 187
 188   // TODO(jar): Remove overly restrictive failsafe test (added per security
 189   // review) when we have a need to be more general.
 190   if (!referring_url.SchemeIsHTTPOrHTTPS()) {
 191     SdchErrorRecovery(ATTEMPT_TO_DECODE_NON_HTTP_DATA);
 192     return false;
 193   }
 194
 195   return true;
 196 }
 197
 198 bool SdchManager::Dictionary::PathMatch(const std::string& path,
 199                                         const std::string& restriction) {
 200   /*  Must be either:
 201   1. P2 is equal to P1
 202   2. P2 is a prefix of P1 and either the final character in P2 is "/" or the
 203       character following P2 in P1 is "/".
 204       */
 205   if (path == restriction)
 206     return true;
 207   size_t prefix_length = restriction.size();
 208   if (prefix_length > path.size())
 209     return false;  // Can't be a prefix.
 210   if (0 != path.compare(0, prefix_length, restriction))
 211     return false;
 212   return restriction[prefix_length - 1] == '/' || path[prefix_length] == '/';
 213 }
 214
 215 // static
 216 bool SdchManager::Dictionary::DomainMatch(const GURL& gurl,
 217                                           const std::string& restriction) {
 218   // TODO(jar): This is not precisely a domain match definition.
 219   return gurl.DomainIs(restriction.data(), restriction.size());
 220 }
 221
 222 //------------------------------------------------------------------------------
 223 SdchManager::SdchManager()
 224     : fetches_count_for_testing_(0) {
 225   DCHECK(CalledOnValidThread());
 226 }
 227
 228 SdchManager::~SdchManager() {
 229   DCHECK(CalledOnValidThread());
 230   while (!dictionaries_.empty()) {
 231     DictionaryMap::iterator it = dictionaries_.begin();
 232     dictionaries_.erase(it->first);
 233   }
 234 }
 235
 236 void SdchManager::ClearData() {
 237   blacklisted_domains_.clear();
 238   allow_latency_experiment_.clear();
 239   if (fetcher_.get())
 240     fetcher_->Cancel();
 241
 242   // Note that this may result in not having dictionaries we've advertised
 243   // for incoming responses.  The window is relatively small (as ClearData()
 244   // is not expected to be called frequently), so we rely on meta-refresh
 245   // to handle this case.
 246   dictionaries_.clear();
 247 }
 248
 249 // static
 250 void SdchManager::SdchErrorRecovery(ProblemCodes problem) {
 251   UMA_HISTOGRAM_ENUMERATION("Sdch3.ProblemCodes_4", problem, MAX_PROBLEM_CODE);
 252 }
 253
 254 void SdchManager::set_sdch_fetcher(scoped_ptr<SdchFetcher> fetcher) {
 255   DCHECK(CalledOnValidThread());
 256   fetcher_ = fetcher.Pass();
 257 }
 258
 259 // static
 260 void SdchManager::EnableSdchSupport(bool enabled) {
 261   g_sdch_enabled_ = enabled;
 262 }
 263
 264 // static
 265 void SdchManager::EnableSecureSchemeSupport(bool enabled) {
 266   g_secure_scheme_supported_ = enabled;
 267 }
 268
 269 void SdchManager::BlacklistDomain(const GURL& url,
 270                                   ProblemCodes blacklist_reason) {
 271   SetAllowLatencyExperiment(url, false);
 272
 273   BlacklistInfo* blacklist_info =
 274       &blacklisted_domains_[base::StringToLowerASCII(url.host())];
 275
 276   if (blacklist_info->count > 0)
 277     return;  // Domain is already blacklisted.
 278
 279   if (blacklist_info->exponential_count > (INT_MAX - 1) / 2) {
 280     blacklist_info->exponential_count = INT_MAX;
 281   } else {
 282     blacklist_info->exponential_count =
 283         blacklist_info->exponential_count * 2 + 1;
 284   }
 285
 286   blacklist_info->count = blacklist_info->exponential_count;
 287   blacklist_info->reason = blacklist_reason;
 288 }
 289
 290 void SdchManager::BlacklistDomainForever(const GURL& url,
 291                                          ProblemCodes blacklist_reason) {
 292   SetAllowLatencyExperiment(url, false);
 293
 294   BlacklistInfo* blacklist_info =
 295       &blacklisted_domains_[base::StringToLowerASCII(url.host())];
 296   blacklist_info->count = INT_MAX;
 297   blacklist_info->exponential_count = INT_MAX;
 298   blacklist_info->reason = blacklist_reason;
 299 }
 300
 301 void SdchManager::ClearBlacklistings() {
 302   blacklisted_domains_.clear();
 303 }
 304
 305 void SdchManager::ClearDomainBlacklisting(const std::string& domain) {
 306   BlacklistInfo* blacklist_info = &blacklisted_domains_[
 307       base::StringToLowerASCII(domain)];
 308   blacklist_info->count = 0;
 309   blacklist_info->reason = MIN_PROBLEM_CODE;
 310 }
 311
 312 int SdchManager::BlackListDomainCount(const std::string& domain) {
 313   std::string domain_lower(base::StringToLowerASCII(domain));
 314
 315   if (blacklisted_domains_.end() == blacklisted_domains_.find(domain_lower))
 316     return 0;
 317   return blacklisted_domains_[domain_lower].count;
 318 }
 319
 320 int SdchManager::BlacklistDomainExponential(const std::string& domain) {
 321   std::string domain_lower(base::StringToLowerASCII(domain));
 322
 323   if (blacklisted_domains_.end() == blacklisted_domains_.find(domain_lower))
 324     return 0;
 325   return blacklisted_domains_[domain_lower].exponential_count;
 326 }
 327
 328 bool SdchManager::IsInSupportedDomain(const GURL& url) {
 329   DCHECK(CalledOnValidThread());
 330   if (!g_sdch_enabled_ )
 331     return false;
 332
 333   if (!secure_scheme_supported() && url.SchemeIsSecure())
 334     return false;
 335
 336   if (blacklisted_domains_.empty())
 337     return true;
 338
 339   DomainBlacklistInfo::iterator it =
 340       blacklisted_domains_.find(base::StringToLowerASCII(url.host()));
 341   if (blacklisted_domains_.end() == it || it->second.count == 0)
 342     return true;
 343
 344   UMA_HISTOGRAM_ENUMERATION("Sdch3.BlacklistReason", it->second.reason,
 345                             MAX_PROBLEM_CODE);
 346   SdchErrorRecovery(DOMAIN_BLACKLIST_INCLUDES_TARGET);
 347
 348   int count = it->second.count - 1;
 349   if (count > 0) {
 350     it->second.count = count;
 351   } else {
 352     it->second.count = 0;
 353     it->second.reason = MIN_PROBLEM_CODE;
 354   }
 355
 356   return false;
 357 }
 358
 359 void SdchManager::FetchDictionary(const GURL& request_url,
 360                                   const GURL& dictionary_url) {
 361   DCHECK(CalledOnValidThread());
 362   if (CanFetchDictionary(request_url, dictionary_url) && fetcher_.get()) {
 363     ++fetches_count_for_testing_;
 364     fetcher_->Schedule(dictionary_url);
 365   }
 366 }
 367
 368 bool SdchManager::CanFetchDictionary(const GURL& referring_url,
 369                                      const GURL& dictionary_url) const {
 370   DCHECK(CalledOnValidThread());
 371   /* The user agent may retrieve a dictionary from the dictionary URL if all of
 372      the following are true:
 373        1 The dictionary URL host name matches the referrer URL host name and
 374            scheme.
 375        2 The dictionary URL host name domain matches the parent domain of the
 376            referrer URL host name
 377        3 The parent domain of the referrer URL host name is not a top level
 378            domain
 379        4 The dictionary URL is not an HTTPS URL.
 380    */
 381   // Item (1) above implies item (2).  Spec should be updated.
 382   // I take "host name match" to be "is identical to"
 383   if (referring_url.host() != dictionary_url.host() ||
 384       referring_url.scheme() != dictionary_url.scheme()) {
 385     SdchErrorRecovery(DICTIONARY_LOAD_ATTEMPT_FROM_DIFFERENT_HOST);
 386     return false;
 387   }
 388   if (!secure_scheme_supported() && referring_url.SchemeIsSecure()) {
 389     SdchErrorRecovery(DICTIONARY_SELECTED_FOR_SSL);
 390     return false;
 391   }
 392
 393   // TODO(jar): Remove this failsafe conservative hack which is more restrictive
 394   // than current SDCH spec when needed, and justified by security audit.
 395   if (!referring_url.SchemeIsHTTPOrHTTPS()) {
 396     SdchErrorRecovery(DICTIONARY_SELECTED_FROM_NON_HTTP);
 397     return false;
 398   }
 399
 400   return true;
 401 }
 402
 403 void SdchManager::GetVcdiffDictionary(
 404     const std::string& server_hash,
 405     const GURL& referring_url,
 406     scoped_refptr<Dictionary>* dictionary) {
 407   DCHECK(CalledOnValidThread());
 408   *dictionary = NULL;
 409   DictionaryMap::iterator it = dictionaries_.find(server_hash);
 410   if (it == dictionaries_.end()) {
 411     return;
 412   }
 413   scoped_refptr<Dictionary> matching_dictionary = it->second;
 414   if (!IsInSupportedDomain(referring_url))
 415     return;
 416   if (!matching_dictionary->CanUse(referring_url))
 417     return;
 418   *dictionary = matching_dictionary;
 419 }
 420
 421 // TODO(jar): If we have evictions from the dictionaries_, then we need to
 422 // change this interface to return a list of reference counted Dictionary
 423 // instances that can be used if/when a server specifies one.
 424 void SdchManager::GetAvailDictionaryList(const GURL& target_url,
 425                                          std::string* list) {
 426   DCHECK(CalledOnValidThread());
 427   int count = 0;
 428   for (DictionaryMap::iterator it = dictionaries_.begin();
 429        it != dictionaries_.end(); ++it) {
 430     if (!IsInSupportedDomain(target_url))
 431       continue;
 432     if (!it->second->CanAdvertise(target_url))
 433       continue;
 434     ++count;
 435     if (!list->empty())
 436       list->append(",");
 437     list->append(it->second->client_hash());
 438   }
 439   // Watch to see if we have corrupt or numerous dictionaries.
 440   if (count > 0)
 441     UMA_HISTOGRAM_COUNTS("Sdch3.Advertisement_Count", count);
 442 }
 443
 444 // static
 445 void SdchManager::GenerateHash(const std::string& dictionary_text,
 446     std::string* client_hash, std::string* server_hash) {
 447   char binary_hash[32];
 448   crypto::SHA256HashString(dictionary_text, binary_hash, sizeof(binary_hash));
 449
 450   std::string first_48_bits(&binary_hash[0], 6);
 451   std::string second_48_bits(&binary_hash[6], 6);
 452   UrlSafeBase64Encode(first_48_bits, client_hash);
 453   UrlSafeBase64Encode(second_48_bits, server_hash);
 454
 455   DCHECK_EQ(server_hash->length(), 8u);
 456   DCHECK_EQ(client_hash->length(), 8u);
 457 }
 458
 459 //------------------------------------------------------------------------------
 460 // Methods for supporting latency experiments.
 461
 462 bool SdchManager::AllowLatencyExperiment(const GURL& url) const {
 463   DCHECK(CalledOnValidThread());
 464   return allow_latency_experiment_.end() !=
 465       allow_latency_experiment_.find(url.host());
 466 }
 467
 468 void SdchManager::SetAllowLatencyExperiment(const GURL& url, bool enable) {
 469   DCHECK(CalledOnValidThread());
 470   if (enable) {
 471     allow_latency_experiment_.insert(url.host());
 472     return;
 473   }
 474   ExperimentSet::iterator it = allow_latency_experiment_.find(url.host());
 475   if (allow_latency_experiment_.end() == it)
 476     return;  // It was already erased, or never allowed.
 477   SdchErrorRecovery(LATENCY_TEST_DISALLOWED);
 478   allow_latency_experiment_.erase(it);
 479 }
 480
 481 void SdchManager::AddSdchDictionary(const std::string& dictionary_text,
 482     const GURL& dictionary_url) {
 483   DCHECK(CalledOnValidThread());
 484   std::string client_hash;
 485   std::string server_hash;
 486   GenerateHash(dictionary_text, &client_hash, &server_hash);
 487   if (dictionaries_.find(server_hash) != dictionaries_.end()) {
 488     SdchErrorRecovery(DICTIONARY_ALREADY_LOADED);
 489     return;                             // Already loaded.
 490   }
 491
 492   std::string domain, path;
 493   std::set<int> ports;
 494   base::Time expiration(base::Time::Now() + base::TimeDelta::FromDays(30));
 495
 496   if (dictionary_text.empty()) {
 497     SdchErrorRecovery(DICTIONARY_HAS_NO_TEXT);
 498     return;                             // Missing header.
 499   }
 500
 501   size_t header_end = dictionary_text.find("\n\n");
 502   if (std::string::npos == header_end) {
 503     SdchErrorRecovery(DICTIONARY_HAS_NO_HEADER);
 504     return;                             // Missing header.
 505   }
 506   size_t line_start = 0;  // Start of line being parsed.
 507   while (1) {
 508     size_t line_end = dictionary_text.find('\n', line_start);
 509     DCHECK(std::string::npos != line_end);
 510     DCHECK_LE(line_end, header_end);
 511
 512     size_t colon_index = dictionary_text.find(':', line_start);
 513     if (std::string::npos == colon_index) {
 514       SdchErrorRecovery(DICTIONARY_HEADER_LINE_MISSING_COLON);
 515       return;                         // Illegal line missing a colon.
 516     }
 517
 518     if (colon_index > line_end)
 519       break;
 520
 521     size_t value_start = dictionary_text.find_first_not_of(" \t",
 522                                                            colon_index + 1);
 523     if (std::string::npos != value_start) {
 524       if (value_start >= line_end)
 525         break;
 526       std::string name(dictionary_text, line_start, colon_index - line_start);
 527       std::string value(dictionary_text, value_start, line_end - value_start);
 528       name = base::StringToLowerASCII(name);
 529       if (name == "domain") {
 530         domain = value;
 531       } else if (name == "path") {
 532         path = value;
 533       } else if (name == "format-version") {
 534         if (value != "1.0")
 535           return;
 536       } else if (name == "max-age") {
 537         int64 seconds;
 538         base::StringToInt64(value, &seconds);
 539         expiration = base::Time::Now() + base::TimeDelta::FromSeconds(seconds);
 540       } else if (name == "port") {
 541         int port;
 542         base::StringToInt(value, &port);
 543         if (port >= 0)
 544           ports.insert(port);
 545       }
 546     }
 547
 548     if (line_end >= header_end)
 549       break;
 550     line_start = line_end + 1;
 551   }
 552
 553   if (!IsInSupportedDomain(dictionary_url))
 554     return;
 555
 556   if (!Dictionary::CanSet(domain, path, ports, dictionary_url))
 557     return;
 558
 559   // TODO(jar): Remove these hacks to preclude a DOS attack involving piles of
 560   // useless dictionaries.  We should probably have a cache eviction plan,
 561   // instead of just blocking additions.  For now, with the spec in flux, it
 562   // is probably not worth doing eviction handling.
 563   if (kMaxDictionarySize < dictionary_text.size()) {
 564     SdchErrorRecovery(DICTIONARY_IS_TOO_LARGE);
 565     return;
 566   }
 567   if (kMaxDictionaryCount <= dictionaries_.size()) {
 568     SdchErrorRecovery(DICTIONARY_COUNT_EXCEEDED);
 569     return;
 570   }
 571
 572   UMA_HISTOGRAM_COUNTS("Sdch3.Dictionary size loaded", dictionary_text.size());
 573   DVLOG(1) << "Loaded dictionary with client hash " << client_hash
 574            << " and server hash " << server_hash;
 575   Dictionary* dictionary =
 576       new Dictionary(dictionary_text, header_end + 2, client_hash,
 577                      dictionary_url, domain, path, expiration, ports);
 578   dictionaries_[server_hash] = dictionary;
 579   return;
 580 }
 581
 582 // static
 583 void SdchManager::UrlSafeBase64Encode(const std::string& input,
 584                                       std::string* output) {
 585   // Since this is only done during a dictionary load, and hashes are only 8
 586   // characters, we just do the simple fixup, rather than rewriting the encoder.
 587   base::Base64Encode(input, output);
 588   std::replace(output->begin(), output->end(), '+', '-');
 589   std::replace(output->begin(), output->end(), '/', '_');
 590 }
 591
 592 }  // namespace net