net/base/sdch_manager.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/base/sdch_manager.h"
   6
   7 #include "base/base64.h"
   8 #include "base/logging.h"
   9 #include "base/metrics/histogram.h"
  10 #include "base/strings/string_number_conversions.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/time/default_clock.h"
  13 #include "base/values.h"
  14 #include "crypto/sha2.h"
  15 #include "net/base/sdch_observer.h"
  16 #include "net/url_request/url_request_http_job.h"
  17
  18 namespace {
  19
  20 void StripTrailingDot(GURL* gurl) {
  21   std::string host(gurl->host());
  22
  23   if (host.empty())
  24     return;
  25
  26   if (*host.rbegin() != '.')
  27     return;
  28
  29   host.resize(host.size() - 1);
  30
  31   GURL::Replacements replacements;
  32   replacements.SetHostStr(host);
  33   *gurl = gurl->ReplaceComponents(replacements);
  34   return;
  35 }
  36
  37 }  // namespace
  38
  39 namespace net {
  40
  41 // static
  42 bool SdchManager::g_sdch_enabled_ = true;
  43
  44 // static
  45 bool SdchManager::g_secure_scheme_supported_ = true;
  46
  47 SdchManager::DictionarySet::DictionarySet() {}
  48
  49 SdchManager::DictionarySet::~DictionarySet() {}
  50
  51 std::string SdchManager::DictionarySet::GetDictionaryClientHashList() const {
  52   std::string result;
  53   bool first = true;
  54   for (const auto& entry: dictionaries_) {
  55     if (!first)
  56       result.append(",");
  57
  58     result.append(entry.second->data.client_hash());
  59     first = false;
  60   }
  61   return result;
  62 }
  63
  64 bool SdchManager::DictionarySet::Empty() const {
  65   return dictionaries_.empty();
  66 }
  67
  68 const std::string* SdchManager::DictionarySet::GetDictionaryText(
  69     const std::string& server_hash) const {
  70   auto it = dictionaries_.find(server_hash);
  71   if (it == dictionaries_.end())
  72     return nullptr;
  73   return &it->second->data.text();
  74 }
  75
  76 void SdchManager::DictionarySet::AddDictionary(
  77     const std::string& server_hash,
  78     const scoped_refptr<base::RefCountedData<SdchDictionary>>& dictionary) {
  79   DCHECK(dictionaries_.end() == dictionaries_.find(server_hash));
  80
  81   dictionaries_[server_hash] = dictionary;
  82 }
  83
  84 SdchManager::SdchManager() {
  85   DCHECK(thread_checker_.CalledOnValidThread());
  86 }
  87
  88 SdchManager::~SdchManager() {
  89   DCHECK(thread_checker_.CalledOnValidThread());
  90   while (!dictionaries_.empty()) {
  91     auto it = dictionaries_.begin();
  92     dictionaries_.erase(it->first);
  93   }
  94 }
  95
  96 void SdchManager::ClearData() {
  97   blacklisted_domains_.clear();
  98   allow_latency_experiment_.clear();
  99   dictionaries_.clear();
 100   FOR_EACH_OBSERVER(SdchObserver, observers_, OnClearDictionaries());
 101 }
 102
 103 // static
 104 void SdchManager::SdchErrorRecovery(SdchProblemCode problem) {
 105   UMA_HISTOGRAM_ENUMERATION("Sdch3.ProblemCodes_5", problem,
 106                             SDCH_MAX_PROBLEM_CODE);
 107 }
 108
 109 // static
 110 void SdchManager::EnableSdchSupport(bool enabled) {
 111   g_sdch_enabled_ = enabled;
 112 }
 113
 114 // static
 115 void SdchManager::EnableSecureSchemeSupport(bool enabled) {
 116   g_secure_scheme_supported_ = enabled;
 117 }
 118
 119 void SdchManager::BlacklistDomain(const GURL& url,
 120                                   SdchProblemCode blacklist_reason) {
 121   SetAllowLatencyExperiment(url, false);
 122
 123   BlacklistInfo* blacklist_info =
 124       &blacklisted_domains_[base::StringToLowerASCII(url.host())];
 125
 126   if (blacklist_info->count > 0)
 127     return;  // Domain is already blacklisted.
 128
 129   if (blacklist_info->exponential_count > (INT_MAX - 1) / 2) {
 130     blacklist_info->exponential_count = INT_MAX;
 131   } else {
 132     blacklist_info->exponential_count =
 133         blacklist_info->exponential_count * 2 + 1;
 134   }
 135
 136   blacklist_info->count = blacklist_info->exponential_count;
 137   blacklist_info->reason = blacklist_reason;
 138 }
 139
 140 void SdchManager::BlacklistDomainForever(const GURL& url,
 141                                          SdchProblemCode blacklist_reason) {
 142   SetAllowLatencyExperiment(url, false);
 143
 144   BlacklistInfo* blacklist_info =
 145       &blacklisted_domains_[base::StringToLowerASCII(url.host())];
 146   blacklist_info->count = INT_MAX;
 147   blacklist_info->exponential_count = INT_MAX;
 148   blacklist_info->reason = blacklist_reason;
 149 }
 150
 151 void SdchManager::ClearBlacklistings() {
 152   blacklisted_domains_.clear();
 153 }
 154
 155 void SdchManager::ClearDomainBlacklisting(const std::string& domain) {
 156   BlacklistInfo* blacklist_info = &blacklisted_domains_[
 157       base::StringToLowerASCII(domain)];
 158   blacklist_info->count = 0;
 159   blacklist_info->reason = SDCH_OK;
 160 }
 161
 162 int SdchManager::BlackListDomainCount(const std::string& domain) {
 163   std::string domain_lower(base::StringToLowerASCII(domain));
 164
 165   if (blacklisted_domains_.end() == blacklisted_domains_.find(domain_lower))
 166     return 0;
 167   return blacklisted_domains_[domain_lower].count;
 168 }
 169
 170 int SdchManager::BlacklistDomainExponential(const std::string& domain) {
 171   std::string domain_lower(base::StringToLowerASCII(domain));
 172
 173   if (blacklisted_domains_.end() == blacklisted_domains_.find(domain_lower))
 174     return 0;
 175   return blacklisted_domains_[domain_lower].exponential_count;
 176 }
 177
 178 SdchProblemCode SdchManager::IsInSupportedDomain(const GURL& url) {
 179   DCHECK(thread_checker_.CalledOnValidThread());
 180   if (!g_sdch_enabled_ )
 181     return SDCH_DISABLED;
 182
 183   if (!secure_scheme_supported() && url.SchemeIsCryptographic())
 184     return SDCH_SECURE_SCHEME_NOT_SUPPORTED;
 185
 186   if (blacklisted_domains_.empty())
 187     return SDCH_OK;
 188
 189   DomainBlacklistInfo::iterator it =
 190       blacklisted_domains_.find(base::StringToLowerASCII(url.host()));
 191   if (blacklisted_domains_.end() == it || it->second.count == 0)
 192     return SDCH_OK;
 193
 194   UMA_HISTOGRAM_ENUMERATION("Sdch3.BlacklistReason", it->second.reason,
 195                             SDCH_MAX_PROBLEM_CODE);
 196
 197   int count = it->second.count - 1;
 198   if (count > 0) {
 199     it->second.count = count;
 200   } else {
 201     it->second.count = 0;
 202     it->second.reason = SDCH_OK;
 203   }
 204
 205   return SDCH_DOMAIN_BLACKLIST_INCLUDES_TARGET;
 206 }
 207
 208 SdchProblemCode SdchManager::OnGetDictionary(const GURL& request_url,
 209                                              const GURL& dictionary_url) {
 210   DCHECK(thread_checker_.CalledOnValidThread());
 211   SdchProblemCode rv = CanFetchDictionary(request_url, dictionary_url);
 212   if (rv != SDCH_OK)
 213     return rv;
 214
 215   FOR_EACH_OBSERVER(SdchObserver,
 216                     observers_,
 217                     OnGetDictionary(request_url, dictionary_url));
 218
 219   return SDCH_OK;
 220 }
 221
 222 void SdchManager::OnDictionaryUsed(const std::string& server_hash) {
 223   FOR_EACH_OBSERVER(SdchObserver, observers_,
 224                     OnDictionaryUsed(server_hash));
 225 }
 226
 227 SdchProblemCode SdchManager::CanFetchDictionary(
 228     const GURL& referring_url,
 229     const GURL& dictionary_url) const {
 230   DCHECK(thread_checker_.CalledOnValidThread());
 231   /* The user agent may retrieve a dictionary from the dictionary URL if all of
 232      the following are true:
 233        1 The dictionary URL host name matches the referrer URL host name and
 234            scheme.
 235        2 The dictionary URL host name domain matches the parent domain of the
 236            referrer URL host name
 237        3 The parent domain of the referrer URL host name is not a top level
 238            domain
 239    */
 240   // Item (1) above implies item (2). Spec should be updated.
 241   // I take "host name match" to be "is identical to"
 242   if (referring_url.host() != dictionary_url.host() ||
 243       referring_url.scheme() != dictionary_url.scheme())
 244     return SDCH_DICTIONARY_LOAD_ATTEMPT_FROM_DIFFERENT_HOST;
 245
 246   if (!secure_scheme_supported() && referring_url.SchemeIsCryptographic())
 247     return SDCH_DICTIONARY_SELECTED_FOR_SSL;
 248
 249   // TODO(jar): Remove this failsafe conservative hack which is more restrictive
 250   // than current SDCH spec when needed, and justified by security audit.
 251   if (!referring_url.SchemeIsHTTPOrHTTPS())
 252     return SDCH_DICTIONARY_SELECTED_FROM_NON_HTTP;
 253
 254   return SDCH_OK;
 255 }
 256
 257 scoped_ptr<SdchManager::DictionarySet>
 258 SdchManager::GetDictionarySet(const GURL& target_url) {
 259   if (IsInSupportedDomain(target_url) != SDCH_OK)
 260     return NULL;
 261
 262   int count = 0;
 263   scoped_ptr<SdchManager::DictionarySet> result(new DictionarySet);
 264   for (const auto& entry: dictionaries_) {
 265     if (!secure_scheme_supported() && target_url.SchemeIsCryptographic())
 266       continue;
 267     if (entry.second->data.CanUse(target_url) != SDCH_OK)
 268       continue;
 269     if (entry.second->data.Expired())
 270       continue;
 271     ++count;
 272     result->AddDictionary(entry.first, entry.second);
 273   }
 274
 275   if (count == 0)
 276     return NULL;
 277
 278   UMA_HISTOGRAM_COUNTS("Sdch3.Advertisement_Count", count);
 279   UMA_HISTOGRAM_BOOLEAN("Sdch3.AdvertisedWithSecureScheme",
 280                         target_url.SchemeIsSecure());
 281
 282   return result.Pass();
 283 }
 284
 285 scoped_ptr<SdchManager::DictionarySet>
 286 SdchManager::GetDictionarySetByHash(
 287     const GURL& target_url,
 288     const std::string& server_hash,
 289     SdchProblemCode* problem_code) {
 290   scoped_ptr<SdchManager::DictionarySet> result;
 291
 292   *problem_code = SDCH_DICTIONARY_HASH_NOT_FOUND;
 293   const auto& it = dictionaries_.find(server_hash);
 294   if (it == dictionaries_.end())
 295     return result.Pass();
 296
 297   if (!SdchManager::secure_scheme_supported() &&
 298       target_url.SchemeIsCryptographic()) {
 299     *problem_code = SDCH_DICTIONARY_FOUND_HAS_WRONG_SCHEME;
 300     return result.Pass();
 301   }
 302
 303   *problem_code = it->second->data.CanUse(target_url);
 304   if (*problem_code != SDCH_OK)
 305     return result.Pass();
 306
 307   result.reset(new DictionarySet);
 308   result->AddDictionary(it->first, it->second);
 309   return result.Pass();
 310 }
 311
 312 // static
 313 void SdchManager::GenerateHash(const std::string& dictionary_text,
 314     std::string* client_hash, std::string* server_hash) {
 315   char binary_hash[32];
 316   crypto::SHA256HashString(dictionary_text, binary_hash, sizeof(binary_hash));
 317
 318   std::string first_48_bits(&binary_hash[0], 6);
 319   std::string second_48_bits(&binary_hash[6], 6);
 320   UrlSafeBase64Encode(first_48_bits, client_hash);
 321   UrlSafeBase64Encode(second_48_bits, server_hash);
 322
 323   DCHECK_EQ(server_hash->length(), 8u);
 324   DCHECK_EQ(client_hash->length(), 8u);
 325 }
 326
 327 // Methods for supporting latency experiments.
 328
 329 bool SdchManager::AllowLatencyExperiment(const GURL& url) const {
 330   DCHECK(thread_checker_.CalledOnValidThread());
 331   return allow_latency_experiment_.end() !=
 332       allow_latency_experiment_.find(url.host());
 333 }
 334
 335 void SdchManager::SetAllowLatencyExperiment(const GURL& url, bool enable) {
 336   DCHECK(thread_checker_.CalledOnValidThread());
 337   if (enable) {
 338     allow_latency_experiment_.insert(url.host());
 339     return;
 340   }
 341   ExperimentSet::iterator it = allow_latency_experiment_.find(url.host());
 342   if (allow_latency_experiment_.end() == it)
 343     return;  // It was already erased, or never allowed.
 344   SdchErrorRecovery(SDCH_LATENCY_TEST_DISALLOWED);
 345   allow_latency_experiment_.erase(it);
 346 }
 347
 348 void SdchManager::AddObserver(SdchObserver* observer) {
 349   observers_.AddObserver(observer);
 350 }
 351
 352 void SdchManager::RemoveObserver(SdchObserver* observer) {
 353   observers_.RemoveObserver(observer);
 354 }
 355
 356 SdchProblemCode SdchManager::AddSdchDictionary(
 357     const std::string& dictionary_text,
 358     const GURL& dictionary_url,
 359     std::string* server_hash_p) {
 360   DCHECK(thread_checker_.CalledOnValidThread());
 361   std::string client_hash;
 362   std::string server_hash;
 363   GenerateHash(dictionary_text, &client_hash, &server_hash);
 364   if (dictionaries_.find(server_hash) != dictionaries_.end())
 365     return SDCH_DICTIONARY_ALREADY_LOADED;  // Already loaded.
 366
 367   std::string domain, path;
 368   std::set<int> ports;
 369   base::Time expiration(base::Time::Now() + base::TimeDelta::FromDays(30));
 370
 371   if (dictionary_text.empty())
 372     return SDCH_DICTIONARY_HAS_NO_TEXT;  // Missing header.
 373
 374   size_t header_end = dictionary_text.find("\n\n");
 375   if (std::string::npos == header_end)
 376     return SDCH_DICTIONARY_HAS_NO_HEADER;  // Missing header.
 377
 378   size_t line_start = 0;  // Start of line being parsed.
 379   while (1) {
 380     size_t line_end = dictionary_text.find('\n', line_start);
 381     DCHECK(std::string::npos != line_end);
 382     DCHECK_LE(line_end, header_end);
 383
 384     size_t colon_index = dictionary_text.find(':', line_start);
 385     if (std::string::npos == colon_index)
 386       return SDCH_DICTIONARY_HEADER_LINE_MISSING_COLON;  // Illegal line missing
 387                                                          // a colon.
 388
 389     if (colon_index > line_end)
 390       break;
 391
 392     size_t value_start = dictionary_text.find_first_not_of(" \t",
 393                                                            colon_index + 1);
 394     if (std::string::npos != value_start) {
 395       if (value_start >= line_end)
 396         break;
 397       std::string name(dictionary_text, line_start, colon_index - line_start);
 398       std::string value(dictionary_text, value_start, line_end - value_start);
 399       name = base::StringToLowerASCII(name);
 400       if (name == "domain") {
 401         domain = value;
 402       } else if (name == "path") {
 403         path = value;
 404       } else if (name == "format-version") {
 405         if (value != "1.0")
 406           return SDCH_DICTIONARY_UNSUPPORTED_VERSION;
 407       } else if (name == "max-age") {
 408         int64_t seconds;
 409         base::StringToInt64(value, &seconds);
 410         expiration = base::Time::Now() + base::TimeDelta::FromSeconds(seconds);
 411       } else if (name == "port") {
 412         int port;
 413         base::StringToInt(value, &port);
 414         if (port >= 0)
 415           ports.insert(port);
 416       }
 417     }
 418
 419     if (line_end >= header_end)
 420       break;
 421     line_start = line_end + 1;
 422   }
 423
 424   // Narrow fix for http://crbug.com/389451.
 425   GURL dictionary_url_normalized(dictionary_url);
 426   StripTrailingDot(&dictionary_url_normalized);
 427
 428   SdchProblemCode rv = IsInSupportedDomain(dictionary_url_normalized);
 429   if (rv != SDCH_OK)
 430     return rv;
 431
 432   rv = SdchDictionary::CanSet(domain, path, ports, dictionary_url_normalized);
 433   if (rv != SDCH_OK)
 434     return rv;
 435
 436   UMA_HISTOGRAM_COUNTS("Sdch3.Dictionary size loaded", dictionary_text.size());
 437   DVLOG(1) << "Loaded dictionary with client hash " << client_hash
 438            << " and server hash " << server_hash;
 439   SdchDictionary dictionary(dictionary_text, header_end + 2, client_hash,
 440                             server_hash, dictionary_url_normalized, domain,
 441                             path, expiration, ports);
 442   dictionaries_[server_hash] =
 443       new base::RefCountedData<SdchDictionary>(dictionary);
 444   if (server_hash_p)
 445     *server_hash_p = server_hash;
 446
 447   FOR_EACH_OBSERVER(SdchObserver, observers_,
 448                     OnDictionaryAdded(dictionary_url, server_hash));
 449
 450   return SDCH_OK;
 451 }
 452
 453 SdchProblemCode SdchManager::RemoveSdchDictionary(
 454     const std::string& server_hash) {
 455   if (dictionaries_.find(server_hash) == dictionaries_.end())
 456     return SDCH_DICTIONARY_HASH_NOT_FOUND;
 457
 458   dictionaries_.erase(server_hash);
 459
 460   FOR_EACH_OBSERVER(SdchObserver, observers_, OnDictionaryRemoved(server_hash));
 461
 462   return SDCH_OK;
 463 }
 464
 465 // static
 466 scoped_ptr<SdchManager::DictionarySet>
 467 SdchManager::CreateEmptyDictionarySetForTesting() {
 468   return scoped_ptr<DictionarySet>(new DictionarySet).Pass();
 469 }
 470
 471 // static
 472 void SdchManager::UrlSafeBase64Encode(const std::string& input,
 473                                       std::string* output) {
 474   // Since this is only done during a dictionary load, and hashes are only 8
 475   // characters, we just do the simple fixup, rather than rewriting the encoder.
 476   base::Base64Encode(input, output);
 477   std::replace(output->begin(), output->end(), '+', '-');
 478   std::replace(output->begin(), output->end(), '/', '_');
 479 }
 480
 481 scoped_ptr<base::Value> SdchManager::SdchInfoToValue() const {
 482   scoped_ptr<base::DictionaryValue> value(new base::DictionaryValue());
 483
 484   value->SetBoolean("sdch_enabled", sdch_enabled());
 485   value->SetBoolean("secure_scheme_support", secure_scheme_supported());
 486
 487   scoped_ptr<base::ListValue> entry_list(new base::ListValue());
 488   for (const auto& entry: dictionaries_) {
 489     scoped_ptr<base::DictionaryValue> entry_dict(new base::DictionaryValue());
 490     entry_dict->SetString("url", entry.second->data.url().spec());
 491     entry_dict->SetString("client_hash", entry.second->data.client_hash());
 492     entry_dict->SetString("domain", entry.second->data.domain());
 493     entry_dict->SetString("path", entry.second->data.path());
 494     scoped_ptr<base::ListValue> port_list(new base::ListValue());
 495     for (std::set<int>::const_iterator port_it =
 496              entry.second->data.ports().begin();
 497          port_it != entry.second->data.ports().end(); ++port_it) {
 498       port_list->AppendInteger(*port_it);
 499     }
 500     entry_dict->Set("ports", port_list.Pass());
 501     entry_dict->SetString("server_hash", entry.first);
 502     entry_list->Append(entry_dict.Pass());
 503   }
 504   value->Set("dictionaries", entry_list.Pass());
 505
 506   entry_list.reset(new base::ListValue());
 507   for (DomainBlacklistInfo::const_iterator it = blacklisted_domains_.begin();
 508        it != blacklisted_domains_.end(); ++it) {
 509     if (it->second.count == 0)
 510       continue;
 511     scoped_ptr<base::DictionaryValue> entry_dict(new base::DictionaryValue());
 512     entry_dict->SetString("domain", it->first);
 513     if (it->second.count != INT_MAX)
 514       entry_dict->SetInteger("tries", it->second.count);
 515     entry_dict->SetInteger("reason", it->second.reason);
 516     entry_list->Append(entry_dict.Pass());
 517   }
 518   value->Set("blacklisted", entry_list.Pass());
 519
 520   return value.Pass();
 521 }
 522
 523 }  // namespace net