net/base/sdch_manager.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/base/sdch_manager.h"
   6
   7 #include "base/base64.h"
   8 #include "base/logging.h"
   9 #include "base/metrics/histogram_macros.h"
  10 #include "base/strings/string_number_conversions.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/time/default_clock.h"
  13 #include "base/values.h"
  14 #include "crypto/sha2.h"
  15 #include "net/base/sdch_observer.h"
  16 #include "net/url_request/url_request_http_job.h"
  17
  18 namespace {
  19
  20 void StripTrailingDot(GURL* gurl) {
  21   std::string host(gurl->host());
  22
  23   if (host.empty())
  24     return;
  25
  26   if (*host.rbegin() != '.')
  27     return;
  28
  29   host.resize(host.size() - 1);
  30
  31   GURL::Replacements replacements;
  32   replacements.SetHostStr(host);
  33   *gurl = gurl->ReplaceComponents(replacements);
  34   return;
  35 }
  36
  37 }  // namespace
  38
  39 namespace net {
  40
  41 SdchManager::DictionarySet::DictionarySet() {}
  42
  43 SdchManager::DictionarySet::~DictionarySet() {}
  44
  45 std::string SdchManager::DictionarySet::GetDictionaryClientHashList() const {
  46   std::string result;
  47   bool first = true;
  48   for (const auto& entry: dictionaries_) {
  49     if (!first)
  50       result.append(",");
  51
  52     result.append(entry.second->data.client_hash());
  53     first = false;
  54   }
  55   return result;
  56 }
  57
  58 bool SdchManager::DictionarySet::Empty() const {
  59   return dictionaries_.empty();
  60 }
  61
  62 const std::string* SdchManager::DictionarySet::GetDictionaryText(
  63     const std::string& server_hash) const {
  64   auto it = dictionaries_.find(server_hash);
  65   if (it == dictionaries_.end())
  66     return nullptr;
  67   return &it->second->data.text();
  68 }
  69
  70 void SdchManager::DictionarySet::AddDictionary(
  71     const std::string& server_hash,
  72     const scoped_refptr<base::RefCountedData<SdchDictionary>>& dictionary) {
  73   DCHECK(dictionaries_.end() == dictionaries_.find(server_hash));
  74
  75   dictionaries_[server_hash] = dictionary;
  76 }
  77
  78 SdchManager::SdchManager() {
  79   DCHECK(thread_checker_.CalledOnValidThread());
  80 }
  81
  82 SdchManager::~SdchManager() {
  83   DCHECK(thread_checker_.CalledOnValidThread());
  84   while (!dictionaries_.empty()) {
  85     auto it = dictionaries_.begin();
  86     dictionaries_.erase(it->first);
  87   }
  88 }
  89
  90 void SdchManager::ClearData() {
  91   blacklisted_domains_.clear();
  92   allow_latency_experiment_.clear();
  93   dictionaries_.clear();
  94   FOR_EACH_OBSERVER(SdchObserver, observers_, OnClearDictionaries());
  95 }
  96
  97 // static
  98 void SdchManager::SdchErrorRecovery(SdchProblemCode problem) {
  99   UMA_HISTOGRAM_ENUMERATION("Sdch3.ProblemCodes_5", problem,
 100                             SDCH_MAX_PROBLEM_CODE);
 101 }
 102
 103 void SdchManager::BlacklistDomain(const GURL& url,
 104                                   SdchProblemCode blacklist_reason) {
 105   SetAllowLatencyExperiment(url, false);
 106
 107   BlacklistInfo* blacklist_info = &blacklisted_domains_[url.host()];
 108
 109   if (blacklist_info->count > 0)
 110     return;  // Domain is already blacklisted.
 111
 112   if (blacklist_info->exponential_count > (INT_MAX - 1) / 2) {
 113     blacklist_info->exponential_count = INT_MAX;
 114   } else {
 115     blacklist_info->exponential_count =
 116         blacklist_info->exponential_count * 2 + 1;
 117   }
 118
 119   blacklist_info->count = blacklist_info->exponential_count;
 120   blacklist_info->reason = blacklist_reason;
 121 }
 122
 123 void SdchManager::BlacklistDomainForever(const GURL& url,
 124                                          SdchProblemCode blacklist_reason) {
 125   SetAllowLatencyExperiment(url, false);
 126
 127   BlacklistInfo* blacklist_info = &blacklisted_domains_[url.host()];
 128   blacklist_info->count = INT_MAX;
 129   blacklist_info->exponential_count = INT_MAX;
 130   blacklist_info->reason = blacklist_reason;
 131 }
 132
 133 void SdchManager::ClearBlacklistings() {
 134   blacklisted_domains_.clear();
 135 }
 136
 137 void SdchManager::ClearDomainBlacklisting(const std::string& domain) {
 138   BlacklistInfo* blacklist_info =
 139       &blacklisted_domains_[base::ToLowerASCII(domain)];
 140   blacklist_info->count = 0;
 141   blacklist_info->reason = SDCH_OK;
 142 }
 143
 144 int SdchManager::BlackListDomainCount(const std::string& domain) {
 145   std::string domain_lower(base::ToLowerASCII(domain));
 146
 147   if (blacklisted_domains_.end() == blacklisted_domains_.find(domain_lower))
 148     return 0;
 149   return blacklisted_domains_[domain_lower].count;
 150 }
 151
 152 int SdchManager::BlacklistDomainExponential(const std::string& domain) {
 153   std::string domain_lower(base::ToLowerASCII(domain));
 154
 155   if (blacklisted_domains_.end() == blacklisted_domains_.find(domain_lower))
 156     return 0;
 157   return blacklisted_domains_[domain_lower].exponential_count;
 158 }
 159
 160 SdchProblemCode SdchManager::IsInSupportedDomain(const GURL& url) {
 161   DCHECK(thread_checker_.CalledOnValidThread());
 162   if (blacklisted_domains_.empty())
 163     return SDCH_OK;
 164
 165   auto it = blacklisted_domains_.find(url.host());
 166   if (blacklisted_domains_.end() == it || it->second.count == 0)
 167     return SDCH_OK;
 168
 169   UMA_HISTOGRAM_ENUMERATION("Sdch3.BlacklistReason", it->second.reason,
 170                             SDCH_MAX_PROBLEM_CODE);
 171
 172   int count = it->second.count - 1;
 173   if (count > 0) {
 174     it->second.count = count;
 175   } else {
 176     it->second.count = 0;
 177     it->second.reason = SDCH_OK;
 178   }
 179
 180   return SDCH_DOMAIN_BLACKLIST_INCLUDES_TARGET;
 181 }
 182
 183 SdchProblemCode SdchManager::OnGetDictionary(const GURL& request_url,
 184                                              const GURL& dictionary_url) {
 185   DCHECK(thread_checker_.CalledOnValidThread());
 186   SdchProblemCode rv = CanFetchDictionary(request_url, dictionary_url);
 187   if (rv != SDCH_OK)
 188     return rv;
 189
 190   FOR_EACH_OBSERVER(SdchObserver,
 191                     observers_,
 192                     OnGetDictionary(request_url, dictionary_url));
 193
 194   return SDCH_OK;
 195 }
 196
 197 void SdchManager::OnDictionaryUsed(const std::string& server_hash) {
 198   FOR_EACH_OBSERVER(SdchObserver, observers_,
 199                     OnDictionaryUsed(server_hash));
 200 }
 201
 202 SdchProblemCode SdchManager::CanFetchDictionary(
 203     const GURL& referring_url,
 204     const GURL& dictionary_url) const {
 205   DCHECK(thread_checker_.CalledOnValidThread());
 206   /* The user agent may retrieve a dictionary from the dictionary URL if all of
 207      the following are true:
 208        1 The dictionary URL host name matches the referrer URL host name and
 209            scheme.
 210        2 The dictionary URL host name domain matches the parent domain of the
 211            referrer URL host name
 212        3 The parent domain of the referrer URL host name is not a top level
 213            domain
 214    */
 215   // Item (1) above implies item (2). Spec should be updated.
 216   // I take "host name match" to be "is identical to"
 217   if (referring_url.host() != dictionary_url.host() ||
 218       referring_url.scheme() != dictionary_url.scheme())
 219     return SDCH_DICTIONARY_LOAD_ATTEMPT_FROM_DIFFERENT_HOST;
 220
 221   // TODO(jar): Remove this failsafe conservative hack which is more restrictive
 222   // than current SDCH spec when needed, and justified by security audit.
 223   if (!referring_url.SchemeIsHTTPOrHTTPS())
 224     return SDCH_DICTIONARY_SELECTED_FROM_NON_HTTP;
 225
 226   return SDCH_OK;
 227 }
 228
 229 scoped_ptr<SdchManager::DictionarySet>
 230 SdchManager::GetDictionarySet(const GURL& target_url) {
 231   if (IsInSupportedDomain(target_url) != SDCH_OK)
 232     return NULL;
 233
 234   int count = 0;
 235   scoped_ptr<SdchManager::DictionarySet> result(new DictionarySet);
 236   for (const auto& entry: dictionaries_) {
 237     if (entry.second->data.CanUse(target_url) != SDCH_OK)
 238       continue;
 239     if (entry.second->data.Expired())
 240       continue;
 241     ++count;
 242     result->AddDictionary(entry.first, entry.second);
 243   }
 244
 245   if (count == 0)
 246     return NULL;
 247
 248   UMA_HISTOGRAM_COUNTS("Sdch3.Advertisement_Count", count);
 249   UMA_HISTOGRAM_BOOLEAN("Sdch3.AdvertisedWithSecureScheme",
 250                         target_url.SchemeIsCryptographic());
 251
 252   return result.Pass();
 253 }
 254
 255 scoped_ptr<SdchManager::DictionarySet>
 256 SdchManager::GetDictionarySetByHash(
 257     const GURL& target_url,
 258     const std::string& server_hash,
 259     SdchProblemCode* problem_code) {
 260   scoped_ptr<SdchManager::DictionarySet> result;
 261
 262   *problem_code = SDCH_DICTIONARY_HASH_NOT_FOUND;
 263   const auto& it = dictionaries_.find(server_hash);
 264   if (it == dictionaries_.end())
 265     return result.Pass();
 266
 267   *problem_code = it->second->data.CanUse(target_url);
 268   if (*problem_code != SDCH_OK)
 269     return result.Pass();
 270
 271   result.reset(new DictionarySet);
 272   result->AddDictionary(it->first, it->second);
 273   return result.Pass();
 274 }
 275
 276 // static
 277 void SdchManager::GenerateHash(const std::string& dictionary_text,
 278     std::string* client_hash, std::string* server_hash) {
 279   char binary_hash[32];
 280   crypto::SHA256HashString(dictionary_text, binary_hash, sizeof(binary_hash));
 281
 282   std::string first_48_bits(&binary_hash[0], 6);
 283   std::string second_48_bits(&binary_hash[6], 6);
 284   UrlSafeBase64Encode(first_48_bits, client_hash);
 285   UrlSafeBase64Encode(second_48_bits, server_hash);
 286
 287   DCHECK_EQ(server_hash->length(), 8u);
 288   DCHECK_EQ(client_hash->length(), 8u);
 289 }
 290
 291 // Methods for supporting latency experiments.
 292
 293 bool SdchManager::AllowLatencyExperiment(const GURL& url) const {
 294   DCHECK(thread_checker_.CalledOnValidThread());
 295   return allow_latency_experiment_.end() !=
 296       allow_latency_experiment_.find(url.host());
 297 }
 298
 299 void SdchManager::SetAllowLatencyExperiment(const GURL& url, bool enable) {
 300   DCHECK(thread_checker_.CalledOnValidThread());
 301   if (enable) {
 302     allow_latency_experiment_.insert(url.host());
 303     return;
 304   }
 305   ExperimentSet::iterator it = allow_latency_experiment_.find(url.host());
 306   if (allow_latency_experiment_.end() == it)
 307     return;  // It was already erased, or never allowed.
 308   SdchErrorRecovery(SDCH_LATENCY_TEST_DISALLOWED);
 309   allow_latency_experiment_.erase(it);
 310 }
 311
 312 void SdchManager::AddObserver(SdchObserver* observer) {
 313   observers_.AddObserver(observer);
 314 }
 315
 316 void SdchManager::RemoveObserver(SdchObserver* observer) {
 317   observers_.RemoveObserver(observer);
 318 }
 319
 320 SdchProblemCode SdchManager::AddSdchDictionary(
 321     const std::string& dictionary_text,
 322     const GURL& dictionary_url,
 323     std::string* server_hash_p) {
 324   DCHECK(thread_checker_.CalledOnValidThread());
 325   std::string client_hash;
 326   std::string server_hash;
 327   GenerateHash(dictionary_text, &client_hash, &server_hash);
 328   if (dictionaries_.find(server_hash) != dictionaries_.end())
 329     return SDCH_DICTIONARY_ALREADY_LOADED;  // Already loaded.
 330
 331   std::string domain, path;
 332   std::set<int> ports;
 333   base::Time expiration(base::Time::Now() + base::TimeDelta::FromDays(30));
 334
 335   if (dictionary_text.empty())
 336     return SDCH_DICTIONARY_HAS_NO_TEXT;  // Missing header.
 337
 338   size_t header_end = dictionary_text.find("\n\n");
 339   if (std::string::npos == header_end)
 340     return SDCH_DICTIONARY_HAS_NO_HEADER;  // Missing header.
 341
 342   size_t line_start = 0;  // Start of line being parsed.
 343   while (1) {
 344     size_t line_end = dictionary_text.find('\n', line_start);
 345     DCHECK(std::string::npos != line_end);
 346     DCHECK_LE(line_end, header_end);
 347
 348     size_t colon_index = dictionary_text.find(':', line_start);
 349     if (std::string::npos == colon_index)
 350       return SDCH_DICTIONARY_HEADER_LINE_MISSING_COLON;  // Illegal line missing
 351                                                          // a colon.
 352
 353     if (colon_index > line_end)
 354       break;
 355
 356     size_t value_start = dictionary_text.find_first_not_of(" \t",
 357                                                            colon_index + 1);
 358     if (std::string::npos != value_start) {
 359       if (value_start >= line_end)
 360         break;
 361       std::string name(dictionary_text, line_start, colon_index - line_start);
 362       std::string value(dictionary_text, value_start, line_end - value_start);
 363       name = base::ToLowerASCII(name);
 364       if (name == "domain") {
 365         domain = value;
 366       } else if (name == "path") {
 367         path = value;
 368       } else if (name == "format-version") {
 369         if (value != "1.0")
 370           return SDCH_DICTIONARY_UNSUPPORTED_VERSION;
 371       } else if (name == "max-age") {
 372         int64_t seconds;
 373         base::StringToInt64(value, &seconds);
 374         expiration = base::Time::Now() + base::TimeDelta::FromSeconds(seconds);
 375       } else if (name == "port") {
 376         int port;
 377         base::StringToInt(value, &port);
 378         if (port >= 0)
 379           ports.insert(port);
 380       }
 381     }
 382
 383     if (line_end >= header_end)
 384       break;
 385     line_start = line_end + 1;
 386   }
 387
 388   // Narrow fix for http://crbug.com/389451.
 389   GURL dictionary_url_normalized(dictionary_url);
 390   StripTrailingDot(&dictionary_url_normalized);
 391
 392   SdchProblemCode rv = IsInSupportedDomain(dictionary_url_normalized);
 393   if (rv != SDCH_OK)
 394     return rv;
 395
 396   rv = SdchDictionary::CanSet(domain, path, ports, dictionary_url_normalized);
 397   if (rv != SDCH_OK)
 398     return rv;
 399
 400   UMA_HISTOGRAM_COUNTS("Sdch3.Dictionary size loaded", dictionary_text.size());
 401   DVLOG(1) << "Loaded dictionary with client hash " << client_hash
 402            << " and server hash " << server_hash;
 403   SdchDictionary dictionary(dictionary_text, header_end + 2, client_hash,
 404                             server_hash, dictionary_url_normalized, domain,
 405                             path, expiration, ports);
 406   dictionaries_[server_hash] =
 407       new base::RefCountedData<SdchDictionary>(dictionary);
 408   if (server_hash_p)
 409     *server_hash_p = server_hash;
 410
 411   FOR_EACH_OBSERVER(SdchObserver, observers_,
 412                     OnDictionaryAdded(dictionary_url, server_hash));
 413
 414   return SDCH_OK;
 415 }
 416
 417 SdchProblemCode SdchManager::RemoveSdchDictionary(
 418     const std::string& server_hash) {
 419   if (dictionaries_.find(server_hash) == dictionaries_.end())
 420     return SDCH_DICTIONARY_HASH_NOT_FOUND;
 421
 422   dictionaries_.erase(server_hash);
 423
 424   FOR_EACH_OBSERVER(SdchObserver, observers_, OnDictionaryRemoved(server_hash));
 425
 426   return SDCH_OK;
 427 }
 428
 429 // static
 430 scoped_ptr<SdchManager::DictionarySet>
 431 SdchManager::CreateEmptyDictionarySetForTesting() {
 432   return scoped_ptr<DictionarySet>(new DictionarySet).Pass();
 433 }
 434
 435 // static
 436 void SdchManager::UrlSafeBase64Encode(const std::string& input,
 437                                       std::string* output) {
 438   // Since this is only done during a dictionary load, and hashes are only 8
 439   // characters, we just do the simple fixup, rather than rewriting the encoder.
 440   base::Base64Encode(input, output);
 441   std::replace(output->begin(), output->end(), '+', '-');
 442   std::replace(output->begin(), output->end(), '/', '_');
 443 }
 444
 445 scoped_ptr<base::Value> SdchManager::SdchInfoToValue() const {
 446   scoped_ptr<base::DictionaryValue> value(new base::DictionaryValue());
 447
 448   value->SetBoolean("sdch_enabled", true);
 449
 450   scoped_ptr<base::ListValue> entry_list(new base::ListValue());
 451   for (const auto& entry: dictionaries_) {
 452     scoped_ptr<base::DictionaryValue> entry_dict(new base::DictionaryValue());
 453     entry_dict->SetString("url", entry.second->data.url().spec());
 454     entry_dict->SetString("client_hash", entry.second->data.client_hash());
 455     entry_dict->SetString("domain", entry.second->data.domain());
 456     entry_dict->SetString("path", entry.second->data.path());
 457     scoped_ptr<base::ListValue> port_list(new base::ListValue());
 458     for (std::set<int>::const_iterator port_it =
 459              entry.second->data.ports().begin();
 460          port_it != entry.second->data.ports().end(); ++port_it) {
 461       port_list->AppendInteger(*port_it);
 462     }
 463     entry_dict->Set("ports", port_list.Pass());
 464     entry_dict->SetString("server_hash", entry.first);
 465     entry_list->Append(entry_dict.Pass());
 466   }
 467   value->Set("dictionaries", entry_list.Pass());
 468
 469   entry_list.reset(new base::ListValue());
 470   for (DomainBlacklistInfo::const_iterator it = blacklisted_domains_.begin();
 471        it != blacklisted_domains_.end(); ++it) {
 472     if (it->second.count == 0)
 473       continue;
 474     scoped_ptr<base::DictionaryValue> entry_dict(new base::DictionaryValue());
 475     entry_dict->SetString("domain", it->first);
 476     if (it->second.count != INT_MAX)
 477       entry_dict->SetInteger("tries", it->second.count);
 478     entry_dict->SetInteger("reason", it->second.reason);
 479     entry_list->Append(entry_dict.Pass());
 480   }
 481   value->Set("blacklisted", entry_list.Pass());
 482
 483   return value.Pass();
 484 }
 485
 486 }  // namespace net