net/base/sdch_manager.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/base/sdch_manager.h"
   6
   7 #include "base/base64.h"
   8 #include "base/logging.h"
   9 #include "base/metrics/histogram_macros.h"
  10 #include "base/strings/string_number_conversions.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/time/default_clock.h"
  13 #include "base/values.h"
  14 #include "crypto/sha2.h"
  15 #include "net/base/sdch_observer.h"
  16 #include "net/url_request/url_request_http_job.h"
  17
  18 namespace {
  19
  20 void StripTrailingDot(GURL* gurl) {
  21   std::string host(gurl->host());
  22
  23   if (host.empty())
  24     return;
  25
  26   if (*host.rbegin() != '.')
  27     return;
  28
  29   host.resize(host.size() - 1);
  30
  31   GURL::Replacements replacements;
  32   replacements.SetHostStr(host);
  33   *gurl = gurl->ReplaceComponents(replacements);
  34   return;
  35 }
  36
  37 }  // namespace
  38
  39 namespace net {
  40
  41 SdchManager::DictionarySet::DictionarySet() {}
  42
  43 SdchManager::DictionarySet::~DictionarySet() {}
  44
  45 std::string SdchManager::DictionarySet::GetDictionaryClientHashList() const {
  46   std::string result;
  47   bool first = true;
  48   for (const auto& entry: dictionaries_) {
  49     if (!first)
  50       result.append(",");
  51
  52     result.append(entry.second->data.client_hash());
  53     first = false;
  54   }
  55   return result;
  56 }
  57
  58 bool SdchManager::DictionarySet::Empty() const {
  59   return dictionaries_.empty();
  60 }
  61
  62 const std::string* SdchManager::DictionarySet::GetDictionaryText(
  63     const std::string& server_hash) const {
  64   auto it = dictionaries_.find(server_hash);
  65   if (it == dictionaries_.end())
  66     return nullptr;
  67   return &it->second->data.text();
  68 }
  69
  70 void SdchManager::DictionarySet::AddDictionary(
  71     const std::string& server_hash,
  72     const scoped_refptr<base::RefCountedData<SdchDictionary>>& dictionary) {
  73   DCHECK(dictionaries_.end() == dictionaries_.find(server_hash));
  74
  75   dictionaries_[server_hash] = dictionary;
  76 }
  77
  78 SdchManager::SdchManager() {
  79   DCHECK(thread_checker_.CalledOnValidThread());
  80 }
  81
  82 SdchManager::~SdchManager() {
  83   DCHECK(thread_checker_.CalledOnValidThread());
  84   while (!dictionaries_.empty()) {
  85     auto it = dictionaries_.begin();
  86     dictionaries_.erase(it->first);
  87   }
  88 }
  89
  90 void SdchManager::ClearData() {
  91   blacklisted_domains_.clear();
  92   allow_latency_experiment_.clear();
  93   dictionaries_.clear();
  94   FOR_EACH_OBSERVER(SdchObserver, observers_, OnClearDictionaries());
  95 }
  96
  97 // static
  98 void SdchManager::SdchErrorRecovery(SdchProblemCode problem) {
  99   UMA_HISTOGRAM_ENUMERATION("Sdch3.ProblemCodes_5", problem,
 100                             SDCH_MAX_PROBLEM_CODE);
 101 }
 102
 103 void SdchManager::BlacklistDomain(const GURL& url,
 104                                   SdchProblemCode blacklist_reason) {
 105   SetAllowLatencyExperiment(url, false);
 106
 107   BlacklistInfo* blacklist_info = &blacklisted_domains_[url.host()];
 108
 109   if (blacklist_info->count > 0)
 110     return;  // Domain is already blacklisted.
 111
 112   if (blacklist_info->exponential_count > (INT_MAX - 1) / 2) {
 113     blacklist_info->exponential_count = INT_MAX;
 114   } else {
 115     blacklist_info->exponential_count =
 116         blacklist_info->exponential_count * 2 + 1;
 117   }
 118
 119   blacklist_info->count = blacklist_info->exponential_count;
 120   blacklist_info->reason = blacklist_reason;
 121 }
 122
 123 void SdchManager::BlacklistDomainForever(const GURL& url,
 124                                          SdchProblemCode blacklist_reason) {
 125   SetAllowLatencyExperiment(url, false);
 126
 127   BlacklistInfo* blacklist_info = &blacklisted_domains_[url.host()];
 128   blacklist_info->count = INT_MAX;
 129   blacklist_info->exponential_count = INT_MAX;
 130   blacklist_info->reason = blacklist_reason;
 131 }
 132
 133 void SdchManager::ClearBlacklistings() {
 134   blacklisted_domains_.clear();
 135 }
 136
 137 void SdchManager::ClearDomainBlacklisting(const std::string& domain) {
 138   BlacklistInfo* blacklist_info =
 139       &blacklisted_domains_[base::ToLowerASCII(domain)];
 140   blacklist_info->count = 0;
 141   blacklist_info->reason = SDCH_OK;
 142 }
 143
 144 int SdchManager::BlackListDomainCount(const std::string& domain) {
 145   std::string domain_lower(base::ToLowerASCII(domain));
 146
 147   if (blacklisted_domains_.end() == blacklisted_domains_.find(domain_lower))
 148     return 0;
 149   return blacklisted_domains_[domain_lower].count;
 150 }
 151
 152 int SdchManager::BlacklistDomainExponential(const std::string& domain) {
 153   std::string domain_lower(base::ToLowerASCII(domain));
 154
 155   if (blacklisted_domains_.end() == blacklisted_domains_.find(domain_lower))
 156     return 0;
 157   return blacklisted_domains_[domain_lower].exponential_count;
 158 }
 159
 160 SdchProblemCode SdchManager::IsInSupportedDomain(const GURL& url) {
 161   DCHECK(thread_checker_.CalledOnValidThread());
 162   if (blacklisted_domains_.empty())
 163     return SDCH_OK;
 164
 165   auto it = blacklisted_domains_.find(url.host());
 166   if (blacklisted_domains_.end() == it || it->second.count == 0)
 167     return SDCH_OK;
 168
 169   UMA_HISTOGRAM_ENUMERATION("Sdch3.BlacklistReason", it->second.reason,
 170                             SDCH_MAX_PROBLEM_CODE);
 171
 172   int count = it->second.count - 1;
 173   if (count > 0) {
 174     it->second.count = count;
 175   } else {
 176     it->second.count = 0;
 177     it->second.reason = SDCH_OK;
 178   }
 179
 180   return SDCH_DOMAIN_BLACKLIST_INCLUDES_TARGET;
 181 }
 182
 183 SdchProblemCode SdchManager::OnGetDictionary(const GURL& request_url,
 184                                              const GURL& dictionary_url) {
 185   DCHECK(thread_checker_.CalledOnValidThread());
 186   SdchProblemCode rv = CanFetchDictionary(request_url, dictionary_url);
 187   if (rv != SDCH_OK)
 188     return rv;
 189
 190   FOR_EACH_OBSERVER(SdchObserver,
 191                     observers_,
 192                     OnGetDictionary(request_url, dictionary_url));
 193
 194   return SDCH_OK;
 195 }
 196
 197 void SdchManager::OnDictionaryUsed(const std::string& server_hash) {
 198   FOR_EACH_OBSERVER(SdchObserver, observers_,
 199                     OnDictionaryUsed(server_hash));
 200 }
 201
 202 SdchProblemCode SdchManager::CanFetchDictionary(
 203     const GURL& referring_url,
 204     const GURL& dictionary_url) const {
 205   DCHECK(thread_checker_.CalledOnValidThread());
 206   /* The user agent may retrieve a dictionary from the dictionary URL if all of
 207      the following are true:
 208        1 The dictionary URL host name matches the referrer URL host name and
 209            scheme.
 210        2 The dictionary URL host name domain matches the parent domain of the
 211            referrer URL host name
 212        3 The parent domain of the referrer URL host name is not a top level
 213            domain
 214    */
 215   // Item (1) above implies item (2). Spec should be updated.
 216   // I take "host name match" to be "is identical to"
 217   if (referring_url.host() != dictionary_url.host() ||
 218       referring_url.scheme() != dictionary_url.scheme())
 219     return SDCH_DICTIONARY_LOAD_ATTEMPT_FROM_DIFFERENT_HOST;
 220
 221   // TODO(jar): Remove this failsafe conservative hack which is more restrictive
 222   // than current SDCH spec when needed, and justified by security audit.
 223   if (!referring_url.SchemeIsHTTPOrHTTPS())
 224     return SDCH_DICTIONARY_SELECTED_FROM_NON_HTTP;
 225
 226   return SDCH_OK;
 227 }
 228
 229 scoped_ptr<SdchManager::DictionarySet>
 230 SdchManager::GetDictionarySet(const GURL& target_url) {
 231   if (IsInSupportedDomain(target_url) != SDCH_OK)
 232     return NULL;
 233
 234   int count = 0;
 235   scoped_ptr<SdchManager::DictionarySet> result(new DictionarySet);
 236   for (const auto& entry: dictionaries_) {
 237     if (entry.second->data.CanUse(target_url) != SDCH_OK)
 238       continue;
 239     if (entry.second->data.Expired())
 240       continue;
 241     ++count;
 242     result->AddDictionary(entry.first, entry.second);
 243   }
 244
 245   if (count == 0)
 246     return NULL;
 247
 248   UMA_HISTOGRAM_COUNTS("Sdch3.Advertisement_Count", count);
 249
 250   return result.Pass();
 251 }
 252
 253 scoped_ptr<SdchManager::DictionarySet>
 254 SdchManager::GetDictionarySetByHash(
 255     const GURL& target_url,
 256     const std::string& server_hash,
 257     SdchProblemCode* problem_code) {
 258   scoped_ptr<SdchManager::DictionarySet> result;
 259
 260   *problem_code = SDCH_DICTIONARY_HASH_NOT_FOUND;
 261   const auto& it = dictionaries_.find(server_hash);
 262   if (it == dictionaries_.end())
 263     return result.Pass();
 264
 265   *problem_code = it->second->data.CanUse(target_url);
 266   if (*problem_code != SDCH_OK)
 267     return result.Pass();
 268
 269   result.reset(new DictionarySet);
 270   result->AddDictionary(it->first, it->second);
 271   return result.Pass();
 272 }
 273
 274 // static
 275 void SdchManager::GenerateHash(const std::string& dictionary_text,
 276     std::string* client_hash, std::string* server_hash) {
 277   char binary_hash[32];
 278   crypto::SHA256HashString(dictionary_text, binary_hash, sizeof(binary_hash));
 279
 280   std::string first_48_bits(&binary_hash[0], 6);
 281   std::string second_48_bits(&binary_hash[6], 6);
 282   UrlSafeBase64Encode(first_48_bits, client_hash);
 283   UrlSafeBase64Encode(second_48_bits, server_hash);
 284
 285   DCHECK_EQ(server_hash->length(), 8u);
 286   DCHECK_EQ(client_hash->length(), 8u);
 287 }
 288
 289 // Methods for supporting latency experiments.
 290
 291 bool SdchManager::AllowLatencyExperiment(const GURL& url) const {
 292   DCHECK(thread_checker_.CalledOnValidThread());
 293   return allow_latency_experiment_.end() !=
 294       allow_latency_experiment_.find(url.host());
 295 }
 296
 297 void SdchManager::SetAllowLatencyExperiment(const GURL& url, bool enable) {
 298   DCHECK(thread_checker_.CalledOnValidThread());
 299   if (enable) {
 300     allow_latency_experiment_.insert(url.host());
 301     return;
 302   }
 303   ExperimentSet::iterator it = allow_latency_experiment_.find(url.host());
 304   if (allow_latency_experiment_.end() == it)
 305     return;  // It was already erased, or never allowed.
 306   SdchErrorRecovery(SDCH_LATENCY_TEST_DISALLOWED);
 307   allow_latency_experiment_.erase(it);
 308 }
 309
 310 void SdchManager::AddObserver(SdchObserver* observer) {
 311   observers_.AddObserver(observer);
 312 }
 313
 314 void SdchManager::RemoveObserver(SdchObserver* observer) {
 315   observers_.RemoveObserver(observer);
 316 }
 317
 318 SdchProblemCode SdchManager::AddSdchDictionary(
 319     const std::string& dictionary_text,
 320     const GURL& dictionary_url,
 321     std::string* server_hash_p) {
 322   DCHECK(thread_checker_.CalledOnValidThread());
 323   std::string client_hash;
 324   std::string server_hash;
 325   GenerateHash(dictionary_text, &client_hash, &server_hash);
 326   if (dictionaries_.find(server_hash) != dictionaries_.end())
 327     return SDCH_DICTIONARY_ALREADY_LOADED;  // Already loaded.
 328
 329   std::string domain, path;
 330   std::set<int> ports;
 331   base::Time expiration(base::Time::Now() + base::TimeDelta::FromDays(30));
 332
 333   if (dictionary_text.empty())
 334     return SDCH_DICTIONARY_HAS_NO_TEXT;  // Missing header.
 335
 336   size_t header_end = dictionary_text.find("\n\n");
 337   if (std::string::npos == header_end)
 338     return SDCH_DICTIONARY_HAS_NO_HEADER;  // Missing header.
 339
 340   size_t line_start = 0;  // Start of line being parsed.
 341   while (1) {
 342     size_t line_end = dictionary_text.find('\n', line_start);
 343     DCHECK(std::string::npos != line_end);
 344     DCHECK_LE(line_end, header_end);
 345
 346     size_t colon_index = dictionary_text.find(':', line_start);
 347     if (std::string::npos == colon_index)
 348       return SDCH_DICTIONARY_HEADER_LINE_MISSING_COLON;  // Illegal line missing
 349                                                          // a colon.
 350
 351     if (colon_index > line_end)
 352       break;
 353
 354     size_t value_start = dictionary_text.find_first_not_of(" \t",
 355                                                            colon_index + 1);
 356     if (std::string::npos != value_start) {
 357       if (value_start >= line_end)
 358         break;
 359       std::string name(dictionary_text, line_start, colon_index - line_start);
 360       std::string value(dictionary_text, value_start, line_end - value_start);
 361       name = base::ToLowerASCII(name);
 362       if (name == "domain") {
 363         domain = value;
 364       } else if (name == "path") {
 365         path = value;
 366       } else if (name == "format-version") {
 367         if (value != "1.0")
 368           return SDCH_DICTIONARY_UNSUPPORTED_VERSION;
 369       } else if (name == "max-age") {
 370         int64_t seconds;
 371         base::StringToInt64(value, &seconds);
 372         expiration = base::Time::Now() + base::TimeDelta::FromSeconds(seconds);
 373       } else if (name == "port") {
 374         int port;
 375         base::StringToInt(value, &port);
 376         if (port >= 0)
 377           ports.insert(port);
 378       }
 379     }
 380
 381     if (line_end >= header_end)
 382       break;
 383     line_start = line_end + 1;
 384   }
 385
 386   // Narrow fix for http://crbug.com/389451.
 387   GURL dictionary_url_normalized(dictionary_url);
 388   StripTrailingDot(&dictionary_url_normalized);
 389
 390   SdchProblemCode rv = IsInSupportedDomain(dictionary_url_normalized);
 391   if (rv != SDCH_OK)
 392     return rv;
 393
 394   rv = SdchDictionary::CanSet(domain, path, ports, dictionary_url_normalized);
 395   if (rv != SDCH_OK)
 396     return rv;
 397
 398   UMA_HISTOGRAM_COUNTS("Sdch3.Dictionary size loaded", dictionary_text.size());
 399   DVLOG(1) << "Loaded dictionary with client hash " << client_hash
 400            << " and server hash " << server_hash;
 401   SdchDictionary dictionary(dictionary_text, header_end + 2, client_hash,
 402                             server_hash, dictionary_url_normalized, domain,
 403                             path, expiration, ports);
 404   dictionaries_[server_hash] =
 405       new base::RefCountedData<SdchDictionary>(dictionary);
 406   if (server_hash_p)
 407     *server_hash_p = server_hash;
 408
 409   FOR_EACH_OBSERVER(SdchObserver, observers_,
 410                     OnDictionaryAdded(dictionary_url, server_hash));
 411
 412   return SDCH_OK;
 413 }
 414
 415 SdchProblemCode SdchManager::RemoveSdchDictionary(
 416     const std::string& server_hash) {
 417   if (dictionaries_.find(server_hash) == dictionaries_.end())
 418     return SDCH_DICTIONARY_HASH_NOT_FOUND;
 419
 420   dictionaries_.erase(server_hash);
 421
 422   FOR_EACH_OBSERVER(SdchObserver, observers_, OnDictionaryRemoved(server_hash));
 423
 424   return SDCH_OK;
 425 }
 426
 427 // static
 428 scoped_ptr<SdchManager::DictionarySet>
 429 SdchManager::CreateEmptyDictionarySetForTesting() {
 430   return scoped_ptr<DictionarySet>(new DictionarySet).Pass();
 431 }
 432
 433 // static
 434 void SdchManager::UrlSafeBase64Encode(const std::string& input,
 435                                       std::string* output) {
 436   // Since this is only done during a dictionary load, and hashes are only 8
 437   // characters, we just do the simple fixup, rather than rewriting the encoder.
 438   base::Base64Encode(input, output);
 439   std::replace(output->begin(), output->end(), '+', '-');
 440   std::replace(output->begin(), output->end(), '/', '_');
 441 }
 442
 443 scoped_ptr<base::Value> SdchManager::SdchInfoToValue() const {
 444   scoped_ptr<base::DictionaryValue> value(new base::DictionaryValue());
 445
 446   value->SetBoolean("sdch_enabled", true);
 447
 448   scoped_ptr<base::ListValue> entry_list(new base::ListValue());
 449   for (const auto& entry: dictionaries_) {
 450     scoped_ptr<base::DictionaryValue> entry_dict(new base::DictionaryValue());
 451     entry_dict->SetString("url", entry.second->data.url().spec());
 452     entry_dict->SetString("client_hash", entry.second->data.client_hash());
 453     entry_dict->SetString("domain", entry.second->data.domain());
 454     entry_dict->SetString("path", entry.second->data.path());
 455     scoped_ptr<base::ListValue> port_list(new base::ListValue());
 456     for (std::set<int>::const_iterator port_it =
 457              entry.second->data.ports().begin();
 458          port_it != entry.second->data.ports().end(); ++port_it) {
 459       port_list->AppendInteger(*port_it);
 460     }
 461     entry_dict->Set("ports", port_list.Pass());
 462     entry_dict->SetString("server_hash", entry.first);
 463     entry_list->Append(entry_dict.Pass());
 464   }
 465   value->Set("dictionaries", entry_list.Pass());
 466
 467   entry_list.reset(new base::ListValue());
 468   for (DomainBlacklistInfo::const_iterator it = blacklisted_domains_.begin();
 469        it != blacklisted_domains_.end(); ++it) {
 470     if (it->second.count == 0)
 471       continue;
 472     scoped_ptr<base::DictionaryValue> entry_dict(new base::DictionaryValue());
 473     entry_dict->SetString("domain", it->first);
 474     if (it->second.count != INT_MAX)
 475       entry_dict->SetInteger("tries", it->second.count);
 476     entry_dict->SetInteger("reason", it->second.reason);
 477     entry_list->Append(entry_dict.Pass());
 478   }
 479   value->Set("blacklisted", entry_list.Pass());
 480
 481   return value.Pass();
 482 }
 483
 484 }  // namespace net