net/base/sdch_manager.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/base/sdch_manager.h"
   6
   7 #include "base/base64.h"
   8 #include "base/logging.h"
   9 #include "base/metrics/histogram_macros.h"
  10 #include "base/strings/string_number_conversions.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/time/default_clock.h"
  13 #include "base/values.h"
  14 #include "crypto/sha2.h"
  15 #include "net/base/sdch_observer.h"
  16 #include "net/url_request/url_request_http_job.h"
  17
  18 namespace {
  19
  20 void StripTrailingDot(GURL* gurl) {
  21   std::string host(gurl->host());
  22
  23   if (host.empty())
  24     return;
  25
  26   if (*host.rbegin() != '.')
  27     return;
  28
  29   host.resize(host.size() - 1);
  30
  31   GURL::Replacements replacements;
  32   replacements.SetHostStr(host);
  33   *gurl = gurl->ReplaceComponents(replacements);
  34   return;
  35 }
  36
  37 }  // namespace
  38
  39 namespace net {
  40
  41 // static
  42 bool SdchManager::g_sdch_enabled_ = true;
  43
  44 SdchManager::DictionarySet::DictionarySet() {}
  45
  46 SdchManager::DictionarySet::~DictionarySet() {}
  47
  48 std::string SdchManager::DictionarySet::GetDictionaryClientHashList() const {
  49   std::string result;
  50   bool first = true;
  51   for (const auto& entry: dictionaries_) {
  52     if (!first)
  53       result.append(",");
  54
  55     result.append(entry.second->data.client_hash());
  56     first = false;
  57   }
  58   return result;
  59 }
  60
  61 bool SdchManager::DictionarySet::Empty() const {
  62   return dictionaries_.empty();
  63 }
  64
  65 const std::string* SdchManager::DictionarySet::GetDictionaryText(
  66     const std::string& server_hash) const {
  67   auto it = dictionaries_.find(server_hash);
  68   if (it == dictionaries_.end())
  69     return nullptr;
  70   return &it->second->data.text();
  71 }
  72
  73 void SdchManager::DictionarySet::AddDictionary(
  74     const std::string& server_hash,
  75     const scoped_refptr<base::RefCountedData<SdchDictionary>>& dictionary) {
  76   DCHECK(dictionaries_.end() == dictionaries_.find(server_hash));
  77
  78   dictionaries_[server_hash] = dictionary;
  79 }
  80
  81 SdchManager::SdchManager() {
  82   DCHECK(thread_checker_.CalledOnValidThread());
  83 }
  84
  85 SdchManager::~SdchManager() {
  86   DCHECK(thread_checker_.CalledOnValidThread());
  87   while (!dictionaries_.empty()) {
  88     auto it = dictionaries_.begin();
  89     dictionaries_.erase(it->first);
  90   }
  91 }
  92
  93 void SdchManager::ClearData() {
  94   blacklisted_domains_.clear();
  95   allow_latency_experiment_.clear();
  96   dictionaries_.clear();
  97   FOR_EACH_OBSERVER(SdchObserver, observers_, OnClearDictionaries());
  98 }
  99
 100 // static
 101 void SdchManager::SdchErrorRecovery(SdchProblemCode problem) {
 102   UMA_HISTOGRAM_ENUMERATION("Sdch3.ProblemCodes_5", problem,
 103                             SDCH_MAX_PROBLEM_CODE);
 104 }
 105
 106 // static
 107 void SdchManager::EnableSdchSupport(bool enabled) {
 108   g_sdch_enabled_ = enabled;
 109 }
 110
 111 void SdchManager::BlacklistDomain(const GURL& url,
 112                                   SdchProblemCode blacklist_reason) {
 113   SetAllowLatencyExperiment(url, false);
 114
 115   BlacklistInfo* blacklist_info =
 116       &blacklisted_domains_[base::StringToLowerASCII(url.host())];
 117
 118   if (blacklist_info->count > 0)
 119     return;  // Domain is already blacklisted.
 120
 121   if (blacklist_info->exponential_count > (INT_MAX - 1) / 2) {
 122     blacklist_info->exponential_count = INT_MAX;
 123   } else {
 124     blacklist_info->exponential_count =
 125         blacklist_info->exponential_count * 2 + 1;
 126   }
 127
 128   blacklist_info->count = blacklist_info->exponential_count;
 129   blacklist_info->reason = blacklist_reason;
 130 }
 131
 132 void SdchManager::BlacklistDomainForever(const GURL& url,
 133                                          SdchProblemCode blacklist_reason) {
 134   SetAllowLatencyExperiment(url, false);
 135
 136   BlacklistInfo* blacklist_info =
 137       &blacklisted_domains_[base::StringToLowerASCII(url.host())];
 138   blacklist_info->count = INT_MAX;
 139   blacklist_info->exponential_count = INT_MAX;
 140   blacklist_info->reason = blacklist_reason;
 141 }
 142
 143 void SdchManager::ClearBlacklistings() {
 144   blacklisted_domains_.clear();
 145 }
 146
 147 void SdchManager::ClearDomainBlacklisting(const std::string& domain) {
 148   BlacklistInfo* blacklist_info = &blacklisted_domains_[
 149       base::StringToLowerASCII(domain)];
 150   blacklist_info->count = 0;
 151   blacklist_info->reason = SDCH_OK;
 152 }
 153
 154 int SdchManager::BlackListDomainCount(const std::string& domain) {
 155   std::string domain_lower(base::StringToLowerASCII(domain));
 156
 157   if (blacklisted_domains_.end() == blacklisted_domains_.find(domain_lower))
 158     return 0;
 159   return blacklisted_domains_[domain_lower].count;
 160 }
 161
 162 int SdchManager::BlacklistDomainExponential(const std::string& domain) {
 163   std::string domain_lower(base::StringToLowerASCII(domain));
 164
 165   if (blacklisted_domains_.end() == blacklisted_domains_.find(domain_lower))
 166     return 0;
 167   return blacklisted_domains_[domain_lower].exponential_count;
 168 }
 169
 170 SdchProblemCode SdchManager::IsInSupportedDomain(const GURL& url) {
 171   DCHECK(thread_checker_.CalledOnValidThread());
 172   if (!g_sdch_enabled_ )
 173     return SDCH_DISABLED;
 174
 175   if (blacklisted_domains_.empty())
 176     return SDCH_OK;
 177
 178   DomainBlacklistInfo::iterator it =
 179       blacklisted_domains_.find(base::StringToLowerASCII(url.host()));
 180   if (blacklisted_domains_.end() == it || it->second.count == 0)
 181     return SDCH_OK;
 182
 183   UMA_HISTOGRAM_ENUMERATION("Sdch3.BlacklistReason", it->second.reason,
 184                             SDCH_MAX_PROBLEM_CODE);
 185
 186   int count = it->second.count - 1;
 187   if (count > 0) {
 188     it->second.count = count;
 189   } else {
 190     it->second.count = 0;
 191     it->second.reason = SDCH_OK;
 192   }
 193
 194   return SDCH_DOMAIN_BLACKLIST_INCLUDES_TARGET;
 195 }
 196
 197 SdchProblemCode SdchManager::OnGetDictionary(const GURL& request_url,
 198                                              const GURL& dictionary_url) {
 199   DCHECK(thread_checker_.CalledOnValidThread());
 200   SdchProblemCode rv = CanFetchDictionary(request_url, dictionary_url);
 201   if (rv != SDCH_OK)
 202     return rv;
 203
 204   FOR_EACH_OBSERVER(SdchObserver,
 205                     observers_,
 206                     OnGetDictionary(request_url, dictionary_url));
 207
 208   return SDCH_OK;
 209 }
 210
 211 void SdchManager::OnDictionaryUsed(const std::string& server_hash) {
 212   FOR_EACH_OBSERVER(SdchObserver, observers_,
 213                     OnDictionaryUsed(server_hash));
 214 }
 215
 216 SdchProblemCode SdchManager::CanFetchDictionary(
 217     const GURL& referring_url,
 218     const GURL& dictionary_url) const {
 219   DCHECK(thread_checker_.CalledOnValidThread());
 220   /* The user agent may retrieve a dictionary from the dictionary URL if all of
 221      the following are true:
 222        1 The dictionary URL host name matches the referrer URL host name and
 223            scheme.
 224        2 The dictionary URL host name domain matches the parent domain of the
 225            referrer URL host name
 226        3 The parent domain of the referrer URL host name is not a top level
 227            domain
 228    */
 229   // Item (1) above implies item (2). Spec should be updated.
 230   // I take "host name match" to be "is identical to"
 231   if (referring_url.host() != dictionary_url.host() ||
 232       referring_url.scheme() != dictionary_url.scheme())
 233     return SDCH_DICTIONARY_LOAD_ATTEMPT_FROM_DIFFERENT_HOST;
 234
 235   // TODO(jar): Remove this failsafe conservative hack which is more restrictive
 236   // than current SDCH spec when needed, and justified by security audit.
 237   if (!referring_url.SchemeIsHTTPOrHTTPS())
 238     return SDCH_DICTIONARY_SELECTED_FROM_NON_HTTP;
 239
 240   return SDCH_OK;
 241 }
 242
 243 scoped_ptr<SdchManager::DictionarySet>
 244 SdchManager::GetDictionarySet(const GURL& target_url) {
 245   if (IsInSupportedDomain(target_url) != SDCH_OK)
 246     return NULL;
 247
 248   int count = 0;
 249   scoped_ptr<SdchManager::DictionarySet> result(new DictionarySet);
 250   for (const auto& entry: dictionaries_) {
 251     if (entry.second->data.CanUse(target_url) != SDCH_OK)
 252       continue;
 253     if (entry.second->data.Expired())
 254       continue;
 255     ++count;
 256     result->AddDictionary(entry.first, entry.second);
 257   }
 258
 259   if (count == 0)
 260     return NULL;
 261
 262   UMA_HISTOGRAM_COUNTS("Sdch3.Advertisement_Count", count);
 263   UMA_HISTOGRAM_BOOLEAN("Sdch3.AdvertisedWithSecureScheme",
 264                         target_url.SchemeIsSecure());
 265
 266   return result.Pass();
 267 }
 268
 269 scoped_ptr<SdchManager::DictionarySet>
 270 SdchManager::GetDictionarySetByHash(
 271     const GURL& target_url,
 272     const std::string& server_hash,
 273     SdchProblemCode* problem_code) {
 274   scoped_ptr<SdchManager::DictionarySet> result;
 275
 276   *problem_code = SDCH_DICTIONARY_HASH_NOT_FOUND;
 277   const auto& it = dictionaries_.find(server_hash);
 278   if (it == dictionaries_.end())
 279     return result.Pass();
 280
 281   *problem_code = it->second->data.CanUse(target_url);
 282   if (*problem_code != SDCH_OK)
 283     return result.Pass();
 284
 285   result.reset(new DictionarySet);
 286   result->AddDictionary(it->first, it->second);
 287   return result.Pass();
 288 }
 289
 290 // static
 291 void SdchManager::GenerateHash(const std::string& dictionary_text,
 292     std::string* client_hash, std::string* server_hash) {
 293   char binary_hash[32];
 294   crypto::SHA256HashString(dictionary_text, binary_hash, sizeof(binary_hash));
 295
 296   std::string first_48_bits(&binary_hash[0], 6);
 297   std::string second_48_bits(&binary_hash[6], 6);
 298   UrlSafeBase64Encode(first_48_bits, client_hash);
 299   UrlSafeBase64Encode(second_48_bits, server_hash);
 300
 301   DCHECK_EQ(server_hash->length(), 8u);
 302   DCHECK_EQ(client_hash->length(), 8u);
 303 }
 304
 305 // Methods for supporting latency experiments.
 306
 307 bool SdchManager::AllowLatencyExperiment(const GURL& url) const {
 308   DCHECK(thread_checker_.CalledOnValidThread());
 309   return allow_latency_experiment_.end() !=
 310       allow_latency_experiment_.find(url.host());
 311 }
 312
 313 void SdchManager::SetAllowLatencyExperiment(const GURL& url, bool enable) {
 314   DCHECK(thread_checker_.CalledOnValidThread());
 315   if (enable) {
 316     allow_latency_experiment_.insert(url.host());
 317     return;
 318   }
 319   ExperimentSet::iterator it = allow_latency_experiment_.find(url.host());
 320   if (allow_latency_experiment_.end() == it)
 321     return;  // It was already erased, or never allowed.
 322   SdchErrorRecovery(SDCH_LATENCY_TEST_DISALLOWED);
 323   allow_latency_experiment_.erase(it);
 324 }
 325
 326 void SdchManager::AddObserver(SdchObserver* observer) {
 327   observers_.AddObserver(observer);
 328 }
 329
 330 void SdchManager::RemoveObserver(SdchObserver* observer) {
 331   observers_.RemoveObserver(observer);
 332 }
 333
 334 SdchProblemCode SdchManager::AddSdchDictionary(
 335     const std::string& dictionary_text,
 336     const GURL& dictionary_url,
 337     std::string* server_hash_p) {
 338   DCHECK(thread_checker_.CalledOnValidThread());
 339   std::string client_hash;
 340   std::string server_hash;
 341   GenerateHash(dictionary_text, &client_hash, &server_hash);
 342   if (dictionaries_.find(server_hash) != dictionaries_.end())
 343     return SDCH_DICTIONARY_ALREADY_LOADED;  // Already loaded.
 344
 345   std::string domain, path;
 346   std::set<int> ports;
 347   base::Time expiration(base::Time::Now() + base::TimeDelta::FromDays(30));
 348
 349   if (dictionary_text.empty())
 350     return SDCH_DICTIONARY_HAS_NO_TEXT;  // Missing header.
 351
 352   size_t header_end = dictionary_text.find("\n\n");
 353   if (std::string::npos == header_end)
 354     return SDCH_DICTIONARY_HAS_NO_HEADER;  // Missing header.
 355
 356   size_t line_start = 0;  // Start of line being parsed.
 357   while (1) {
 358     size_t line_end = dictionary_text.find('\n', line_start);
 359     DCHECK(std::string::npos != line_end);
 360     DCHECK_LE(line_end, header_end);
 361
 362     size_t colon_index = dictionary_text.find(':', line_start);
 363     if (std::string::npos == colon_index)
 364       return SDCH_DICTIONARY_HEADER_LINE_MISSING_COLON;  // Illegal line missing
 365                                                          // a colon.
 366
 367     if (colon_index > line_end)
 368       break;
 369
 370     size_t value_start = dictionary_text.find_first_not_of(" \t",
 371                                                            colon_index + 1);
 372     if (std::string::npos != value_start) {
 373       if (value_start >= line_end)
 374         break;
 375       std::string name(dictionary_text, line_start, colon_index - line_start);
 376       std::string value(dictionary_text, value_start, line_end - value_start);
 377       name = base::StringToLowerASCII(name);
 378       if (name == "domain") {
 379         domain = value;
 380       } else if (name == "path") {
 381         path = value;
 382       } else if (name == "format-version") {
 383         if (value != "1.0")
 384           return SDCH_DICTIONARY_UNSUPPORTED_VERSION;
 385       } else if (name == "max-age") {
 386         int64_t seconds;
 387         base::StringToInt64(value, &seconds);
 388         expiration = base::Time::Now() + base::TimeDelta::FromSeconds(seconds);
 389       } else if (name == "port") {
 390         int port;
 391         base::StringToInt(value, &port);
 392         if (port >= 0)
 393           ports.insert(port);
 394       }
 395     }
 396
 397     if (line_end >= header_end)
 398       break;
 399     line_start = line_end + 1;
 400   }
 401
 402   // Narrow fix for http://crbug.com/389451.
 403   GURL dictionary_url_normalized(dictionary_url);
 404   StripTrailingDot(&dictionary_url_normalized);
 405
 406   SdchProblemCode rv = IsInSupportedDomain(dictionary_url_normalized);
 407   if (rv != SDCH_OK)
 408     return rv;
 409
 410   rv = SdchDictionary::CanSet(domain, path, ports, dictionary_url_normalized);
 411   if (rv != SDCH_OK)
 412     return rv;
 413
 414   UMA_HISTOGRAM_COUNTS("Sdch3.Dictionary size loaded", dictionary_text.size());
 415   DVLOG(1) << "Loaded dictionary with client hash " << client_hash
 416            << " and server hash " << server_hash;
 417   SdchDictionary dictionary(dictionary_text, header_end + 2, client_hash,
 418                             server_hash, dictionary_url_normalized, domain,
 419                             path, expiration, ports);
 420   dictionaries_[server_hash] =
 421       new base::RefCountedData<SdchDictionary>(dictionary);
 422   if (server_hash_p)
 423     *server_hash_p = server_hash;
 424
 425   FOR_EACH_OBSERVER(SdchObserver, observers_,
 426                     OnDictionaryAdded(dictionary_url, server_hash));
 427
 428   return SDCH_OK;
 429 }
 430
 431 SdchProblemCode SdchManager::RemoveSdchDictionary(
 432     const std::string& server_hash) {
 433   if (dictionaries_.find(server_hash) == dictionaries_.end())
 434     return SDCH_DICTIONARY_HASH_NOT_FOUND;
 435
 436   dictionaries_.erase(server_hash);
 437
 438   FOR_EACH_OBSERVER(SdchObserver, observers_, OnDictionaryRemoved(server_hash));
 439
 440   return SDCH_OK;
 441 }
 442
 443 // static
 444 scoped_ptr<SdchManager::DictionarySet>
 445 SdchManager::CreateEmptyDictionarySetForTesting() {
 446   return scoped_ptr<DictionarySet>(new DictionarySet).Pass();
 447 }
 448
 449 // static
 450 void SdchManager::UrlSafeBase64Encode(const std::string& input,
 451                                       std::string* output) {
 452   // Since this is only done during a dictionary load, and hashes are only 8
 453   // characters, we just do the simple fixup, rather than rewriting the encoder.
 454   base::Base64Encode(input, output);
 455   std::replace(output->begin(), output->end(), '+', '-');
 456   std::replace(output->begin(), output->end(), '/', '_');
 457 }
 458
 459 scoped_ptr<base::Value> SdchManager::SdchInfoToValue() const {
 460   scoped_ptr<base::DictionaryValue> value(new base::DictionaryValue());
 461
 462   value->SetBoolean("sdch_enabled", sdch_enabled());
 463
 464   scoped_ptr<base::ListValue> entry_list(new base::ListValue());
 465   for (const auto& entry: dictionaries_) {
 466     scoped_ptr<base::DictionaryValue> entry_dict(new base::DictionaryValue());
 467     entry_dict->SetString("url", entry.second->data.url().spec());
 468     entry_dict->SetString("client_hash", entry.second->data.client_hash());
 469     entry_dict->SetString("domain", entry.second->data.domain());
 470     entry_dict->SetString("path", entry.second->data.path());
 471     scoped_ptr<base::ListValue> port_list(new base::ListValue());
 472     for (std::set<int>::const_iterator port_it =
 473              entry.second->data.ports().begin();
 474          port_it != entry.second->data.ports().end(); ++port_it) {
 475       port_list->AppendInteger(*port_it);
 476     }
 477     entry_dict->Set("ports", port_list.Pass());
 478     entry_dict->SetString("server_hash", entry.first);
 479     entry_list->Append(entry_dict.Pass());
 480   }
 481   value->Set("dictionaries", entry_list.Pass());
 482
 483   entry_list.reset(new base::ListValue());
 484   for (DomainBlacklistInfo::const_iterator it = blacklisted_domains_.begin();
 485        it != blacklisted_domains_.end(); ++it) {
 486     if (it->second.count == 0)
 487       continue;
 488     scoped_ptr<base::DictionaryValue> entry_dict(new base::DictionaryValue());
 489     entry_dict->SetString("domain", it->first);
 490     if (it->second.count != INT_MAX)
 491       entry_dict->SetInteger("tries", it->second.count);
 492     entry_dict->SetInteger("reason", it->second.reason);
 493     entry_list->Append(entry_dict.Pass());
 494   }
 495   value->Set("blacklisted", entry_list.Pass());
 496
 497   return value.Pass();
 498 }
 499
 500 }  // namespace net