net/base/sdch_manager.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/base/sdch_manager.h"
   6
   7 #include "base/base64.h"
   8 #include "base/logging.h"
   9 #include "base/metrics/histogram.h"
  10 #include "base/strings/string_number_conversions.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/time/default_clock.h"
  13 #include "base/values.h"
  14 #include "crypto/sha2.h"
  15 #include "net/base/sdch_observer.h"
  16 #include "net/url_request/url_request_http_job.h"
  17
  18 namespace {
  19
  20 void StripTrailingDot(GURL* gurl) {
  21   std::string host(gurl->host());
  22
  23   if (host.empty())
  24     return;
  25
  26   if (*host.rbegin() != '.')
  27     return;
  28
  29   host.resize(host.size() - 1);
  30
  31   GURL::Replacements replacements;
  32   replacements.SetHostStr(host);
  33   *gurl = gurl->ReplaceComponents(replacements);
  34   return;
  35 }
  36
  37 }  // namespace
  38
  39 namespace net {
  40
  41 // static
  42 bool SdchManager::g_sdch_enabled_ = true;
  43
  44 // static
  45 bool SdchManager::g_secure_scheme_supported_ = true;
  46
  47 SdchManager::DictionarySet::DictionarySet() {}
  48
  49 SdchManager::DictionarySet::~DictionarySet() {}
  50
  51 std::string SdchManager::DictionarySet::GetDictionaryClientHashList() const {
  52   std::string result;
  53   bool first = true;
  54   for (const auto& entry: dictionaries_) {
  55     if (!first)
  56       result.append(",");
  57
  58     result.append(entry.second->data.client_hash());
  59     first = false;
  60   }
  61   return result;
  62 }
  63
  64 bool SdchManager::DictionarySet::Empty() const {
  65   return dictionaries_.empty();
  66 }
  67
  68 const std::string* SdchManager::DictionarySet::GetDictionaryText(
  69     const std::string& server_hash) const {
  70   auto it = dictionaries_.find(server_hash);
  71   if (it == dictionaries_.end())
  72     return nullptr;
  73   return &it->second->data.text();
  74 }
  75
  76 void SdchManager::DictionarySet::AddDictionary(
  77     const std::string& server_hash,
  78     const scoped_refptr<base::RefCountedData<SdchDictionary>>& dictionary) {
  79   DCHECK(dictionaries_.end() == dictionaries_.find(server_hash));
  80
  81   dictionaries_[server_hash] = dictionary;
  82 }
  83
  84 SdchManager::SdchManager() : factory_(this) {
  85   DCHECK(thread_checker_.CalledOnValidThread());
  86 }
  87
  88 SdchManager::~SdchManager() {
  89   DCHECK(thread_checker_.CalledOnValidThread());
  90   while (!dictionaries_.empty()) {
  91     auto it = dictionaries_.begin();
  92     dictionaries_.erase(it->first);
  93   }
  94 #if defined(OS_CHROMEOS)
  95   // For debugging http://crbug.com/454198; remove when resolved.
  96
  97   // Explicitly confirm that we can't notify any observers anymore.
  98   CHECK(!observers_.might_have_observers());
  99 #endif
 100 }
 101
 102 void SdchManager::ClearData() {
 103   blacklisted_domains_.clear();
 104   allow_latency_experiment_.clear();
 105   dictionaries_.clear();
 106   FOR_EACH_OBSERVER(SdchObserver, observers_, OnClearDictionaries(this));
 107 }
 108
 109 // static
 110 void SdchManager::SdchErrorRecovery(SdchProblemCode problem) {
 111   UMA_HISTOGRAM_ENUMERATION("Sdch3.ProblemCodes_5", problem,
 112                             SDCH_MAX_PROBLEM_CODE);
 113 }
 114
 115 // static
 116 void SdchManager::EnableSdchSupport(bool enabled) {
 117   g_sdch_enabled_ = enabled;
 118 }
 119
 120 // static
 121 void SdchManager::EnableSecureSchemeSupport(bool enabled) {
 122   g_secure_scheme_supported_ = enabled;
 123 }
 124
 125 void SdchManager::BlacklistDomain(const GURL& url,
 126                                   SdchProblemCode blacklist_reason) {
 127   SetAllowLatencyExperiment(url, false);
 128
 129   BlacklistInfo* blacklist_info =
 130       &blacklisted_domains_[base::StringToLowerASCII(url.host())];
 131
 132   if (blacklist_info->count > 0)
 133     return;  // Domain is already blacklisted.
 134
 135   if (blacklist_info->exponential_count > (INT_MAX - 1) / 2) {
 136     blacklist_info->exponential_count = INT_MAX;
 137   } else {
 138     blacklist_info->exponential_count =
 139         blacklist_info->exponential_count * 2 + 1;
 140   }
 141
 142   blacklist_info->count = blacklist_info->exponential_count;
 143   blacklist_info->reason = blacklist_reason;
 144 }
 145
 146 void SdchManager::BlacklistDomainForever(const GURL& url,
 147                                          SdchProblemCode blacklist_reason) {
 148   SetAllowLatencyExperiment(url, false);
 149
 150   BlacklistInfo* blacklist_info =
 151       &blacklisted_domains_[base::StringToLowerASCII(url.host())];
 152   blacklist_info->count = INT_MAX;
 153   blacklist_info->exponential_count = INT_MAX;
 154   blacklist_info->reason = blacklist_reason;
 155 }
 156
 157 void SdchManager::ClearBlacklistings() {
 158   blacklisted_domains_.clear();
 159 }
 160
 161 void SdchManager::ClearDomainBlacklisting(const std::string& domain) {
 162   BlacklistInfo* blacklist_info = &blacklisted_domains_[
 163       base::StringToLowerASCII(domain)];
 164   blacklist_info->count = 0;
 165   blacklist_info->reason = SDCH_OK;
 166 }
 167
 168 int SdchManager::BlackListDomainCount(const std::string& domain) {
 169   std::string domain_lower(base::StringToLowerASCII(domain));
 170
 171   if (blacklisted_domains_.end() == blacklisted_domains_.find(domain_lower))
 172     return 0;
 173   return blacklisted_domains_[domain_lower].count;
 174 }
 175
 176 int SdchManager::BlacklistDomainExponential(const std::string& domain) {
 177   std::string domain_lower(base::StringToLowerASCII(domain));
 178
 179   if (blacklisted_domains_.end() == blacklisted_domains_.find(domain_lower))
 180     return 0;
 181   return blacklisted_domains_[domain_lower].exponential_count;
 182 }
 183
 184 SdchProblemCode SdchManager::IsInSupportedDomain(const GURL& url) {
 185   DCHECK(thread_checker_.CalledOnValidThread());
 186   if (!g_sdch_enabled_ )
 187     return SDCH_DISABLED;
 188
 189   if (!secure_scheme_supported() && url.SchemeIsSecure())
 190     return SDCH_SECURE_SCHEME_NOT_SUPPORTED;
 191
 192   if (blacklisted_domains_.empty())
 193     return SDCH_OK;
 194
 195   DomainBlacklistInfo::iterator it =
 196       blacklisted_domains_.find(base::StringToLowerASCII(url.host()));
 197   if (blacklisted_domains_.end() == it || it->second.count == 0)
 198     return SDCH_OK;
 199
 200   UMA_HISTOGRAM_ENUMERATION("Sdch3.BlacklistReason", it->second.reason,
 201                             SDCH_MAX_PROBLEM_CODE);
 202
 203   int count = it->second.count - 1;
 204   if (count > 0) {
 205     it->second.count = count;
 206   } else {
 207     it->second.count = 0;
 208     it->second.reason = SDCH_OK;
 209   }
 210
 211   return SDCH_DOMAIN_BLACKLIST_INCLUDES_TARGET;
 212 }
 213
 214 SdchProblemCode SdchManager::OnGetDictionary(const GURL& request_url,
 215                                              const GURL& dictionary_url) {
 216   DCHECK(thread_checker_.CalledOnValidThread());
 217   SdchProblemCode rv = CanFetchDictionary(request_url, dictionary_url);
 218   if (rv != SDCH_OK)
 219     return rv;
 220
 221   FOR_EACH_OBSERVER(SdchObserver,
 222                     observers_,
 223                     OnGetDictionary(this, request_url, dictionary_url));
 224
 225   return SDCH_OK;
 226 }
 227
 228 void SdchManager::OnDictionaryUsed(const std::string& server_hash) {
 229   FOR_EACH_OBSERVER(SdchObserver, observers_,
 230                     OnDictionaryUsed(this, server_hash));
 231 }
 232
 233 SdchProblemCode SdchManager::CanFetchDictionary(
 234     const GURL& referring_url,
 235     const GURL& dictionary_url) const {
 236   DCHECK(thread_checker_.CalledOnValidThread());
 237   /* The user agent may retrieve a dictionary from the dictionary URL if all of
 238      the following are true:
 239        1 The dictionary URL host name matches the referrer URL host name and
 240            scheme.
 241        2 The dictionary URL host name domain matches the parent domain of the
 242            referrer URL host name
 243        3 The parent domain of the referrer URL host name is not a top level
 244            domain
 245    */
 246   // Item (1) above implies item (2). Spec should be updated.
 247   // I take "host name match" to be "is identical to"
 248   if (referring_url.host() != dictionary_url.host() ||
 249       referring_url.scheme() != dictionary_url.scheme())
 250     return SDCH_DICTIONARY_LOAD_ATTEMPT_FROM_DIFFERENT_HOST;
 251
 252   if (!secure_scheme_supported() && referring_url.SchemeIsSecure())
 253     return SDCH_DICTIONARY_SELECTED_FOR_SSL;
 254
 255   // TODO(jar): Remove this failsafe conservative hack which is more restrictive
 256   // than current SDCH spec when needed, and justified by security audit.
 257   if (!referring_url.SchemeIsHTTPOrHTTPS())
 258     return SDCH_DICTIONARY_SELECTED_FROM_NON_HTTP;
 259
 260   return SDCH_OK;
 261 }
 262
 263 scoped_ptr<SdchManager::DictionarySet>
 264 SdchManager::GetDictionarySet(const GURL& target_url) {
 265   if (IsInSupportedDomain(target_url) != SDCH_OK)
 266     return NULL;
 267
 268   int count = 0;
 269   scoped_ptr<SdchManager::DictionarySet> result(new DictionarySet);
 270   for (const auto& entry: dictionaries_) {
 271     if (!secure_scheme_supported() && target_url.SchemeIsSecure())
 272       continue;
 273     if (entry.second->data.CanUse(target_url) != SDCH_OK)
 274       continue;
 275     if (entry.second->data.Expired())
 276       continue;
 277     ++count;
 278     result->AddDictionary(entry.first, entry.second);
 279   }
 280
 281   if (count == 0)
 282     return NULL;
 283
 284   UMA_HISTOGRAM_COUNTS("Sdch3.Advertisement_Count", count);
 285
 286   return result.Pass();
 287 }
 288
 289 scoped_ptr<SdchManager::DictionarySet>
 290 SdchManager::GetDictionarySetByHash(
 291     const GURL& target_url,
 292     const std::string& server_hash,
 293     SdchProblemCode* problem_code) {
 294   scoped_ptr<SdchManager::DictionarySet> result;
 295
 296   *problem_code = SDCH_DICTIONARY_HASH_NOT_FOUND;
 297   const auto& it = dictionaries_.find(server_hash);
 298   if (it == dictionaries_.end())
 299     return result.Pass();
 300
 301   if (!SdchManager::secure_scheme_supported() && target_url.SchemeIsSecure()) {
 302     *problem_code = SDCH_DICTIONARY_FOUND_HAS_WRONG_SCHEME;
 303     return result.Pass();
 304   }
 305
 306   *problem_code = it->second->data.CanUse(target_url);
 307   if (*problem_code != SDCH_OK)
 308     return result.Pass();
 309
 310   result.reset(new DictionarySet);
 311   result->AddDictionary(it->first, it->second);
 312   return result.Pass();
 313 }
 314
 315 // static
 316 void SdchManager::GenerateHash(const std::string& dictionary_text,
 317     std::string* client_hash, std::string* server_hash) {
 318   char binary_hash[32];
 319   crypto::SHA256HashString(dictionary_text, binary_hash, sizeof(binary_hash));
 320
 321   std::string first_48_bits(&binary_hash[0], 6);
 322   std::string second_48_bits(&binary_hash[6], 6);
 323   UrlSafeBase64Encode(first_48_bits, client_hash);
 324   UrlSafeBase64Encode(second_48_bits, server_hash);
 325
 326   DCHECK_EQ(server_hash->length(), 8u);
 327   DCHECK_EQ(client_hash->length(), 8u);
 328 }
 329
 330 // Methods for supporting latency experiments.
 331
 332 bool SdchManager::AllowLatencyExperiment(const GURL& url) const {
 333   DCHECK(thread_checker_.CalledOnValidThread());
 334   return allow_latency_experiment_.end() !=
 335       allow_latency_experiment_.find(url.host());
 336 }
 337
 338 void SdchManager::SetAllowLatencyExperiment(const GURL& url, bool enable) {
 339   DCHECK(thread_checker_.CalledOnValidThread());
 340   if (enable) {
 341     allow_latency_experiment_.insert(url.host());
 342     return;
 343   }
 344   ExperimentSet::iterator it = allow_latency_experiment_.find(url.host());
 345   if (allow_latency_experiment_.end() == it)
 346     return;  // It was already erased, or never allowed.
 347   SdchErrorRecovery(SDCH_LATENCY_TEST_DISALLOWED);
 348   allow_latency_experiment_.erase(it);
 349 }
 350
 351 void SdchManager::AddObserver(SdchObserver* observer) {
 352   observers_.AddObserver(observer);
 353 }
 354
 355 void SdchManager::RemoveObserver(SdchObserver* observer) {
 356   observers_.RemoveObserver(observer);
 357 }
 358
 359 SdchProblemCode SdchManager::AddSdchDictionary(
 360     const std::string& dictionary_text,
 361     const GURL& dictionary_url,
 362     std::string* server_hash_p) {
 363   DCHECK(thread_checker_.CalledOnValidThread());
 364   std::string client_hash;
 365   std::string server_hash;
 366   GenerateHash(dictionary_text, &client_hash, &server_hash);
 367   if (dictionaries_.find(server_hash) != dictionaries_.end())
 368     return SDCH_DICTIONARY_ALREADY_LOADED;  // Already loaded.
 369
 370   std::string domain, path;
 371   std::set<int> ports;
 372   base::Time expiration(base::Time::Now() + base::TimeDelta::FromDays(30));
 373
 374   if (dictionary_text.empty())
 375     return SDCH_DICTIONARY_HAS_NO_TEXT;  // Missing header.
 376
 377   size_t header_end = dictionary_text.find("\n\n");
 378   if (std::string::npos == header_end)
 379     return SDCH_DICTIONARY_HAS_NO_HEADER;  // Missing header.
 380
 381   size_t line_start = 0;  // Start of line being parsed.
 382   while (1) {
 383     size_t line_end = dictionary_text.find('\n', line_start);
 384     DCHECK(std::string::npos != line_end);
 385     DCHECK_LE(line_end, header_end);
 386
 387     size_t colon_index = dictionary_text.find(':', line_start);
 388     if (std::string::npos == colon_index)
 389       return SDCH_DICTIONARY_HEADER_LINE_MISSING_COLON;  // Illegal line missing
 390                                                          // a colon.
 391
 392     if (colon_index > line_end)
 393       break;
 394
 395     size_t value_start = dictionary_text.find_first_not_of(" \t",
 396                                                            colon_index + 1);
 397     if (std::string::npos != value_start) {
 398       if (value_start >= line_end)
 399         break;
 400       std::string name(dictionary_text, line_start, colon_index - line_start);
 401       std::string value(dictionary_text, value_start, line_end - value_start);
 402       name = base::StringToLowerASCII(name);
 403       if (name == "domain") {
 404         domain = value;
 405       } else if (name == "path") {
 406         path = value;
 407       } else if (name == "format-version") {
 408         if (value != "1.0")
 409           return SDCH_DICTIONARY_UNSUPPORTED_VERSION;
 410       } else if (name == "max-age") {
 411         int64 seconds;
 412         base::StringToInt64(value, &seconds);
 413         expiration = base::Time::Now() + base::TimeDelta::FromSeconds(seconds);
 414       } else if (name == "port") {
 415         int port;
 416         base::StringToInt(value, &port);
 417         if (port >= 0)
 418           ports.insert(port);
 419       }
 420     }
 421
 422     if (line_end >= header_end)
 423       break;
 424     line_start = line_end + 1;
 425   }
 426
 427   // Narrow fix for http://crbug.com/389451.
 428   GURL dictionary_url_normalized(dictionary_url);
 429   StripTrailingDot(&dictionary_url_normalized);
 430
 431   SdchProblemCode rv = IsInSupportedDomain(dictionary_url_normalized);
 432   if (rv != SDCH_OK)
 433     return rv;
 434
 435   rv = SdchDictionary::CanSet(domain, path, ports, dictionary_url_normalized);
 436   if (rv != SDCH_OK)
 437     return rv;
 438
 439   UMA_HISTOGRAM_COUNTS("Sdch3.Dictionary size loaded", dictionary_text.size());
 440   DVLOG(1) << "Loaded dictionary with client hash " << client_hash
 441            << " and server hash " << server_hash;
 442   SdchDictionary dictionary(dictionary_text, header_end + 2, client_hash,
 443                             server_hash, dictionary_url_normalized, domain,
 444                             path, expiration, ports);
 445   dictionaries_[server_hash] =
 446       new base::RefCountedData<SdchDictionary>(dictionary);
 447   if (server_hash_p)
 448     *server_hash_p = server_hash;
 449
 450   return SDCH_OK;
 451 }
 452
 453 SdchProblemCode SdchManager::RemoveSdchDictionary(
 454     const std::string& server_hash) {
 455   if (dictionaries_.find(server_hash) == dictionaries_.end())
 456     return SDCH_DICTIONARY_HASH_NOT_FOUND;
 457
 458   dictionaries_.erase(server_hash);
 459   return SDCH_OK;
 460 }
 461
 462 // static
 463 scoped_ptr<SdchManager::DictionarySet>
 464 SdchManager::CreateEmptyDictionarySetForTesting() {
 465   return scoped_ptr<DictionarySet>(new DictionarySet).Pass();
 466 }
 467
 468 // For investigation of http://crbug.com/454198; remove when resolved.
 469 base::WeakPtr<SdchManager> SdchManager::GetWeakPtr() {
 470   return factory_.GetWeakPtr();
 471 }
 472
 473 // static
 474 void SdchManager::UrlSafeBase64Encode(const std::string& input,
 475                                       std::string* output) {
 476   // Since this is only done during a dictionary load, and hashes are only 8
 477   // characters, we just do the simple fixup, rather than rewriting the encoder.
 478   base::Base64Encode(input, output);
 479   std::replace(output->begin(), output->end(), '+', '-');
 480   std::replace(output->begin(), output->end(), '/', '_');
 481 }
 482
 483 base::Value* SdchManager::SdchInfoToValue() const {
 484   base::DictionaryValue* value = new base::DictionaryValue();
 485
 486   value->SetBoolean("sdch_enabled", sdch_enabled());
 487   value->SetBoolean("secure_scheme_support", secure_scheme_supported());
 488
 489   base::ListValue* entry_list = new base::ListValue();
 490   for (const auto& entry: dictionaries_) {
 491     base::DictionaryValue* entry_dict = new base::DictionaryValue();
 492     entry_dict->SetString("url", entry.second->data.url().spec());
 493     entry_dict->SetString("client_hash", entry.second->data.client_hash());
 494     entry_dict->SetString("domain", entry.second->data.domain());
 495     entry_dict->SetString("path", entry.second->data.path());
 496     base::ListValue* port_list = new base::ListValue();
 497     for (std::set<int>::const_iterator port_it =
 498              entry.second->data.ports().begin();
 499          port_it != entry.second->data.ports().end(); ++port_it) {
 500       port_list->AppendInteger(*port_it);
 501     }
 502     entry_dict->Set("ports", port_list);
 503     entry_dict->SetString("server_hash", entry.first);
 504     entry_list->Append(entry_dict);
 505   }
 506   value->Set("dictionaries", entry_list);
 507
 508   entry_list = new base::ListValue();
 509   for (DomainBlacklistInfo::const_iterator it = blacklisted_domains_.begin();
 510        it != blacklisted_domains_.end(); ++it) {
 511     if (it->second.count == 0)
 512       continue;
 513     base::DictionaryValue* entry_dict = new base::DictionaryValue();
 514     entry_dict->SetString("domain", it->first);
 515     if (it->second.count != INT_MAX)
 516       entry_dict->SetInteger("tries", it->second.count);
 517     entry_dict->SetInteger("reason", it->second.reason);
 518     entry_list->Append(entry_dict);
 519   }
 520   value->Set("blacklisted", entry_list);
 521
 522   return value;
 523 }
 524
 525 }  // namespace net