chrome/browser/net/spdyproxy/data_saving_metrics.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/net/spdyproxy/data_saving_metrics.h"
   6
   7 #include "base/metrics/histogram.h"
   8 #include "base/prefs/pref_service.h"
   9 #include "base/prefs/scoped_user_pref_update.h"
  10 #include "base/strings/string_number_conversions.h"
  11 #include "base/strings/string_util.h"
  12 #include "chrome/browser/net/spdyproxy/data_reduction_proxy_settings.h"
  13 #include "chrome/common/pref_names.h"
  14 #include "content/public/common/url_constants.h"
  15 #include "net/base/host_port_pair.h"
  16 #include "net/http/http_response_headers.h"
  17 #include "net/proxy/proxy_retry_info.h"
  18 #include "net/proxy/proxy_service.h"
  19 #include "net/url_request/url_request_context.h"
  20
  21 namespace {
  22
  23 // A bypass delay more than this is treated as a long delay.
  24 const int kLongBypassDelayInSeconds = 30 * 60;
  25
  26 #if defined(OS_ANDROID) || defined(OS_IOS)
  27
  28 // The number of days of history stored in the content lengths prefs.
  29 const size_t kNumDaysInHistory = 60;
  30
  31 // Increments an int64, stored as a string, in a ListPref at the specified
  32 // index.  The value must already exist and be a string representation of a
  33 // number.
  34 void AddInt64ToListPref(size_t index,
  35                         int64 length,
  36                         base::ListValue* list_update) {
  37   int64 value = 0;
  38   std::string old_string_value;
  39   bool rv = list_update->GetString(index, &old_string_value);
  40   DCHECK(rv);
  41   if (rv) {
  42     rv = base::StringToInt64(old_string_value, &value);
  43     DCHECK(rv);
  44   }
  45   value += length;
  46   list_update->Set(index,
  47                    base::Value::CreateStringValue(base::Int64ToString(value)));
  48 }
  49
  50 int64 ListPrefInt64Value(const base::ListValue& list_update, size_t index) {
  51   std::string string_value;
  52   if (!list_update.GetString(index, &string_value)) {
  53     NOTREACHED();
  54     return 0;
  55   }
  56
  57   int64 value = 0;
  58   bool rv = base::StringToInt64(string_value, &value);
  59   DCHECK(rv);
  60   return value;
  61 }
  62
  63 // Report UMA metrics for daily data reductions.
  64 void RecordDailyContentLengthHistograms(
  65     int64 original_length,
  66     int64 received_length,
  67     int64 original_length_with_data_reduction_enabled,
  68     int64 received_length_with_data_reduction_enabled,
  69     int64 original_length_via_data_reduction_proxy,
  70     int64 received_length_via_data_reduction_proxy,
  71     int64 https_length_with_data_reduction_enabled,
  72     int64 short_bypass_length_with_data_reduction_enabled,
  73     int64 long_bypass_length_with_data_reduction_enabled,
  74     int64 unknown_length_with_data_reduction_enabled) {
  75   // Report daily UMA only for days having received content.
  76   if (original_length <= 0 || received_length <= 0)
  77     return;
  78
  79   // Record metrics in KB.
  80   UMA_HISTOGRAM_COUNTS(
  81       "Net.DailyOriginalContentLength", original_length >> 10);
  82   UMA_HISTOGRAM_COUNTS(
  83       "Net.DailyContentLength", received_length >> 10);
  84   int percent = 0;
  85   // UMA percentage cannot be negative.
  86   if (original_length > received_length) {
  87     percent = (100 * (original_length - received_length)) / original_length;
  88   }
  89   UMA_HISTOGRAM_PERCENTAGE("Net.DailyContentSavingPercent", percent);
  90
  91   if (original_length_with_data_reduction_enabled <= 0 ||
  92       received_length_with_data_reduction_enabled <= 0) {
  93     return;
  94   }
  95
  96   UMA_HISTOGRAM_COUNTS(
  97       "Net.DailyOriginalContentLength_DataReductionProxyEnabled",
  98       original_length_with_data_reduction_enabled >> 10);
  99   UMA_HISTOGRAM_COUNTS(
 100       "Net.DailyContentLength_DataReductionProxyEnabled",
 101       received_length_with_data_reduction_enabled >> 10);
 102
 103   int percent_data_reduction_proxy_enabled = 0;
 104   // UMA percentage cannot be negative.
 105   if (original_length_with_data_reduction_enabled >
 106       received_length_with_data_reduction_enabled) {
 107     percent_data_reduction_proxy_enabled =
 108         100 * (original_length_with_data_reduction_enabled -
 109                received_length_with_data_reduction_enabled) /
 110         original_length_with_data_reduction_enabled;
 111   }
 112   UMA_HISTOGRAM_PERCENTAGE(
 113       "Net.DailyContentSavingPercent_DataReductionProxyEnabled",
 114       percent_data_reduction_proxy_enabled);
 115
 116   UMA_HISTOGRAM_PERCENTAGE(
 117       "Net.DailyContentPercent_DataReductionProxyEnabled",
 118       (100 * received_length_with_data_reduction_enabled) / received_length);
 119
 120   if (https_length_with_data_reduction_enabled > 0) {
 121     UMA_HISTOGRAM_COUNTS(
 122         "Net.DailyContentLength_DataReductionProxyEnabled_Https",
 123         https_length_with_data_reduction_enabled >> 10);
 124     UMA_HISTOGRAM_PERCENTAGE(
 125         "Net.DailyContentPercent_DataReductionProxyEnabled_Https",
 126         (100 * https_length_with_data_reduction_enabled) / received_length);
 127   }
 128
 129   if (short_bypass_length_with_data_reduction_enabled > 0) {
 130     UMA_HISTOGRAM_COUNTS(
 131         "Net.DailyContentLength_DataReductionProxyEnabled_ShortBypass",
 132         short_bypass_length_with_data_reduction_enabled >> 10);
 133     UMA_HISTOGRAM_PERCENTAGE(
 134         "Net.DailyContentPercent_DataReductionProxyEnabled_ShortBypass",
 135         ((100 * short_bypass_length_with_data_reduction_enabled) /
 136          received_length));
 137   }
 138
 139   if (long_bypass_length_with_data_reduction_enabled > 0) {
 140     UMA_HISTOGRAM_COUNTS(
 141         "Net.DailyContentLength_DataReductionProxyEnabled_LongBypass",
 142         long_bypass_length_with_data_reduction_enabled >> 10);
 143     UMA_HISTOGRAM_PERCENTAGE(
 144         "Net.DailyContentPercent_DataReductionProxyEnabled_LongBypass",
 145         ((100 * long_bypass_length_with_data_reduction_enabled) /
 146          received_length));
 147   }
 148
 149   if (unknown_length_with_data_reduction_enabled > 0) {
 150     UMA_HISTOGRAM_COUNTS(
 151         "Net.DailyContentLength_DataReductionProxyEnabled_Unknown",
 152         unknown_length_with_data_reduction_enabled >> 10);
 153     UMA_HISTOGRAM_PERCENTAGE(
 154         "Net.DailyContentPercent_DataReductionProxyEnabled_Unknown",
 155         ((100 * unknown_length_with_data_reduction_enabled) /
 156          received_length));
 157   }
 158
 159   if (original_length_via_data_reduction_proxy <= 0 ||
 160       received_length_via_data_reduction_proxy <= 0) {
 161     return;
 162   }
 163
 164   UMA_HISTOGRAM_COUNTS(
 165       "Net.DailyOriginalContentLength_ViaDataReductionProxy",
 166       original_length_via_data_reduction_proxy >> 10);
 167   UMA_HISTOGRAM_COUNTS(
 168       "Net.DailyContentLength_ViaDataReductionProxy",
 169       received_length_via_data_reduction_proxy >> 10);
 170   int percent_via_data_reduction_proxy = 0;
 171   if (original_length_via_data_reduction_proxy >
 172       received_length_via_data_reduction_proxy) {
 173     percent_via_data_reduction_proxy =
 174         100 * (original_length_via_data_reduction_proxy -
 175                received_length_via_data_reduction_proxy) /
 176         original_length_via_data_reduction_proxy;
 177   }
 178   UMA_HISTOGRAM_PERCENTAGE(
 179       "Net.DailyContentSavingPercent_ViaDataReductionProxy",
 180       percent_via_data_reduction_proxy);
 181   UMA_HISTOGRAM_PERCENTAGE(
 182       "Net.DailyContentPercent_ViaDataReductionProxy",
 183       (100 * received_length_via_data_reduction_proxy) / received_length);
 184 }
 185
 186 // Ensure list has exactly |length| elements, either by truncating at the
 187 // front, or appending "0"'s to the back.
 188 void MaintainContentLengthPrefsWindow(base::ListValue* list, size_t length) {
 189   // Remove data for old days from the front.
 190   while (list->GetSize() > length)
 191     list->Remove(0, NULL);
 192   // Newly added lists are empty. Add entries to back to fill the window,
 193   // each initialized to zero.
 194   while (list->GetSize() < length)
 195     list->AppendString(base::Int64ToString(0));
 196   DCHECK_EQ(length, list->GetSize());
 197 }
 198
 199 // DailyContentLengthUpdate maintains a data saving pref. The pref is a list
 200 // of |kNumDaysInHistory| elements of daily total content lengths for the past
 201 // |kNumDaysInHistory| days.
 202 class DailyContentLengthUpdate {
 203  public:
 204   DailyContentLengthUpdate(
 205       const char* pref,
 206       PrefService* pref_service)
 207       : update_(pref_service, pref) {
 208   }
 209
 210   void UpdateForDataChange(int days_since_last_update) {
 211     // New empty lists may have been created. Maintain the invariant that
 212     // there should be exactly |kNumDaysInHistory| days in the histories.
 213     MaintainContentLengthPrefsWindow(update_.Get(), kNumDaysInHistory);
 214     if (days_since_last_update) {
 215       MaintainContentLengthPrefForDateChange(days_since_last_update);
 216     }
 217   }
 218
 219   // Update the lengths for the current day.
 220   void Add(int content_length) {
 221     AddInt64ToListPref(kNumDaysInHistory - 1, content_length, update_.Get());
 222   }
 223
 224   int64 GetListPrefValue(size_t index) {
 225     return ListPrefInt64Value(*update_, index);
 226   }
 227
 228  private:
 229   // Update the list for date change and ensure the list has exactly |length|
 230   // elements. The last entry in the list will be for the current day after
 231   // the update.
 232   void MaintainContentLengthPrefForDateChange(int days_since_last_update) {
 233     if (days_since_last_update == -1) {
 234       // The system may go backwards in time by up to a day for legitimate
 235       // reasons, such as with changes to the time zone. In such cases, we
 236       // keep adding to the current day.
 237       // Note: we accept the fact that some reported data is shifted to
 238       // the adjacent day if users travel back and forth across time zones.
 239       days_since_last_update = 0;
 240     } else if (days_since_last_update < -1) {
 241       // Erase all entries if the system went backwards in time by more than
 242       // a day.
 243       update_->Clear();
 244
 245       days_since_last_update = kNumDaysInHistory;
 246     }
 247     DCHECK_GE(days_since_last_update, 0);
 248
 249     // Add entries for days since last update event. This will make the
 250     // lists longer than kNumDaysInHistory. The additional items will be cut off
 251     // from the head of the lists by |MaintainContentLengthPrefsWindow|, below.
 252     for (int i = 0;
 253          i < days_since_last_update && i < static_cast<int>(kNumDaysInHistory);
 254          ++i) {
 255       update_->AppendString(base::Int64ToString(0));
 256     }
 257
 258     // Entries for new days may have been appended. Maintain the invariant that
 259     // there should be exactly |kNumDaysInHistory| days in the histories.
 260     MaintainContentLengthPrefsWindow(update_.Get(), kNumDaysInHistory);
 261   }
 262
 263   ListPrefUpdate update_;
 264 };
 265
 266 // DailyDataSavingUpdate maintains a pair of data saving prefs, original_update_
 267 // and received_update_. pref_original is a list of |kNumDaysInHistory| elements
 268 // of daily total original content lengths for the past |kNumDaysInHistory|
 269 // days. pref_received is the corresponding list of the daily total received
 270 // content lengths.
 271 class DailyDataSavingUpdate {
 272  public:
 273   DailyDataSavingUpdate(
 274       const char* pref_original,
 275       const char* pref_received,
 276       PrefService* pref_service)
 277       : original_(pref_original, pref_service),
 278         received_(pref_received, pref_service) {
 279   }
 280
 281   void UpdateForDataChange(int days_since_last_update) {
 282     original_.UpdateForDataChange(days_since_last_update);
 283     received_.UpdateForDataChange(days_since_last_update);
 284   }
 285
 286   // Update the lengths for the current day.
 287   void Add(int original_content_length, int received_content_length) {
 288     original_.Add(original_content_length);
 289     received_.Add(received_content_length);
 290   }
 291
 292   int64 GetOriginalListPrefValue(size_t index) {
 293     return original_.GetListPrefValue(index);
 294   }
 295   int64 GetReceivedListPrefValue(size_t index) {
 296     return received_.GetListPrefValue(index);
 297   }
 298
 299  private:
 300   DailyContentLengthUpdate original_;
 301   DailyContentLengthUpdate received_;
 302 };
 303
 304 #endif  // defined(OS_ANDROID) || defined(OS_IOS)
 305
 306 // Returns true if the request is bypassed by all configured data reduction
 307 // proxies. It returns the bypass delay in delay_seconds (if not NULL). If
 308 // the request is bypassed by more than one proxy, delay_seconds returns
 309 // shortest delay.
 310 bool IsBypassRequest(const net::URLRequest* request, int64* delay_seconds) {
 311 #if defined(OS_ANDROID) || defined(OS_IOS)
 312   DataReductionProxySettings::DataReductionProxyList proxies =
 313       DataReductionProxySettings::GetDataReductionProxies();
 314   if (proxies.size() == 0)
 315     return false;
 316
 317   if (request == NULL || request->context() == NULL ||
 318       request->context()->proxy_service() == NULL) {
 319     return false;
 320   }
 321
 322   const net::ProxyRetryInfoMap& retry_map =
 323       request->context()->proxy_service()->proxy_retry_info();
 324   if (retry_map.size() == 0)
 325     return false;
 326
 327   int64 shortest_delay = 0;
 328   // The request is bypassed if all configured proxies are in the retry map.
 329   for (size_t i = 0; i < proxies.size(); ++i) {
 330     std::string proxy = net::HostPortPair::FromURL(proxies[i]).ToString();
 331     // The retry list has the scheme prefix for https but not for http.
 332     if (proxies[i].SchemeIs(content::kHttpsScheme))
 333       proxy = std::string(content::kHttpsScheme) + "://" + proxy;
 334
 335     net::ProxyRetryInfoMap::const_iterator found = retry_map.find(proxy);
 336     if (found == retry_map.end())
 337       return false;
 338     if (shortest_delay == 0 ||
 339         shortest_delay > found->second.current_delay.InSeconds()) {
 340       shortest_delay = found->second.current_delay.InSeconds();
 341     }
 342   }
 343   if (delay_seconds != NULL)
 344     *delay_seconds = shortest_delay;
 345   return true;
 346 #else
 347   return false;
 348 #endif  // defined(OS_ANDROID) || defined(OS_IOS)
 349 }
 350
 351 // IsDataReductionProxyReponse returns true if response_headers contains the
 352 // data reduction proxy Via header value.
 353 bool IsDataReductionProxyReponse(
 354     const net::HttpResponseHeaders* response_headers) {
 355   const char kDatReductionProxyViaValue[] = "1.1 Chrome Compression Proxy";
 356   size_t value_len = strlen(kDatReductionProxyViaValue);
 357   void* iter = NULL;
 358   std::string temp;
 359   while (response_headers->EnumerateHeader(&iter, "Via", &temp)) {
 360     string::const_iterator it =
 361         std::search(temp.begin(),
 362                     temp.end(),
 363                     kDatReductionProxyViaValue,
 364                     kDatReductionProxyViaValue + value_len,
 365                     base::CaseInsensitiveCompareASCII<char>());
 366     if (it != temp.end()) {
 367       return true;
 368     }
 369   }
 370   return false;
 371 }
 372
 373 }  // namespace
 374
 375 namespace spdyproxy {
 376
 377 DataReductionRequestType GetDataReductionRequestType(
 378     const net::URLRequest* request) {
 379   if (request->url().SchemeIs(content::kHttpsScheme))
 380     return HTTPS;
 381   if (!request->url().SchemeIs(content::kHttpScheme)) {
 382     NOTREACHED();
 383     return UNKNOWN_TYPE;
 384   }
 385   int64 bypass_delay = 0;
 386   if (IsBypassRequest(request, &bypass_delay)) {
 387     return (bypass_delay > kLongBypassDelayInSeconds) ?
 388       LONG_BYPASS : SHORT_BYPASS;
 389   }
 390   return IsDataReductionProxyReponse(request->response_info().headers) ?
 391     VIA_DATA_REDUCTION_PROXY: UNKNOWN_TYPE;
 392 }
 393
 394 int64 GetAdjustedOriginalContentLength(
 395     DataReductionRequestType data_reduction_type,
 396     int64 original_content_length,
 397     int64 received_content_length) {
 398   // Since there was no indication of the original content length, presume
 399   // it is no different from the number of bytes read.
 400   if (original_content_length == -1 ||
 401       data_reduction_type != spdyproxy::VIA_DATA_REDUCTION_PROXY) {
 402     return received_content_length;
 403   }
 404   return original_content_length;
 405 }
 406
 407 #if defined(OS_ANDROID) || defined(OS_IOS)
 408 void UpdateContentLengthPrefsForDataReductionProxy(
 409     int received_content_length,
 410     int original_content_length,
 411     bool with_data_reduction_proxy_enabled,
 412     DataReductionRequestType data_reduction_type,
 413     base::Time now, PrefService* prefs) {
 414   // TODO(bengr): Remove this check once the underlying cause of
 415   // http://crbug.com/287821 is fixed. For now, only continue if the current
 416   // year is reported as being between 1972 and 2970.
 417   base::TimeDelta time_since_unix_epoch = now - base::Time::UnixEpoch();
 418   const int kMinDaysSinceUnixEpoch = 365 * 2;  // 2 years.
 419   const int kMaxDaysSinceUnixEpoch = 365 * 1000;  // 1000 years.
 420   if (time_since_unix_epoch.InDays() < kMinDaysSinceUnixEpoch ||
 421       time_since_unix_epoch.InDays() > kMaxDaysSinceUnixEpoch) {
 422     return;
 423   }
 424
 425   // Determine how many days it has been since the last update.
 426   int64 then_internal = prefs->GetInt64(
 427       prefs::kDailyHttpContentLengthLastUpdateDate);
 428   // Local midnight could have been shifted due to time zone change.
 429   base::Time then_midnight =
 430       base::Time::FromInternalValue(then_internal).LocalMidnight();
 431   base::Time midnight = now.LocalMidnight();
 432   int days_since_last_update = (midnight - then_midnight).InDays();
 433
 434   // Each day, we calculate the total number of bytes received and the total
 435   // size of all corresponding resources before any data-reducing recompression
 436   // is applied. These values are used to compute the data savings realized
 437   // by applying our compression techniques. Totals for the last
 438   // |kNumDaysInHistory| days are maintained.
 439   DailyDataSavingUpdate total(
 440       prefs::kDailyHttpOriginalContentLength,
 441       prefs::kDailyHttpReceivedContentLength,
 442       prefs);
 443   total.UpdateForDataChange(days_since_last_update);
 444
 445   DailyDataSavingUpdate proxy_enabled(
 446       prefs::kDailyOriginalContentLengthWithDataReductionProxyEnabled,
 447       prefs::kDailyContentLengthWithDataReductionProxyEnabled,
 448       prefs);
 449   proxy_enabled.UpdateForDataChange(days_since_last_update);
 450
 451   DailyDataSavingUpdate via_proxy(
 452       prefs::kDailyOriginalContentLengthViaDataReductionProxy,
 453       prefs::kDailyContentLengthViaDataReductionProxy,
 454       prefs);
 455   via_proxy.UpdateForDataChange(days_since_last_update);
 456
 457   DailyContentLengthUpdate https(
 458       prefs::kDailyContentLengthHttpsWithDataReductionProxyEnabled, prefs);
 459   https.UpdateForDataChange(days_since_last_update);
 460
 461   DailyContentLengthUpdate short_bypass(
 462       prefs::kDailyContentLengthShortBypassWithDataReductionProxyEnabled,
 463       prefs);
 464   short_bypass.UpdateForDataChange(days_since_last_update);
 465
 466   DailyContentLengthUpdate long_bypass(
 467       prefs::kDailyContentLengthLongBypassWithDataReductionProxyEnabled, prefs);
 468   long_bypass.UpdateForDataChange(days_since_last_update);
 469
 470   DailyContentLengthUpdate unknown(
 471       prefs::kDailyContentLengthUnknownWithDataReductionProxyEnabled, prefs);
 472   unknown.UpdateForDataChange(days_since_last_update);
 473
 474   total.Add(original_content_length, received_content_length);
 475   if (with_data_reduction_proxy_enabled) {
 476     proxy_enabled.Add(original_content_length, received_content_length);
 477     // Ignore data source cases, if exist, when
 478     // "with_data_reduction_proxy_enabled == false"
 479     switch (data_reduction_type) {
 480       case VIA_DATA_REDUCTION_PROXY:
 481         via_proxy.Add(original_content_length, received_content_length);
 482         break;
 483       case HTTPS:
 484         https.Add(received_content_length);
 485         break;
 486       case SHORT_BYPASS:
 487         short_bypass.Add(received_content_length);
 488         break;
 489       case LONG_BYPASS:
 490         long_bypass.Add(received_content_length);
 491         break;
 492       case UNKNOWN_TYPE:
 493         unknown.Add(received_content_length);
 494         break;
 495     }
 496   }
 497
 498   if (days_since_last_update) {
 499     // Record the last update time in microseconds in UTC.
 500     prefs->SetInt64(prefs::kDailyHttpContentLengthLastUpdateDate,
 501                     midnight.ToInternalValue());
 502
 503     // A new day. Report the previous day's data if exists. We'll lose usage
 504     // data if the last time Chrome was run was more than a day ago.
 505     // Here, we prefer collecting less data but the collected data is
 506     // associated with an accurate date.
 507     if (days_since_last_update == 1) {
 508       // The previous day's data point is the second one from the tail.
 509       // Therefore (kNumDaysInHistory - 2) below.
 510       RecordDailyContentLengthHistograms(
 511           total.GetOriginalListPrefValue(kNumDaysInHistory - 2),
 512           total.GetReceivedListPrefValue(kNumDaysInHistory - 2),
 513           proxy_enabled.GetOriginalListPrefValue(kNumDaysInHistory - 2),
 514           proxy_enabled.GetReceivedListPrefValue(kNumDaysInHistory - 2),
 515           via_proxy.GetOriginalListPrefValue(kNumDaysInHistory - 2),
 516           via_proxy.GetReceivedListPrefValue(kNumDaysInHistory - 2),
 517           https.GetListPrefValue(kNumDaysInHistory - 2),
 518           short_bypass.GetListPrefValue(kNumDaysInHistory - 2),
 519           long_bypass.GetListPrefValue(kNumDaysInHistory - 2),
 520           unknown.GetListPrefValue(kNumDaysInHistory - 2));
 521     }
 522   }
 523 }
 524 #endif  // defined(OS_ANDROID) || defined(OS_IOS)
 525
 526 void UpdateContentLengthPrefs(
 527     int received_content_length,
 528     int original_content_length,
 529     bool with_data_reduction_proxy_enabled,
 530     DataReductionRequestType data_reduction_type,
 531     PrefService* prefs) {
 532   int64 total_received = prefs->GetInt64(prefs::kHttpReceivedContentLength);
 533   int64 total_original = prefs->GetInt64(prefs::kHttpOriginalContentLength);
 534   total_received += received_content_length;
 535   total_original += original_content_length;
 536   prefs->SetInt64(prefs::kHttpReceivedContentLength, total_received);
 537   prefs->SetInt64(prefs::kHttpOriginalContentLength, total_original);
 538
 539 #if defined(OS_ANDROID) || defined(OS_IOS)
 540   UpdateContentLengthPrefsForDataReductionProxy(
 541       received_content_length,
 542       original_content_length,
 543       with_data_reduction_proxy_enabled,
 544       data_reduction_type,
 545       base::Time::Now(),
 546       prefs);
 547 #endif  // defined(OS_ANDROID) || defined(OS_IOS)
 548
 549 }
 550
 551 }  // namespace spdyproxy