components/data_reduction_proxy/browser/data_reduction_proxy_metrics.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/data_reduction_proxy/browser/data_reduction_proxy_metrics.h"
   6
   7 #include "base/metrics/histogram.h"
   8 #include "base/prefs/pref_service.h"
   9 #include "base/strings/string_number_conversions.h"
  10 #include "base/strings/string_util.h"
  11 #include "components/data_reduction_proxy/browser/data_reduction_proxy_settings.h"
  12 #include "components/data_reduction_proxy/browser/data_reduction_proxy_statistics_prefs.h"
  13 #include "components/data_reduction_proxy/common/data_reduction_proxy_headers.h"
  14 #include "components/data_reduction_proxy/common/data_reduction_proxy_pref_names.h"
  15 #include "net/base/host_port_pair.h"
  16 #include "net/http/http_response_headers.h"
  17 #include "net/proxy/proxy_retry_info.h"
  18 #include "net/proxy/proxy_service.h"
  19 #include "net/url_request/url_request_context.h"
  20
  21 namespace data_reduction_proxy {
  22
  23 namespace {
  24
  25 // A bypass delay more than this is treated as a long delay.
  26 const int kLongBypassDelayInSeconds = 30 * 60;
  27
  28 // Increments an int64, stored as a string, in a ListPref at the specified
  29 // index.  The value must already exist and be a string representation of a
  30 // number.
  31 void AddInt64ToListPref(size_t index,
  32                         int64 length,
  33                         base::ListValue* list_update) {
  34   int64 value = 0;
  35   std::string old_string_value;
  36   bool rv = list_update->GetString(index, &old_string_value);
  37   DCHECK(rv);
  38   if (rv) {
  39     rv = base::StringToInt64(old_string_value, &value);
  40     DCHECK(rv);
  41   }
  42   value += length;
  43   list_update->Set(index, new base::StringValue(base::Int64ToString(value)));
  44 }
  45
  46 int64 ListPrefInt64Value(const base::ListValue& list_update, size_t index) {
  47   std::string string_value;
  48   if (!list_update.GetString(index, &string_value)) {
  49     NOTREACHED();
  50     return 0;
  51   }
  52
  53   int64 value = 0;
  54   bool rv = base::StringToInt64(string_value, &value);
  55   DCHECK(rv);
  56   return value;
  57 }
  58
  59 // Report UMA metrics for daily data reductions.
  60 void RecordDailyContentLengthHistograms(
  61     int64 original_length,
  62     int64 received_length,
  63     int64 original_length_with_data_reduction_enabled,
  64     int64 received_length_with_data_reduction_enabled,
  65     int64 original_length_via_data_reduction_proxy,
  66     int64 received_length_via_data_reduction_proxy,
  67     int64 https_length_with_data_reduction_enabled,
  68     int64 short_bypass_length_with_data_reduction_enabled,
  69     int64 long_bypass_length_with_data_reduction_enabled,
  70     int64 unknown_length_with_data_reduction_enabled) {
  71   // Report daily UMA only for days having received content.
  72   if (original_length <= 0 || received_length <= 0)
  73     return;
  74
  75   // Record metrics in KB.
  76   UMA_HISTOGRAM_COUNTS(
  77       "Net.DailyOriginalContentLength", original_length >> 10);
  78   UMA_HISTOGRAM_COUNTS(
  79       "Net.DailyContentLength", received_length >> 10);
  80   int percent = 0;
  81   // UMA percentage cannot be negative.
  82   if (original_length > received_length) {
  83     percent = (100 * (original_length - received_length)) / original_length;
  84   }
  85   UMA_HISTOGRAM_PERCENTAGE("Net.DailyContentSavingPercent", percent);
  86
  87   if (original_length_with_data_reduction_enabled <= 0 ||
  88       received_length_with_data_reduction_enabled <= 0) {
  89     return;
  90   }
  91
  92   UMA_HISTOGRAM_COUNTS(
  93       "Net.DailyOriginalContentLength_DataReductionProxyEnabled",
  94       original_length_with_data_reduction_enabled >> 10);
  95   UMA_HISTOGRAM_COUNTS(
  96       "Net.DailyContentLength_DataReductionProxyEnabled",
  97       received_length_with_data_reduction_enabled >> 10);
  98
  99   int percent_data_reduction_proxy_enabled = 0;
 100   // UMA percentage cannot be negative.
 101   if (original_length_with_data_reduction_enabled >
 102       received_length_with_data_reduction_enabled) {
 103     percent_data_reduction_proxy_enabled =
 104         100 * (original_length_with_data_reduction_enabled -
 105                received_length_with_data_reduction_enabled) /
 106         original_length_with_data_reduction_enabled;
 107   }
 108   UMA_HISTOGRAM_PERCENTAGE(
 109       "Net.DailyContentSavingPercent_DataReductionProxyEnabled",
 110       percent_data_reduction_proxy_enabled);
 111
 112   UMA_HISTOGRAM_PERCENTAGE(
 113       "Net.DailyContentPercent_DataReductionProxyEnabled",
 114       (100 * received_length_with_data_reduction_enabled) / received_length);
 115
 116   DCHECK_GE(https_length_with_data_reduction_enabled, 0);
 117   UMA_HISTOGRAM_COUNTS(
 118       "Net.DailyContentLength_DataReductionProxyEnabled_Https",
 119       https_length_with_data_reduction_enabled >> 10);
 120   UMA_HISTOGRAM_PERCENTAGE(
 121       "Net.DailyContentPercent_DataReductionProxyEnabled_Https",
 122       (100 * https_length_with_data_reduction_enabled) / received_length);
 123
 124   DCHECK_GE(short_bypass_length_with_data_reduction_enabled, 0);
 125   UMA_HISTOGRAM_COUNTS(
 126       "Net.DailyContentLength_DataReductionProxyEnabled_ShortBypass",
 127       short_bypass_length_with_data_reduction_enabled >> 10);
 128   UMA_HISTOGRAM_PERCENTAGE(
 129       "Net.DailyContentPercent_DataReductionProxyEnabled_ShortBypass",
 130       ((100 * short_bypass_length_with_data_reduction_enabled) /
 131        received_length));
 132
 133   DCHECK_GE(long_bypass_length_with_data_reduction_enabled, 0);
 134   UMA_HISTOGRAM_COUNTS(
 135       "Net.DailyContentLength_DataReductionProxyEnabled_LongBypass",
 136       long_bypass_length_with_data_reduction_enabled >> 10);
 137   UMA_HISTOGRAM_PERCENTAGE(
 138       "Net.DailyContentPercent_DataReductionProxyEnabled_LongBypass",
 139       ((100 * long_bypass_length_with_data_reduction_enabled) /
 140        received_length));
 141
 142   DCHECK_GE(unknown_length_with_data_reduction_enabled, 0);
 143   UMA_HISTOGRAM_COUNTS(
 144       "Net.DailyContentLength_DataReductionProxyEnabled_Unknown",
 145       unknown_length_with_data_reduction_enabled >> 10);
 146   UMA_HISTOGRAM_PERCENTAGE(
 147       "Net.DailyContentPercent_DataReductionProxyEnabled_Unknown",
 148       ((100 * unknown_length_with_data_reduction_enabled) /
 149        received_length));
 150
 151   DCHECK_GE(original_length_via_data_reduction_proxy, 0);
 152   UMA_HISTOGRAM_COUNTS(
 153       "Net.DailyOriginalContentLength_ViaDataReductionProxy",
 154       original_length_via_data_reduction_proxy >> 10);
 155   DCHECK_GE(received_length_via_data_reduction_proxy, 0);
 156   UMA_HISTOGRAM_COUNTS(
 157       "Net.DailyContentLength_ViaDataReductionProxy",
 158       received_length_via_data_reduction_proxy >> 10);
 159   int percent_via_data_reduction_proxy = 0;
 160   if (original_length_via_data_reduction_proxy >
 161       received_length_via_data_reduction_proxy) {
 162     percent_via_data_reduction_proxy =
 163         100 * (original_length_via_data_reduction_proxy -
 164                received_length_via_data_reduction_proxy) /
 165         original_length_via_data_reduction_proxy;
 166   }
 167   UMA_HISTOGRAM_PERCENTAGE(
 168       "Net.DailyContentSavingPercent_ViaDataReductionProxy",
 169       percent_via_data_reduction_proxy);
 170   UMA_HISTOGRAM_PERCENTAGE(
 171       "Net.DailyContentPercent_ViaDataReductionProxy",
 172       (100 * received_length_via_data_reduction_proxy) / received_length);
 173 }
 174
 175 // Ensure list has exactly |length| elements, either by truncating at the
 176 // front, or appending "0"'s to the back.
 177 void MaintainContentLengthPrefsWindow(base::ListValue* list, size_t length) {
 178   // Remove data for old days from the front.
 179   while (list->GetSize() > length)
 180     list->Remove(0, NULL);
 181   // Newly added lists are empty. Add entries to back to fill the window,
 182   // each initialized to zero.
 183   while (list->GetSize() < length)
 184     list->AppendString(base::Int64ToString(0));
 185   DCHECK_EQ(length, list->GetSize());
 186 }
 187
 188 // DailyContentLengthUpdate maintains a data saving pref. The pref is a list
 189 // of |kNumDaysInHistory| elements of daily total content lengths for the past
 190 // |kNumDaysInHistory| days.
 191 class DailyContentLengthUpdate {
 192  public:
 193   DailyContentLengthUpdate(const char* pref,
 194                            DataReductionProxyStatisticsPrefs* pref_service)
 195       : update_(pref_service->GetList(pref)) {
 196   }
 197
 198   void UpdateForDataChange(int days_since_last_update) {
 199     // New empty lists may have been created. Maintain the invariant that
 200     // there should be exactly |kNumDaysInHistory| days in the histories.
 201     MaintainContentLengthPrefsWindow(update_, kNumDaysInHistory);
 202     if (days_since_last_update) {
 203       MaintainContentLengthPrefForDateChange(days_since_last_update);
 204     }
 205   }
 206
 207   // Update the lengths for the current day.
 208   void Add(int content_length) {
 209     AddInt64ToListPref(kNumDaysInHistory - 1, content_length, update_);
 210   }
 211
 212   int64 GetListPrefValue(size_t index) {
 213     return ListPrefInt64Value(*update_, index);
 214   }
 215
 216  private:
 217   // Update the list for date change and ensure the list has exactly |length|
 218   // elements. The last entry in the list will be for the current day after
 219   // the update.
 220   void MaintainContentLengthPrefForDateChange(int days_since_last_update) {
 221     if (days_since_last_update == -1) {
 222       // The system may go backwards in time by up to a day for legitimate
 223       // reasons, such as with changes to the time zone. In such cases, we
 224       // keep adding to the current day.
 225       // Note: we accept the fact that some reported data is shifted to
 226       // the adjacent day if users travel back and forth across time zones.
 227       days_since_last_update = 0;
 228     } else if (days_since_last_update < -1) {
 229       // Erase all entries if the system went backwards in time by more than
 230       // a day.
 231       update_->Clear();
 232
 233       days_since_last_update = kNumDaysInHistory;
 234     }
 235     DCHECK_GE(days_since_last_update, 0);
 236
 237     // Add entries for days since last update event. This will make the
 238     // lists longer than kNumDaysInHistory. The additional items will be cut off
 239     // from the head of the lists by |MaintainContentLengthPrefsWindow|, below.
 240     for (int i = 0;
 241          i < days_since_last_update && i < static_cast<int>(kNumDaysInHistory);
 242          ++i) {
 243       update_->AppendString(base::Int64ToString(0));
 244     }
 245
 246     // Entries for new days may have been appended. Maintain the invariant that
 247     // there should be exactly |kNumDaysInHistory| days in the histories.
 248     MaintainContentLengthPrefsWindow(update_, kNumDaysInHistory);
 249   }
 250
 251   base::ListValue* update_;
 252 };
 253
 254 // DailyDataSavingUpdate maintains a pair of data saving prefs, original_update_
 255 // and received_update_. pref_original is a list of |kNumDaysInHistory| elements
 256 // of daily total original content lengths for the past |kNumDaysInHistory|
 257 // days. pref_received is the corresponding list of the daily total received
 258 // content lengths.
 259 class DailyDataSavingUpdate {
 260  public:
 261   DailyDataSavingUpdate(const char* pref_original,
 262                         const char* pref_received,
 263                         DataReductionProxyStatisticsPrefs* prefs)
 264       : original_(pref_original, prefs),
 265         received_(pref_received, prefs) {
 266   }
 267
 268   void UpdateForDataChange(int days_since_last_update) {
 269     original_.UpdateForDataChange(days_since_last_update);
 270     received_.UpdateForDataChange(days_since_last_update);
 271   }
 272
 273   // Update the lengths for the current day.
 274   void Add(int original_content_length, int received_content_length) {
 275     original_.Add(original_content_length);
 276     received_.Add(received_content_length);
 277   }
 278
 279   int64 GetOriginalListPrefValue(size_t index) {
 280     return original_.GetListPrefValue(index);
 281   }
 282   int64 GetReceivedListPrefValue(size_t index) {
 283     return received_.GetListPrefValue(index);
 284   }
 285
 286  private:
 287   DailyContentLengthUpdate original_;
 288   DailyContentLengthUpdate received_;
 289 };
 290
 291 }  // namespace
 292
 293 DataReductionProxyRequestType GetDataReductionProxyRequestType(
 294     const net::URLRequest* request) {
 295   if (request->url().SchemeIs("https"))
 296     return HTTPS;
 297   if (!request->url().SchemeIs("http")) {
 298     NOTREACHED();
 299     return UNKNOWN_TYPE;
 300   }
 301   DataReductionProxyParams params(
 302         DataReductionProxyParams::kAllowed |
 303         DataReductionProxyParams::kFallbackAllowed |
 304         DataReductionProxyParams::kPromoAllowed);
 305   base::TimeDelta bypass_delay;
 306   if (params.AreDataReductionProxiesBypassed(*request, &bypass_delay)) {
 307     if (bypass_delay > base::TimeDelta::FromSeconds(kLongBypassDelayInSeconds))
 308       return LONG_BYPASS;
 309     return SHORT_BYPASS;
 310   }
 311   if (request->response_info().headers.get() &&
 312       HasDataReductionProxyViaHeader(request->response_info().headers.get(),
 313                                      NULL)) {
 314     return VIA_DATA_REDUCTION_PROXY;
 315   }
 316   return UNKNOWN_TYPE;
 317 }
 318
 319 int64 GetAdjustedOriginalContentLength(
 320     DataReductionProxyRequestType request_type,
 321     int64 original_content_length,
 322     int64 received_content_length) {
 323   // Since there was no indication of the original content length, presume
 324   // it is no different from the number of bytes read.
 325   if (original_content_length == -1 ||
 326       request_type != VIA_DATA_REDUCTION_PROXY) {
 327     return received_content_length;
 328   }
 329   return original_content_length;
 330 }
 331
 332 void UpdateContentLengthPrefsForDataReductionProxy(
 333     int received_content_length,
 334     int original_content_length,
 335     bool with_data_reduction_proxy_enabled,
 336     DataReductionProxyRequestType request_type,
 337     base::Time now,
 338     DataReductionProxyStatisticsPrefs* prefs) {
 339   // TODO(bengr): Remove this check once the underlying cause of
 340   // http://crbug.com/287821 is fixed. For now, only continue if the current
 341   // year is reported as being between 1972 and 2970.
 342   base::TimeDelta time_since_unix_epoch = now - base::Time::UnixEpoch();
 343   const int kMinDaysSinceUnixEpoch = 365 * 2;  // 2 years.
 344   const int kMaxDaysSinceUnixEpoch = 365 * 1000;  // 1000 years.
 345   if (time_since_unix_epoch.InDays() < kMinDaysSinceUnixEpoch ||
 346       time_since_unix_epoch.InDays() > kMaxDaysSinceUnixEpoch) {
 347     return;
 348   }
 349
 350   // Determine how many days it has been since the last update.
 351   int64 then_internal = prefs->GetInt64(
 352       data_reduction_proxy::prefs::kDailyHttpContentLengthLastUpdateDate);
 353
 354   // Local midnight could have been shifted due to time zone change.
 355   // If time is null then don't care if midnight will be wrong shifted due to
 356   // time zone change because it's still too much time ago.
 357   base::Time then_midnight = base::Time::FromInternalValue(then_internal);
 358   if (!then_midnight.is_null()) {
 359     then_midnight = then_midnight.LocalMidnight();
 360   }
 361   base::Time midnight = now.LocalMidnight();
 362
 363   int days_since_last_update = (midnight - then_midnight).InDays();
 364
 365   // Each day, we calculate the total number of bytes received and the total
 366   // size of all corresponding resources before any data-reducing recompression
 367   // is applied. These values are used to compute the data savings realized
 368   // by applying our compression techniques. Totals for the last
 369   // |kNumDaysInHistory| days are maintained.
 370   DailyDataSavingUpdate total(
 371       data_reduction_proxy::prefs::kDailyHttpOriginalContentLength,
 372       data_reduction_proxy::prefs::kDailyHttpReceivedContentLength,
 373       prefs);
 374   total.UpdateForDataChange(days_since_last_update);
 375
 376   DailyDataSavingUpdate proxy_enabled(
 377       data_reduction_proxy::prefs::
 378           kDailyOriginalContentLengthWithDataReductionProxyEnabled,
 379       data_reduction_proxy::prefs::
 380           kDailyContentLengthWithDataReductionProxyEnabled,
 381       prefs);
 382   proxy_enabled.UpdateForDataChange(days_since_last_update);
 383
 384   DailyDataSavingUpdate via_proxy(
 385       data_reduction_proxy::prefs::
 386           kDailyOriginalContentLengthViaDataReductionProxy,
 387       data_reduction_proxy::prefs::
 388           kDailyContentLengthViaDataReductionProxy,
 389       prefs);
 390   via_proxy.UpdateForDataChange(days_since_last_update);
 391
 392   DailyContentLengthUpdate https(
 393       data_reduction_proxy::prefs::
 394           kDailyContentLengthHttpsWithDataReductionProxyEnabled,
 395       prefs);
 396   https.UpdateForDataChange(days_since_last_update);
 397
 398   DailyContentLengthUpdate short_bypass(
 399       data_reduction_proxy::prefs::
 400           kDailyContentLengthShortBypassWithDataReductionProxyEnabled,
 401       prefs);
 402   short_bypass.UpdateForDataChange(days_since_last_update);
 403
 404   DailyContentLengthUpdate long_bypass(
 405       data_reduction_proxy::prefs::
 406           kDailyContentLengthLongBypassWithDataReductionProxyEnabled,
 407       prefs);
 408   long_bypass.UpdateForDataChange(days_since_last_update);
 409
 410   DailyContentLengthUpdate unknown(
 411       data_reduction_proxy::prefs::
 412           kDailyContentLengthUnknownWithDataReductionProxyEnabled,
 413       prefs);
 414   unknown.UpdateForDataChange(days_since_last_update);
 415
 416   total.Add(original_content_length, received_content_length);
 417   if (with_data_reduction_proxy_enabled) {
 418     proxy_enabled.Add(original_content_length, received_content_length);
 419     // Ignore data source cases, if exist, when
 420     // "with_data_reduction_proxy_enabled == false"
 421     switch (request_type) {
 422       case VIA_DATA_REDUCTION_PROXY:
 423         via_proxy.Add(original_content_length, received_content_length);
 424         break;
 425       case HTTPS:
 426         https.Add(received_content_length);
 427         break;
 428       case SHORT_BYPASS:
 429         short_bypass.Add(received_content_length);
 430         break;
 431       case LONG_BYPASS:
 432         long_bypass.Add(received_content_length);
 433         break;
 434       case UNKNOWN_TYPE:
 435         unknown.Add(received_content_length);
 436         break;
 437     }
 438   }
 439
 440   if (days_since_last_update) {
 441     // Record the last update time in microseconds in UTC.
 442     prefs->SetInt64(
 443         data_reduction_proxy::prefs::kDailyHttpContentLengthLastUpdateDate,
 444         midnight.ToInternalValue());
 445
 446     // A new day. Report the previous day's data if exists. We'll lose usage
 447     // data if the last time Chrome was run was more than a day ago.
 448     // Here, we prefer collecting less data but the collected data is
 449     // associated with an accurate date.
 450     if (days_since_last_update == 1) {
 451       // The previous day's data point is the second one from the tail.
 452       // Therefore (kNumDaysInHistory - 2) below.
 453       RecordDailyContentLengthHistograms(
 454           total.GetOriginalListPrefValue(kNumDaysInHistory - 2),
 455           total.GetReceivedListPrefValue(kNumDaysInHistory - 2),
 456           proxy_enabled.GetOriginalListPrefValue(kNumDaysInHistory - 2),
 457           proxy_enabled.GetReceivedListPrefValue(kNumDaysInHistory - 2),
 458           via_proxy.GetOriginalListPrefValue(kNumDaysInHistory - 2),
 459           via_proxy.GetReceivedListPrefValue(kNumDaysInHistory - 2),
 460           https.GetListPrefValue(kNumDaysInHistory - 2),
 461           short_bypass.GetListPrefValue(kNumDaysInHistory - 2),
 462           long_bypass.GetListPrefValue(kNumDaysInHistory - 2),
 463           unknown.GetListPrefValue(kNumDaysInHistory - 2));
 464     }
 465   }
 466 }
 467
 468 void UpdateContentLengthPrefs(int received_content_length,
 469                               int original_content_length,
 470                               PrefService* profile_prefs,
 471                               DataReductionProxyRequestType request_type,
 472                               DataReductionProxyStatisticsPrefs* prefs) {
 473   int64 total_received = prefs->GetInt64(
 474       data_reduction_proxy::prefs::kHttpReceivedContentLength);
 475   int64 total_original = prefs->GetInt64(
 476       data_reduction_proxy::prefs::kHttpOriginalContentLength);
 477   total_received += received_content_length;
 478   total_original += original_content_length;
 479   prefs->SetInt64(
 480       data_reduction_proxy::prefs::kHttpReceivedContentLength,
 481       total_received);
 482   prefs->SetInt64(
 483       data_reduction_proxy::prefs::kHttpOriginalContentLength,
 484       total_original);
 485
 486   bool with_data_reduction_proxy_enabled =
 487       profile_prefs->GetBoolean(
 488           data_reduction_proxy::prefs::kDataReductionProxyEnabled);
 489   UpdateContentLengthPrefsForDataReductionProxy(
 490       received_content_length,
 491       original_content_length,
 492       with_data_reduction_proxy_enabled,
 493       request_type,
 494       base::Time::Now(),
 495       prefs);
 496 }
 497
 498 }  // namespace data_reduction_proxy