chrome/browser/chromeos/contacts/gdata_contacts_service.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/chromeos/contacts/gdata_contacts_service.h"
   6
   7 #include <cstring>
   8 #include <map>
   9 #include <string>
  10 #include <utility>
  11
  12 #include "base/json/json_value_converter.h"
  13 #include "base/json/json_writer.h"
  14 #include "base/logging.h"
  15 #include "base/memory/weak_ptr.h"
  16 #include "base/metrics/histogram.h"
  17 #include "base/stl_util.h"
  18 #include "base/string_util.h"
  19 #include "base/time.h"
  20 #include "base/timer.h"
  21 #include "base/values.h"
  22 #include "chrome/browser/chromeos/contacts/contact.pb.h"
  23 #include "chrome/browser/google_apis/gdata_errorcode.h"
  24 #include "chrome/browser/google_apis/gdata_operations.h"
  25 #include "chrome/browser/google_apis/operation_runner.h"
  26 #include "chrome/browser/google_apis/time_util.h"
  27 #include "content/public/browser/browser_thread.h"
  28
  29 using content::BrowserThread;
  30
  31 namespace contacts {
  32
  33 namespace {
  34
  35 // Download outcomes reported via the "Contacts.FullUpdateResult" and
  36 // "Contacts.IncrementalUpdateResult" histograms.
  37 enum HistogramResult {
  38   HISTOGRAM_RESULT_SUCCESS = 0,
  39   HISTOGRAM_RESULT_GROUPS_DOWNLOAD_FAILURE = 1,
  40   HISTOGRAM_RESULT_GROUPS_PARSE_FAILURE = 2,
  41   HISTOGRAM_RESULT_MY_CONTACTS_GROUP_NOT_FOUND = 3,
  42   HISTOGRAM_RESULT_CONTACTS_DOWNLOAD_FAILURE = 4,
  43   HISTOGRAM_RESULT_CONTACTS_PARSE_FAILURE = 5,
  44   HISTOGRAM_RESULT_PHOTO_DOWNLOAD_FAILURE = 6,
  45   HISTOGRAM_RESULT_MAX_VALUE = 7,
  46 };
  47
  48 // Maximum number of profile photos that we'll download per second.
  49 // At values above 10, Google starts returning 503 errors.
  50 const int kMaxPhotoDownloadsPerSecond = 10;
  51
  52 // Give up after seeing more than this many transient errors while trying to
  53 // download a photo for a single contact.
  54 const int kMaxTransientPhotoDownloadErrorsPerContact = 2;
  55
  56 // Hardcoded system group ID for the "My Contacts" group, per
  57 // https://developers.google.com/google-apps/contacts/v3/#contact_group_entry.
  58 const char kMyContactsSystemGroupId[] = "Contacts";
  59
  60 // Top-level field in a contact groups feed containing the list of entries.
  61 const char kGroupEntryField[] = "feed.entry";
  62
  63 // Field in group entries containing the system group ID (e.g. ID "Contacts"
  64 // for the "My Contacts" system group).  See
  65 // https://developers.google.com/google-apps/contacts/v3/#contact_group_entry
  66 // for more details.
  67 const char kSystemGroupIdField[] = "gContact$systemGroup.id";
  68
  69 // Field in the top-level object containing the contacts feed.
  70 const char kFeedField[] = "feed";
  71
  72 // Field in the contacts feed containing a list of category information, along
  73 // with fields within the dictionaries contained in the list and expected
  74 // values.
  75 const char kCategoryField[] = "category";
  76 const char kCategorySchemeField[] = "scheme";
  77 const char kCategorySchemeValue[] = "http://schemas.google.com/g/2005#kind";
  78 const char kCategoryTermField[] = "term";
  79 const char kCategoryTermValue[] =
  80     "http://schemas.google.com/contact/2008#contact";
  81
  82 // Field in the contacts feed containing a list of contact entries.
  83 const char kEntryField[] = "entry";
  84
  85 // Field in group and contact entries containing the item's ID.
  86 const char kIdField[] = "id.$t";
  87
  88 // Top-level fields in contact entries.
  89 const char kDeletedField[] = "gd$deleted";
  90 const char kFullNameField[] = "gd$name.gd$fullName.$t";
  91 const char kGivenNameField[] = "gd$name.gd$givenName.$t";
  92 const char kAdditionalNameField[] = "gd$name.gd$additionalName.$t";
  93 const char kFamilyNameField[] = "gd$name.gd$familyName.$t";
  94 const char kNamePrefixField[] = "gd$name.gd$namePrefix.$t";
  95 const char kNameSuffixField[] = "gd$name.gd$nameSuffix.$t";
  96 const char kEmailField[] = "gd$email";
  97 const char kPhoneField[] = "gd$phoneNumber";
  98 const char kPostalAddressField[] = "gd$structuredPostalAddress";
  99 const char kInstantMessagingField[] = "gd$im";
 100 const char kLinkField[] = "link";
 101 const char kUpdatedField[] = "updated.$t";
 102
 103 // Fields in entries in the |kEmailField| list.
 104 const char kEmailAddressField[] = "address";
 105
 106 // Fields in entries in the |kPhoneField| list.
 107 const char kPhoneNumberField[] = "$t";
 108
 109 // Fields in entries in the |kPostalAddressField| list.
 110 const char kPostalAddressFormattedField[] = "gd$formattedAddress.$t";
 111
 112 // Fields in entries in the |kInstantMessagingField| list.
 113 const char kInstantMessagingAddressField[] = "address";
 114 const char kInstantMessagingProtocolField[] = "protocol";
 115 const char kInstantMessagingProtocolAimValue[] =
 116     "http://schemas.google.com/g/2005#AIM";
 117 const char kInstantMessagingProtocolMsnValue[] =
 118     "http://schemas.google.com/g/2005#MSN";
 119 const char kInstantMessagingProtocolYahooValue[] =
 120     "http://schemas.google.com/g/2005#YAHOO";
 121 const char kInstantMessagingProtocolSkypeValue[] =
 122     "http://schemas.google.com/g/2005#SKYPE";
 123 const char kInstantMessagingProtocolQqValue[] =
 124     "http://schemas.google.com/g/2005#QQ";
 125 const char kInstantMessagingProtocolGoogleTalkValue[] =
 126     "http://schemas.google.com/g/2005#GOOGLE_TALK";
 127 const char kInstantMessagingProtocolIcqValue[] =
 128     "http://schemas.google.com/g/2005#ICQ";
 129 const char kInstantMessagingProtocolJabberValue[] =
 130     "http://schemas.google.com/g/2005#JABBER";
 131
 132 // Generic fields shared between address-like items (email, postal, etc.).
 133 const char kAddressPrimaryField[] = "primary";
 134 const char kAddressPrimaryTrueValue[] = "true";
 135 const char kAddressRelField[] = "rel";
 136 const char kAddressRelHomeValue[] = "http://schemas.google.com/g/2005#home";
 137 const char kAddressRelWorkValue[] = "http://schemas.google.com/g/2005#work";
 138 const char kAddressRelMobileValue[] = "http://schemas.google.com/g/2005#mobile";
 139 const char kAddressLabelField[] = "label";
 140
 141 // Fields in entries in the |kLinkField| list.
 142 const char kLinkHrefField[] = "href";
 143 const char kLinkRelField[] = "rel";
 144 const char kLinkETagField[] = "gd$etag";
 145 const char kLinkRelPhotoValue[] =
 146     "http://schemas.google.com/contacts/2008/rel#photo";
 147
 148 // OAuth2 scope for the Contacts API.
 149 const char kContactsScope[] = "https://www.google.com/m8/feeds/";
 150
 151 // Returns a string containing a pretty-printed JSON representation of |value|.
 152 std::string PrettyPrintValue(const base::Value& value) {
 153   std::string out;
 154   base::JSONWriter::WriteWithOptions(
 155       &value, base::JSONWriter::OPTIONS_PRETTY_PRINT, &out);
 156   return out;
 157 }
 158
 159 // Assigns the value at |path| within |dict| to |out|, returning false if the
 160 // path wasn't present.  Unicode byte order marks are removed from the string.
 161 bool GetCleanedString(const DictionaryValue& dict,
 162                       const std::string& path,
 163                       std::string* out) {
 164   if (!dict.GetString(path, out))
 165     return false;
 166
 167   // The Unicode byte order mark, U+FEFF, is useless in UTF-8 strings (which are
 168   // interpreted one byte at a time).
 169   ReplaceSubstringsAfterOffset(out, 0, "\xEF\xBB\xBF", "");
 170   return true;
 171 }
 172
 173 // Returns whether an address is primary, given a dictionary representing a
 174 // single address.
 175 bool IsAddressPrimary(const DictionaryValue& address_dict) {
 176   std::string primary;
 177   address_dict.GetString(kAddressPrimaryField, &primary);
 178   return primary == kAddressPrimaryTrueValue;
 179 }
 180
 181 // Initializes an AddressType message given a dictionary representing a single
 182 // address.
 183 void InitAddressType(const DictionaryValue& address_dict,
 184                      Contact_AddressType* type) {
 185   DCHECK(type);
 186   type->Clear();
 187
 188   std::string rel;
 189   address_dict.GetString(kAddressRelField, &rel);
 190   if (rel == kAddressRelHomeValue)
 191     type->set_relation(Contact_AddressType_Relation_HOME);
 192   else if (rel == kAddressRelWorkValue)
 193     type->set_relation(Contact_AddressType_Relation_WORK);
 194   else if (rel == kAddressRelMobileValue)
 195     type->set_relation(Contact_AddressType_Relation_MOBILE);
 196   else
 197     type->set_relation(Contact_AddressType_Relation_OTHER);
 198
 199   GetCleanedString(address_dict, kAddressLabelField, type->mutable_label());
 200 }
 201
 202 // Maps the protocol from a dictionary representing a contact's IM address to a
 203 // contacts::Contact_InstantMessagingAddress_Protocol value.
 204 contacts::Contact_InstantMessagingAddress_Protocol
 205 GetInstantMessagingProtocol(const DictionaryValue& im_dict) {
 206   std::string protocol;
 207   im_dict.GetString(kInstantMessagingProtocolField, &protocol);
 208   if (protocol == kInstantMessagingProtocolAimValue)
 209     return contacts::Contact_InstantMessagingAddress_Protocol_AIM;
 210   else if (protocol == kInstantMessagingProtocolMsnValue)
 211     return contacts::Contact_InstantMessagingAddress_Protocol_MSN;
 212   else if (protocol == kInstantMessagingProtocolYahooValue)
 213     return contacts::Contact_InstantMessagingAddress_Protocol_YAHOO;
 214   else if (protocol == kInstantMessagingProtocolSkypeValue)
 215     return contacts::Contact_InstantMessagingAddress_Protocol_SKYPE;
 216   else if (protocol == kInstantMessagingProtocolQqValue)
 217     return contacts::Contact_InstantMessagingAddress_Protocol_QQ;
 218   else if (protocol == kInstantMessagingProtocolGoogleTalkValue)
 219     return contacts::Contact_InstantMessagingAddress_Protocol_GOOGLE_TALK;
 220   else if (protocol == kInstantMessagingProtocolIcqValue)
 221     return contacts::Contact_InstantMessagingAddress_Protocol_ICQ;
 222   else if (protocol == kInstantMessagingProtocolJabberValue)
 223     return contacts::Contact_InstantMessagingAddress_Protocol_JABBER;
 224   else
 225     return contacts::Contact_InstantMessagingAddress_Protocol_OTHER;
 226 }
 227
 228 // Gets the photo URL from a contact's dictionary (within the "entry" list).
 229 // Returns an empty string if no photo was found.
 230 std::string GetPhotoUrl(const DictionaryValue& dict) {
 231   const ListValue* link_list = NULL;
 232   if (!dict.GetList(kLinkField, &link_list))
 233     return std::string();
 234
 235   for (size_t i = 0; i < link_list->GetSize(); ++i) {
 236     const DictionaryValue* link_dict = NULL;
 237     if (!link_list->GetDictionary(i, &link_dict))
 238       continue;
 239
 240     std::string rel;
 241     if (!link_dict->GetString(kLinkRelField, &rel))
 242       continue;
 243     if (rel != kLinkRelPhotoValue)
 244       continue;
 245
 246     // From https://goo.gl/7T6Od: "If a contact does not have a photo, then the
 247     // photo link element has no gd:etag attribute."
 248     std::string etag;
 249     if (!link_dict->GetString(kLinkETagField, &etag))
 250       continue;
 251
 252     std::string url;
 253     if (link_dict->GetString(kLinkHrefField, &url))
 254       return url;
 255   }
 256   return std::string();
 257 }
 258
 259 // Fills a Contact's fields using an entry from a GData feed.
 260 bool FillContactFromDictionary(const base::DictionaryValue& dict,
 261                                contacts::Contact* contact) {
 262   DCHECK(contact);
 263   contact->Clear();
 264
 265   if (!dict.GetString(kIdField, contact->mutable_contact_id()))
 266     return false;
 267
 268   std::string updated;
 269   if (dict.GetString(kUpdatedField, &updated)) {
 270     base::Time update_time;
 271     if (!google_apis::util::GetTimeFromString(updated, &update_time)) {
 272       LOG(WARNING) << "Unable to parse time \"" << updated << "\"";
 273       return false;
 274     }
 275     contact->set_update_time(update_time.ToInternalValue());
 276   }
 277
 278   const base::Value* deleted_value = NULL;
 279   contact->set_deleted(dict.Get(kDeletedField, &deleted_value));
 280   if (contact->deleted())
 281     return true;
 282
 283   GetCleanedString(dict, kFullNameField, contact->mutable_full_name());
 284   GetCleanedString(dict, kGivenNameField, contact->mutable_given_name());
 285   GetCleanedString(
 286       dict, kAdditionalNameField, contact->mutable_additional_name());
 287   GetCleanedString(dict, kFamilyNameField, contact->mutable_family_name());
 288   GetCleanedString(dict, kNamePrefixField, contact->mutable_name_prefix());
 289   GetCleanedString(dict, kNameSuffixField, contact->mutable_name_suffix());
 290
 291   const ListValue* email_list = NULL;
 292   if (dict.GetList(kEmailField, &email_list)) {
 293     for (size_t i = 0; i < email_list->GetSize(); ++i) {
 294       const DictionaryValue* email_dict = NULL;
 295       if (!email_list->GetDictionary(i, &email_dict))
 296         return false;
 297
 298       contacts::Contact_EmailAddress* email = contact->add_email_addresses();
 299       if (!GetCleanedString(*email_dict,
 300                             kEmailAddressField,
 301                             email->mutable_address())) {
 302         return false;
 303       }
 304       email->set_primary(IsAddressPrimary(*email_dict));
 305       InitAddressType(*email_dict, email->mutable_type());
 306     }
 307   }
 308
 309   const ListValue* phone_list = NULL;
 310   if (dict.GetList(kPhoneField, &phone_list)) {
 311     for (size_t i = 0; i < phone_list->GetSize(); ++i) {
 312       const DictionaryValue* phone_dict = NULL;
 313       if (!phone_list->GetDictionary(i, &phone_dict))
 314         return false;
 315
 316       contacts::Contact_PhoneNumber* phone = contact->add_phone_numbers();
 317       if (!GetCleanedString(*phone_dict,
 318                             kPhoneNumberField,
 319                             phone->mutable_number())) {
 320         return false;
 321       }
 322       phone->set_primary(IsAddressPrimary(*phone_dict));
 323       InitAddressType(*phone_dict, phone->mutable_type());
 324     }
 325   }
 326
 327   const ListValue* address_list = NULL;
 328   if (dict.GetList(kPostalAddressField, &address_list)) {
 329     for (size_t i = 0; i < address_list->GetSize(); ++i) {
 330       const DictionaryValue* address_dict = NULL;
 331       if (!address_list->GetDictionary(i, &address_dict))
 332         return false;
 333
 334       contacts::Contact_PostalAddress* address =
 335           contact->add_postal_addresses();
 336       if (!GetCleanedString(*address_dict,
 337                             kPostalAddressFormattedField,
 338                             address->mutable_address())) {
 339         return false;
 340       }
 341       address->set_primary(IsAddressPrimary(*address_dict));
 342       InitAddressType(*address_dict, address->mutable_type());
 343     }
 344   }
 345
 346   const ListValue* im_list = NULL;
 347   if (dict.GetList(kInstantMessagingField, &im_list)) {
 348     for (size_t i = 0; i < im_list->GetSize(); ++i) {
 349       const DictionaryValue* im_dict = NULL;
 350       if (!im_list->GetDictionary(i, &im_dict))
 351         return false;
 352
 353       contacts::Contact_InstantMessagingAddress* im =
 354           contact->add_instant_messaging_addresses();
 355       if (!GetCleanedString(*im_dict,
 356                             kInstantMessagingAddressField,
 357                             im->mutable_address())) {
 358         return false;
 359       }
 360       im->set_primary(IsAddressPrimary(*im_dict));
 361       InitAddressType(*im_dict, im->mutable_type());
 362       im->set_protocol(GetInstantMessagingProtocol(*im_dict));
 363     }
 364   }
 365
 366   return true;
 367 }
 368
 369 // Structure into which we parse the contact groups feed using
 370 // JSONValueConverter.
 371 struct ContactGroups {
 372   struct ContactGroup {
 373     // Group ID, e.g.
 374     // "http://www.google.com/m8/feeds/groups/user%40gmail.com/base/6".
 375     std::string group_id;
 376
 377     // System group ID (e.g. "Contacts" for the "My Contacts" system group) if
 378     // this is a system group, and empty otherwise.  See http://goo.gl/oWVnN
 379     // for more details.
 380     std::string system_group_id;
 381   };
 382
 383   // Given a system group ID, returns the corresponding group ID or an empty
 384   // string if the requested system group wasn't present.
 385   std::string GetGroupIdForSystemGroup(const std::string& system_group_id) {
 386     for (size_t i = 0; i < groups.size(); ++i) {
 387       const ContactGroup& group = *groups[i];
 388       if (group.system_group_id == system_group_id)
 389         return group.group_id;
 390     }
 391     return std::string();
 392   }
 393
 394   // Given |value| corresponding to a dictionary in a contact group feed's
 395   // "entry" list, fills |result| with information about the group.
 396   static bool GetContactGroup(const base::Value* value, ContactGroup* result) {
 397     DCHECK(value);
 398     DCHECK(result);
 399     const base::DictionaryValue* dict = NULL;
 400     if (!value->GetAsDictionary(&dict))
 401       return false;
 402
 403     dict->GetString(kIdField, &result->group_id);
 404     dict->GetString(kSystemGroupIdField, &result->system_group_id);
 405     return true;
 406   }
 407
 408   static void RegisterJSONConverter(
 409       base::JSONValueConverter<ContactGroups>* converter) {
 410     DCHECK(converter);
 411     converter->RegisterRepeatedCustomValue<ContactGroup>(
 412         kGroupEntryField, &ContactGroups::groups, &GetContactGroup);
 413   }
 414
 415   ScopedVector<ContactGroup> groups;
 416 };
 417
 418 }  // namespace
 419
 420 // This class handles a single request to download all of a user's contacts.
 421 //
 422 // First, the feed containing the user's contact groups is downloaded via
 423 // GetContactGroupsOperation and examined to find the ID for the "My Contacts"
 424 // group (by default, the contacts API also returns suggested contacts).  The
 425 // group ID is cached in GDataContactsService so that this step can be skipped
 426 // by later DownloadContactRequests.
 427 //
 428 // Next, the contacts feed is downloaded via GetContactsOperation and parsed.
 429 // Individual contacts::Contact objects are created using the data from the
 430 // feed.
 431 //
 432 // Finally, GetContactPhotoOperations are created and used to start downloading
 433 // contacts' photos in parallel.  When all photos have been downloaded, the
 434 // contacts are passed to the passed-in callback.
 435 class GDataContactsService::DownloadContactsRequest {
 436  public:
 437   DownloadContactsRequest(GDataContactsService* service,
 438                           google_apis::OperationRunner* runner,
 439                           SuccessCallback success_callback,
 440                           FailureCallback failure_callback,
 441                           const base::Time& min_update_time)
 442       : service_(service),
 443         runner_(runner),
 444         success_callback_(success_callback),
 445         failure_callback_(failure_callback),
 446         min_update_time_(min_update_time),
 447         contacts_(new ScopedVector<contacts::Contact>),
 448         my_contacts_group_id_(service->cached_my_contacts_group_id_),
 449         num_in_progress_photo_downloads_(0),
 450         photo_download_failed_(false),
 451         num_photo_download_404_errors_(0),
 452         total_photo_bytes_(0),
 453         ALLOW_THIS_IN_INITIALIZER_LIST(weak_ptr_factory_(this)) {
 454     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 455     DCHECK(service_);
 456     DCHECK(runner_);
 457   }
 458
 459   ~DownloadContactsRequest() {
 460     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 461     service_ = NULL;
 462     runner_ = NULL;
 463   }
 464
 465   const std::string my_contacts_group_id() const {
 466     return my_contacts_group_id_;
 467   }
 468
 469   // Begins the contacts-downloading process.  If the ID for the "My Contacts"
 470   // group has previously been cached, then the contacts download is started.
 471   // Otherwise, the contact groups download is started.
 472   void Run() {
 473     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 474     download_start_time_ = base::TimeTicks::Now();
 475     if (!my_contacts_group_id_.empty()) {
 476       StartContactsDownload();
 477     } else {
 478       google_apis::GetContactGroupsOperation* operation =
 479           new google_apis::GetContactGroupsOperation(
 480               runner_->operation_registry(),
 481               base::Bind(&DownloadContactsRequest::HandleGroupsFeedData,
 482                          weak_ptr_factory_.GetWeakPtr()));
 483       if (!service_->groups_feed_url_for_testing_.is_empty()) {
 484         operation->set_feed_url_for_testing(
 485             service_->groups_feed_url_for_testing_);
 486       }
 487       runner_->StartOperationWithRetry(operation);
 488     }
 489   }
 490
 491  private:
 492   // Invokes the failure callback and notifies GDataContactsService that the
 493   // request is done.
 494   void ReportFailure(HistogramResult histogram_result) {
 495     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 496     SendHistograms(histogram_result);
 497     failure_callback_.Run();
 498     service_->OnRequestComplete(this);
 499   }
 500
 501   // Reports UMA stats after the request has completed.
 502   void SendHistograms(HistogramResult result) {
 503     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 504     DCHECK_GE(result, 0);
 505     DCHECK_LT(result, HISTOGRAM_RESULT_MAX_VALUE);
 506
 507     bool success = (result == HISTOGRAM_RESULT_SUCCESS);
 508     base::TimeDelta elapsed_time =
 509         base::TimeTicks::Now() - download_start_time_;
 510     int photo_error_percent = static_cast<int>(
 511         100.0 * transient_photo_download_errors_per_contact_.size() /
 512         contact_photo_urls_.size() + 0.5);
 513
 514     if (min_update_time_.is_null()) {
 515       UMA_HISTOGRAM_ENUMERATION("Contacts.FullUpdateResult",
 516                                 result, HISTOGRAM_RESULT_MAX_VALUE);
 517       if (success) {
 518         UMA_HISTOGRAM_MEDIUM_TIMES("Contacts.FullUpdateDuration",
 519                                    elapsed_time);
 520         UMA_HISTOGRAM_COUNTS_10000("Contacts.FullUpdateContacts",
 521                                    contacts_->size());
 522         UMA_HISTOGRAM_COUNTS_10000("Contacts.FullUpdatePhotos",
 523                                    contact_photo_urls_.size());
 524         UMA_HISTOGRAM_MEMORY_KB("Contacts.FullUpdatePhotoBytes",
 525                                 total_photo_bytes_);
 526         UMA_HISTOGRAM_COUNTS_10000("Contacts.FullUpdatePhoto404Errors",
 527                                    num_photo_download_404_errors_);
 528         UMA_HISTOGRAM_PERCENTAGE("Contacts.FullUpdatePhotoErrorPercent",
 529                                  photo_error_percent);
 530       }
 531     } else {
 532       UMA_HISTOGRAM_ENUMERATION("Contacts.IncrementalUpdateResult",
 533                                 result, HISTOGRAM_RESULT_MAX_VALUE);
 534       if (success) {
 535         UMA_HISTOGRAM_MEDIUM_TIMES("Contacts.IncrementalUpdateDuration",
 536                                    elapsed_time);
 537         UMA_HISTOGRAM_COUNTS_10000("Contacts.IncrementalUpdateContacts",
 538                                    contacts_->size());
 539         UMA_HISTOGRAM_COUNTS_10000("Contacts.IncrementalUpdatePhotos",
 540                                    contact_photo_urls_.size());
 541         UMA_HISTOGRAM_MEMORY_KB("Contacts.IncrementalUpdatePhotoBytes",
 542                                 total_photo_bytes_);
 543         UMA_HISTOGRAM_COUNTS_10000("Contacts.IncrementalUpdatePhoto404Errors",
 544                                    num_photo_download_404_errors_);
 545         UMA_HISTOGRAM_PERCENTAGE("Contacts.IncrementalUpdatePhotoErrorPercent",
 546                                  photo_error_percent);
 547       }
 548     }
 549   }
 550
 551   // Callback for GetContactGroupsOperation calls.  Starts downloading the
 552   // actual contacts after finding the "My Contacts" group ID.
 553   void HandleGroupsFeedData(google_apis::GDataErrorCode error,
 554                             scoped_ptr<base::Value> feed_data) {
 555     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 556     if (error != google_apis::HTTP_SUCCESS) {
 557       LOG(WARNING) << "Got error " << error << " while downloading groups";
 558       ReportFailure(HISTOGRAM_RESULT_GROUPS_DOWNLOAD_FAILURE);
 559       return;
 560     }
 561
 562     VLOG(2) << "Got groups feed data:\n"
 563             << PrettyPrintValue(*(feed_data.get()));
 564     ContactGroups groups;
 565     base::JSONValueConverter<ContactGroups> converter;
 566     if (!converter.Convert(*feed_data, &groups)) {
 567       LOG(WARNING) << "Unable to parse groups feed";
 568       ReportFailure(HISTOGRAM_RESULT_GROUPS_PARSE_FAILURE);
 569       return;
 570     }
 571
 572     my_contacts_group_id_ =
 573         groups.GetGroupIdForSystemGroup(kMyContactsSystemGroupId);
 574     if (!my_contacts_group_id_.empty()) {
 575       StartContactsDownload();
 576     } else {
 577       LOG(WARNING) << "Unable to find ID for \"My Contacts\" group";
 578       ReportFailure(HISTOGRAM_RESULT_MY_CONTACTS_GROUP_NOT_FOUND);
 579     }
 580   }
 581
 582   // Starts a download of the contacts from the "My Contacts" group.
 583   void StartContactsDownload() {
 584     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 585     google_apis::GetContactsOperation* operation =
 586         new google_apis::GetContactsOperation(
 587             runner_->operation_registry(),
 588             my_contacts_group_id_,
 589             min_update_time_,
 590             base::Bind(&DownloadContactsRequest::HandleContactsFeedData,
 591                        weak_ptr_factory_.GetWeakPtr()));
 592     if (!service_->contacts_feed_url_for_testing_.is_empty()) {
 593       operation->set_feed_url_for_testing(
 594           service_->contacts_feed_url_for_testing_);
 595     }
 596     runner_->StartOperationWithRetry(operation);
 597   }
 598
 599   // Callback for GetContactsOperation calls.
 600   void HandleContactsFeedData(google_apis::GDataErrorCode error,
 601                               scoped_ptr<base::Value> feed_data) {
 602     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 603     if (error != google_apis::HTTP_SUCCESS) {
 604       LOG(WARNING) << "Got error " << error << " while downloading contacts";
 605       ReportFailure(HISTOGRAM_RESULT_CONTACTS_DOWNLOAD_FAILURE);
 606       return;
 607     }
 608
 609     VLOG(2) << "Got contacts feed data:\n"
 610             << PrettyPrintValue(*(feed_data.get()));
 611     if (!ProcessContactsFeedData(*feed_data.get())) {
 612       LOG(WARNING) << "Unable to process contacts feed data";
 613       ReportFailure(HISTOGRAM_RESULT_CONTACTS_PARSE_FAILURE);
 614       return;
 615     }
 616
 617     StartPhotoDownloads();
 618     photo_download_timer_.Start(
 619         FROM_HERE, service_->photo_download_timer_interval_,
 620         this, &DownloadContactsRequest::StartPhotoDownloads);
 621     CheckCompletion();
 622   }
 623
 624   // Processes the raw contacts feed from |feed_data| and fills |contacts_|.
 625   // Returns true on success.
 626   bool ProcessContactsFeedData(const base::Value& feed_data) {
 627     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 628     const DictionaryValue* toplevel_dict = NULL;
 629     if (!feed_data.GetAsDictionary(&toplevel_dict)) {
 630       LOG(WARNING) << "Top-level object is not a dictionary";
 631       return false;
 632     }
 633
 634     const DictionaryValue* feed_dict = NULL;
 635     if (!toplevel_dict->GetDictionary(kFeedField, &feed_dict)) {
 636       LOG(WARNING) << "Feed dictionary missing";
 637       return false;
 638     }
 639
 640     // Check the category field to confirm that this is actually a contact feed.
 641     const ListValue* category_list = NULL;
 642     if (!feed_dict->GetList(kCategoryField, &category_list)) {
 643       LOG(WARNING) << "Category list missing";
 644       return false;
 645     }
 646     const DictionaryValue* category_dict = NULL;
 647     if (!category_list->GetSize() == 1 ||
 648         !category_list->GetDictionary(0, &category_dict)) {
 649       LOG(WARNING) << "Unable to get dictionary from category list of size "
 650                    << category_list->GetSize();
 651       return false;
 652     }
 653     std::string category_scheme, category_term;
 654     if (!category_dict->GetString(kCategorySchemeField, &category_scheme) ||
 655         !category_dict->GetString(kCategoryTermField, &category_term) ||
 656         category_scheme != kCategorySchemeValue ||
 657         category_term != kCategoryTermValue) {
 658       LOG(WARNING) << "Unexpected category (scheme was \"" << category_scheme
 659                    << "\", term was \"" << category_term << "\")";
 660       return false;
 661     }
 662
 663     // A missing entry list means no entries (maybe we're doing an incremental
 664     // update and nothing has changed).
 665     const ListValue* entry_list = NULL;
 666     if (!feed_dict->GetList(kEntryField, &entry_list))
 667       return true;
 668
 669     contacts_needing_photo_downloads_.reserve(entry_list->GetSize());
 670
 671     for (ListValue::const_iterator entry_it = entry_list->begin();
 672          entry_it != entry_list->end(); ++entry_it) {
 673       const size_t index = (entry_it - entry_list->begin());
 674       const DictionaryValue* contact_dict = NULL;
 675       if (!(*entry_it)->GetAsDictionary(&contact_dict)) {
 676         LOG(WARNING) << "Entry " << index << " isn't a dictionary";
 677         return false;
 678       }
 679
 680       scoped_ptr<contacts::Contact> contact(new contacts::Contact);
 681       if (!FillContactFromDictionary(*contact_dict, contact.get())) {
 682         LOG(WARNING) << "Unable to fill entry " << index;
 683         return false;
 684       }
 685
 686       VLOG(1) << "Got contact " << index << ":"
 687               << " id=" << contact->contact_id()
 688               << " full_name=\"" << contact->full_name() << "\""
 689               << " update_time=" << contact->update_time();
 690
 691       std::string photo_url = GetPhotoUrl(*contact_dict);
 692       if (!photo_url.empty()) {
 693         if (!service_->rewrite_photo_url_callback_for_testing_.is_null()) {
 694           photo_url =
 695               service_->rewrite_photo_url_callback_for_testing_.Run(photo_url);
 696         }
 697         contact_photo_urls_[contact.get()] = photo_url;
 698         contacts_needing_photo_downloads_.push_back(contact.get());
 699       }
 700
 701       contacts_->push_back(contact.release());
 702     }
 703
 704     return true;
 705   }
 706
 707   // If we're done downloading photos, invokes a callback and deletes |this|.
 708   // Otherwise, starts one or more downloads of URLs from
 709   // |contacts_needing_photo_downloads_|.
 710   void CheckCompletion() {
 711     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 712     if (contacts_needing_photo_downloads_.empty() &&
 713         num_in_progress_photo_downloads_ == 0) {
 714       VLOG(1) << "Done downloading photos; invoking callback";
 715       photo_download_timer_.Stop();
 716       if (photo_download_failed_) {
 717         ReportFailure(HISTOGRAM_RESULT_PHOTO_DOWNLOAD_FAILURE );
 718       } else {
 719         SendHistograms(HISTOGRAM_RESULT_SUCCESS);
 720         success_callback_.Run(contacts_.Pass());
 721         service_->OnRequestComplete(this);
 722       }
 723       return;
 724     }
 725   }
 726
 727   // Starts photo downloads for contacts in |contacts_needing_photo_downloads_|.
 728   // Should be invoked only once per second.
 729   void StartPhotoDownloads() {
 730     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 731     while (!contacts_needing_photo_downloads_.empty() &&
 732            (num_in_progress_photo_downloads_ <
 733             service_->max_photo_downloads_per_second_)) {
 734       contacts::Contact* contact = contacts_needing_photo_downloads_.back();
 735       contacts_needing_photo_downloads_.pop_back();
 736       DCHECK(contact_photo_urls_.count(contact));
 737       std::string url = contact_photo_urls_[contact];
 738
 739       VLOG(1) << "Starting download of photo " << url << " for "
 740               << contact->contact_id();
 741       runner_->StartOperationWithRetry(
 742           new google_apis::GetContactPhotoOperation(
 743               runner_->operation_registry(),
 744               GURL(url),
 745               base::Bind(&DownloadContactsRequest::HandlePhotoData,
 746                          weak_ptr_factory_.GetWeakPtr(),
 747                          contact)));
 748       num_in_progress_photo_downloads_++;
 749     }
 750   }
 751
 752   // Callback for GetContactPhotoOperation calls.  Updates the associated
 753   // Contact and checks for completion.
 754   void HandlePhotoData(contacts::Contact* contact,
 755                        google_apis::GDataErrorCode error,
 756                        scoped_ptr<std::string> download_data) {
 757     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 758     VLOG(1) << "Got photo data for " << contact->contact_id()
 759             << " (error=" << error << " size=" << download_data->size() << ")";
 760     num_in_progress_photo_downloads_--;
 761
 762     if (error == google_apis::HTTP_INTERNAL_SERVER_ERROR ||
 763         error == google_apis::HTTP_SERVICE_UNAVAILABLE) {
 764       int num_errors = ++transient_photo_download_errors_per_contact_[contact];
 765       if (num_errors <= kMaxTransientPhotoDownloadErrorsPerContact) {
 766         LOG(WARNING) << "Got error " << error << " while downloading photo "
 767                      << "for " << contact->contact_id() << "; retrying";
 768         contacts_needing_photo_downloads_.push_back(contact);
 769         return;
 770       }
 771     }
 772
 773     if (error == google_apis::HTTP_NOT_FOUND) {
 774       LOG(WARNING) << "Got error " << error << " while downloading photo "
 775                    << "for " << contact->contact_id() << "; skipping";
 776       num_photo_download_404_errors_++;
 777       CheckCompletion();
 778       return;
 779     }
 780
 781     if (error != google_apis::HTTP_SUCCESS) {
 782       LOG(WARNING) << "Got error " << error << " while downloading photo "
 783                    << "for " << contact->contact_id() << "; giving up";
 784       photo_download_failed_ = true;
 785       // Make sure we don't start any more downloads.
 786       contacts_needing_photo_downloads_.clear();
 787       CheckCompletion();
 788       return;
 789     }
 790
 791     total_photo_bytes_ += download_data->size();
 792     contact->set_raw_untrusted_photo(*download_data);
 793     CheckCompletion();
 794   }
 795
 796   typedef std::map<contacts::Contact*, std::string> ContactPhotoUrls;
 797
 798   GDataContactsService* service_;  // not owned
 799   google_apis::OperationRunner* runner_;  // not owned
 800
 801   SuccessCallback success_callback_;
 802   FailureCallback failure_callback_;
 803
 804   base::Time min_update_time_;
 805
 806   scoped_ptr<ScopedVector<contacts::Contact> > contacts_;
 807
 808   // ID of the "My Contacts" contacts group.
 809   std::string my_contacts_group_id_;
 810
 811   // Map from a contact to the URL at which its photo is located.
 812   // Contacts without photos do not appear in this map.
 813   ContactPhotoUrls contact_photo_urls_;
 814
 815   // Invokes StartPhotoDownloads() once per second.
 816   base::RepeatingTimer<DownloadContactsRequest> photo_download_timer_;
 817
 818   // Contacts that have photos that we still need to start downloading.
 819   // When we start a download, the contact is removed from this list.
 820   std::vector<contacts::Contact*> contacts_needing_photo_downloads_;
 821
 822   // Number of in-progress photo downloads.
 823   int num_in_progress_photo_downloads_;
 824
 825   // Map from a contact to the number of transient errors that we've encountered
 826   // while trying to download its photo.  Contacts for which no errors have been
 827   // encountered aren't represented in the map.
 828   std::map<contacts::Contact*, int>
 829       transient_photo_download_errors_per_contact_;
 830
 831   // Did we encounter a fatal error while downloading a photo?
 832   bool photo_download_failed_;
 833
 834   // How many photos did we skip due to 404 errors?
 835   int num_photo_download_404_errors_;
 836
 837   // Total size of all photos that were downloaded.
 838   size_t total_photo_bytes_;
 839
 840   // Time at which Run() was called.
 841   base::TimeTicks download_start_time_;
 842
 843   // Note: This should remain the last member so it'll be destroyed and
 844   // invalidate its weak pointers before any other members are destroyed.
 845   base::WeakPtrFactory<DownloadContactsRequest> weak_ptr_factory_;
 846
 847   DISALLOW_COPY_AND_ASSIGN(DownloadContactsRequest);
 848 };
 849
 850 GDataContactsService::GDataContactsService(Profile* profile)
 851     : max_photo_downloads_per_second_(kMaxPhotoDownloadsPerSecond),
 852       photo_download_timer_interval_(base::TimeDelta::FromSeconds(1)) {
 853   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 854   std::vector<std::string> scopes;
 855   scopes.push_back(kContactsScope);
 856   runner_.reset(new google_apis::OperationRunner(profile, scopes));
 857 }
 858
 859 GDataContactsService::~GDataContactsService() {
 860   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 861   runner_->CancelAll();
 862   STLDeleteContainerPointers(requests_.begin(), requests_.end());
 863   requests_.clear();
 864 }
 865
 866 google_apis::AuthService* GDataContactsService::auth_service_for_testing() {
 867   return runner_->auth_service();
 868 }
 869
 870 void GDataContactsService::Initialize() {
 871   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 872   runner_->Initialize();
 873 }
 874
 875 void GDataContactsService::DownloadContacts(SuccessCallback success_callback,
 876                                             FailureCallback failure_callback,
 877                                             const base::Time& min_update_time) {
 878   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 879   DownloadContactsRequest* request =
 880       new DownloadContactsRequest(this,
 881                                   runner_.get(),
 882                                   success_callback,
 883                                   failure_callback,
 884                                   min_update_time);
 885   VLOG(1) << "Starting contacts download with request " << request;
 886   requests_.insert(request);
 887   request->Run();
 888 }
 889
 890 void GDataContactsService::OnRequestComplete(DownloadContactsRequest* request) {
 891   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 892   DCHECK(request);
 893   VLOG(1) << "Download request " << request << " complete";
 894   if (!request->my_contacts_group_id().empty())
 895     cached_my_contacts_group_id_ = request->my_contacts_group_id();
 896   requests_.erase(request);
 897   delete request;
 898 }
 899
 900 }  // namespace contacts