chrome/browser/spellchecker/feedback_sender.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // The |FeedbackSender| object stores the user feedback to spellcheck
   6 // suggestions in a |Feedback| object.
   7 //
   8 // When spelling service returns spellcheck results, these results first arrive
   9 // in |FeedbackSender| to assign hash identifiers for each
  10 // misspelling-suggestion pair. If the spelling service identifies the same
  11 // misspelling as already displayed to the user, then |FeedbackSender| reuses
  12 // the same hash identifiers to avoid duplication. It detects the duplicates by
  13 // comparing misspelling offsets in text. Spelling service can return duplicates
  14 // because we request spellcheck for whole paragraphs, as context around a
  15 // misspelled word is important to the spellcheck algorithm.
  16 //
  17 // All feedback is initially pending. When a user acts upon a misspelling such
  18 // that the misspelling is no longer displayed (red squiggly line goes away),
  19 // then the feedback for this misspelling is finalized. All finalized feedback
  20 // is erased after being sent to the spelling service. Pending feedback is kept
  21 // around for |kSessionHours| hours and then finalized even if user did not act
  22 // on the misspellings.
  23 //
  24 // |FeedbackSender| periodically requests a list of hashes of all remaining
  25 // misspellings in renderers. When a renderer responds with a list of hashes,
  26 // |FeedbackSender| uses the list to determine which misspellings are no longer
  27 // displayed to the user and sends the current state of user feedback to the
  28 // spelling service.
  29
  30 #include "chrome/browser/spellchecker/feedback_sender.h"
  31
  32 #include <algorithm>
  33 #include <iterator>
  34
  35 #include "base/command_line.h"
  36 #include "base/hash.h"
  37 #include "base/json/json_writer.h"
  38 #include "base/location.h"
  39 #include "base/metrics/field_trial.h"
  40 #include "base/single_thread_task_runner.h"
  41 #include "base/stl_util.h"
  42 #include "base/strings/string_number_conversions.h"
  43 #include "base/strings/stringprintf.h"
  44 #include "base/thread_task_runner_handle.h"
  45 #include "base/values.h"
  46 #include "chrome/browser/spellchecker/word_trimmer.h"
  47 #include "chrome/common/chrome_switches.h"
  48 #include "chrome/common/spellcheck_common.h"
  49 #include "chrome/common/spellcheck_marker.h"
  50 #include "chrome/common/spellcheck_messages.h"
  51 #include "components/data_use_measurement/core/data_use_user_data.h"
  52 #include "content/public/browser/render_process_host.h"
  53 #include "google_apis/google_api_keys.h"
  54 #include "net/base/load_flags.h"
  55 #include "net/url_request/url_fetcher.h"
  56 #include "net/url_request/url_request_context_getter.h"
  57
  58 namespace spellcheck {
  59
  60 namespace {
  61
  62 const size_t kMaxFeedbackSizeBytes = 10 * 1024 * 1024;  // 10 MB
  63
  64 // The default URL where feedback data is sent.
  65 const char kFeedbackServiceURL[] = "https://www.googleapis.com/rpc";
  66
  67 // The minimum number of seconds between sending batches of feedback.
  68 const int kMinIntervalSeconds = 5;
  69
  70 // Returns a hash of |session_start|, the current timestamp, and
  71 // |suggestion_index|.
  72 uint32 BuildHash(const base::Time& session_start, size_t suggestion_index) {
  73   return base::Hash(
  74       base::StringPrintf("%" PRId64 "%" PRId64 "%" PRIuS,
  75                          session_start.ToInternalValue(),
  76                          base::Time::Now().ToInternalValue(),
  77                          suggestion_index));
  78 }
  79
  80 // Returns a pending feedback data structure for the spellcheck |result| and
  81 // |text|.
  82 Misspelling BuildFeedback(const SpellCheckResult& result,
  83                           const base::string16& text) {
  84   size_t start = result.location;
  85   base::string16 context = TrimWords(&start,
  86                                start + result.length,
  87                                text,
  88                                chrome::spellcheck_common::kContextWordCount);
  89   return Misspelling(context,
  90                      start,
  91                      result.length,
  92                      std::vector<base::string16>(1, result.replacement),
  93                      result.hash);
  94 }
  95
  96 // Builds suggestion info from |suggestions|. The caller owns the result.
  97 base::ListValue* BuildSuggestionInfo(
  98     const std::vector<Misspelling>& suggestions,
  99     bool is_first_feedback_batch) {
 100   base::ListValue* list = new base::ListValue;
 101   for (std::vector<Misspelling>::const_iterator suggestion_it =
 102            suggestions.begin();
 103        suggestion_it != suggestions.end();
 104        ++suggestion_it) {
 105     base::DictionaryValue* suggestion = SerializeMisspelling(*suggestion_it);
 106     suggestion->SetBoolean("isFirstInSession", is_first_feedback_batch);
 107     suggestion->SetBoolean("isAutoCorrection", false);
 108     list->Append(suggestion);
 109   }
 110   return list;
 111 }
 112
 113 // Builds feedback parameters from |suggestion_info|, |language|, and |country|.
 114 // Takes ownership of |suggestion_list|. The caller owns the result.
 115 base::DictionaryValue* BuildParams(base::ListValue* suggestion_info,
 116                                    const std::string& language,
 117                                    const std::string& country) {
 118   base::DictionaryValue* params = new base::DictionaryValue;
 119   params->Set("suggestionInfo", suggestion_info);
 120   params->SetString("key", google_apis::GetAPIKey());
 121   params->SetString("language", language);
 122   params->SetString("originCountry", country);
 123   params->SetString("clientName", "Chrome");
 124   return params;
 125 }
 126
 127 // Builds feedback data from |params|. Takes ownership of |params|. The caller
 128 // owns the result.
 129 base::Value* BuildFeedbackValue(base::DictionaryValue* params,
 130                                 const std::string& api_version) {
 131   base::DictionaryValue* result = new base::DictionaryValue;
 132   result->Set("params", params);
 133   result->SetString("method", "spelling.feedback");
 134   result->SetString("apiVersion", api_version);
 135   return result;
 136 }
 137
 138 // Returns true if the misspelling location is within text bounds.
 139 bool IsInBounds(int misspelling_location,
 140                 int misspelling_length,
 141                 size_t text_length) {
 142   return misspelling_location >= 0 && misspelling_length > 0 &&
 143          static_cast<size_t>(misspelling_location) < text_length &&
 144          static_cast<size_t>(misspelling_location + misspelling_length) <=
 145              text_length;
 146 }
 147
 148 // Returns the feedback API version.
 149 std::string GetApiVersion() {
 150   // This guard is temporary.
 151   // TODO(rouslan): Remove the guard. http://crbug.com/247726
 152   if (base::FieldTrialList::FindFullName(kFeedbackFieldTrialName) ==
 153           kFeedbackFieldTrialEnabledGroupName &&
 154       base::CommandLine::ForCurrentProcess()->HasSwitch(
 155           switches::kEnableSpellingFeedbackFieldTrial)) {
 156     return "v2-internal";
 157   }
 158   return "v2";
 159 }
 160
 161 }  // namespace
 162
 163 FeedbackSender::FeedbackSender(net::URLRequestContextGetter* request_context,
 164                                const std::string& language,
 165                                const std::string& country)
 166     : request_context_(request_context),
 167       api_version_(GetApiVersion()),
 168       language_(language),
 169       country_(country),
 170       misspelling_counter_(0),
 171       feedback_(kMaxFeedbackSizeBytes),
 172       session_start_(base::Time::Now()),
 173       feedback_service_url_(kFeedbackServiceURL) {
 174   // The command-line switch is for testing and temporary.
 175   // TODO(rouslan): Remove the command-line switch when testing is complete.
 176   // http://crbug.com/247726
 177   if (base::CommandLine::ForCurrentProcess()->HasSwitch(
 178           switches::kSpellingServiceFeedbackUrl)) {
 179     feedback_service_url_ =
 180         GURL(base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 181             switches::kSpellingServiceFeedbackUrl));
 182   }
 183 }
 184
 185 FeedbackSender::~FeedbackSender() {
 186 }
 187
 188 void FeedbackSender::SelectedSuggestion(uint32 hash, int suggestion_index) {
 189   Misspelling* misspelling = feedback_.GetMisspelling(hash);
 190   // GetMisspelling() returns null for flushed feedback. Feedback is flushed
 191   // when the session expires every |kSessionHours| hours.
 192   if (!misspelling)
 193     return;
 194   misspelling->action.set_type(SpellcheckAction::TYPE_SELECT);
 195   misspelling->action.set_index(suggestion_index);
 196   misspelling->timestamp = base::Time::Now();
 197 }
 198
 199 void FeedbackSender::AddedToDictionary(uint32 hash) {
 200   Misspelling* misspelling = feedback_.GetMisspelling(hash);
 201   // GetMisspelling() returns null for flushed feedback. Feedback is flushed
 202   // when the session expires every |kSessionHours| hours.
 203   if (!misspelling)
 204     return;
 205   misspelling->action.set_type(SpellcheckAction::TYPE_ADD_TO_DICT);
 206   misspelling->timestamp = base::Time::Now();
 207   const std::set<uint32>& hashes =
 208       feedback_.FindMisspellings(GetMisspelledString(*misspelling));
 209   for (std::set<uint32>::const_iterator hash_it = hashes.begin();
 210        hash_it != hashes.end();
 211        ++hash_it) {
 212     Misspelling* duplicate_misspelling = feedback_.GetMisspelling(*hash_it);
 213     if (!duplicate_misspelling || duplicate_misspelling->action.IsFinal())
 214       continue;
 215     duplicate_misspelling->action.set_type(SpellcheckAction::TYPE_ADD_TO_DICT);
 216     duplicate_misspelling->timestamp = misspelling->timestamp;
 217   }
 218 }
 219
 220 void FeedbackSender::RecordInDictionary(uint32 hash) {
 221   Misspelling* misspelling = feedback_.GetMisspelling(hash);
 222   // GetMisspelling() returns null for flushed feedback. Feedback is flushed
 223   // when the session expires every |kSessionHours| hours.
 224   if (!misspelling)
 225     return;
 226   misspelling->action.set_type(SpellcheckAction::TYPE_IN_DICTIONARY);
 227 }
 228
 229 void FeedbackSender::IgnoredSuggestions(uint32 hash) {
 230   Misspelling* misspelling = feedback_.GetMisspelling(hash);
 231   // GetMisspelling() returns null for flushed feedback. Feedback is flushed
 232   // when the session expires every |kSessionHours| hours.
 233   if (!misspelling)
 234     return;
 235   misspelling->action.set_type(SpellcheckAction::TYPE_PENDING_IGNORE);
 236   misspelling->timestamp = base::Time::Now();
 237 }
 238
 239 void FeedbackSender::ManuallyCorrected(uint32 hash,
 240                                        const base::string16& correction) {
 241   Misspelling* misspelling = feedback_.GetMisspelling(hash);
 242   // GetMisspelling() returns null for flushed feedback. Feedback is flushed
 243   // when the session expires every |kSessionHours| hours.
 244   if (!misspelling)
 245     return;
 246   misspelling->action.set_type(SpellcheckAction::TYPE_MANUALLY_CORRECTED);
 247   misspelling->action.set_value(correction);
 248   misspelling->timestamp = base::Time::Now();
 249 }
 250
 251 void FeedbackSender::OnReceiveDocumentMarkers(
 252     int renderer_process_id,
 253     const std::vector<uint32>& markers) {
 254   if ((base::Time::Now() - session_start_).InHours() >=
 255       chrome::spellcheck_common::kSessionHours) {
 256     FlushFeedback();
 257     return;
 258   }
 259
 260   if (!feedback_.RendererHasMisspellings(renderer_process_id))
 261     return;
 262
 263   feedback_.FinalizeRemovedMisspellings(renderer_process_id, markers);
 264   SendFeedback(feedback_.GetMisspellingsInRenderer(renderer_process_id),
 265                !renderers_sent_feedback_.count(renderer_process_id));
 266   renderers_sent_feedback_.insert(renderer_process_id);
 267   feedback_.EraseFinalizedMisspellings(renderer_process_id);
 268 }
 269
 270 void FeedbackSender::OnSpellcheckResults(
 271     int renderer_process_id,
 272     const base::string16& text,
 273     const std::vector<SpellCheckMarker>& markers,
 274     std::vector<SpellCheckResult>* results) {
 275   // Don't collect feedback if not going to send it.
 276   if (!timer_.IsRunning())
 277     return;
 278
 279   // Generate a map of marker offsets to marker hashes. This map helps to
 280   // efficiently lookup feedback data based on the position of the misspelling
 281   // in text.
 282   typedef std::map<size_t, uint32> MarkerMap;
 283   MarkerMap marker_map;
 284   for (size_t i = 0; i < markers.size(); ++i)
 285     marker_map[markers[i].offset] = markers[i].hash;
 286
 287   for (std::vector<SpellCheckResult>::iterator result_it = results->begin();
 288        result_it != results->end();
 289        ++result_it) {
 290     if (!IsInBounds(result_it->location, result_it->length, text.length()))
 291       continue;
 292     MarkerMap::const_iterator marker_it = marker_map.find(result_it->location);
 293     if (marker_it != marker_map.end() &&
 294         feedback_.HasMisspelling(marker_it->second)) {
 295       // If the renderer already has a marker for this spellcheck result, then
 296       // set the hash of the spellcheck result to be the same as the marker.
 297       result_it->hash = marker_it->second;
 298     } else {
 299       // If the renderer does not yet have a marker for this spellcheck result,
 300       // then generate a new hash for the spellcheck result.
 301       result_it->hash = BuildHash(session_start_, ++misspelling_counter_);
 302     }
 303     // Save the feedback data for the spellcheck result.
 304     feedback_.AddMisspelling(renderer_process_id,
 305                              BuildFeedback(*result_it, text));
 306   }
 307 }
 308
 309 void FeedbackSender::OnLanguageCountryChange(const std::string& language,
 310                                              const std::string& country) {
 311   FlushFeedback();
 312   language_ = language;
 313   country_ = country;
 314 }
 315
 316 void FeedbackSender::StartFeedbackCollection() {
 317   if (timer_.IsRunning())
 318     return;
 319
 320   int interval_seconds = chrome::spellcheck_common::kFeedbackIntervalSeconds;
 321   // This command-line switch is for testing and temporary.
 322   // TODO(rouslan): Remove the command-line switch when testing is complete.
 323   // http://crbug.com/247726
 324   if (base::CommandLine::ForCurrentProcess()->HasSwitch(
 325           switches::kSpellingServiceFeedbackIntervalSeconds)) {
 326     base::StringToInt(
 327         base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 328             switches::kSpellingServiceFeedbackIntervalSeconds),
 329         &interval_seconds);
 330     if (interval_seconds < kMinIntervalSeconds)
 331       interval_seconds = kMinIntervalSeconds;
 332     static const int kSessionSeconds =
 333         chrome::spellcheck_common::kSessionHours * 60 * 60;
 334     if (interval_seconds >  kSessionSeconds)
 335       interval_seconds = kSessionSeconds;
 336   }
 337   timer_.Start(FROM_HERE,
 338                base::TimeDelta::FromSeconds(interval_seconds),
 339                this,
 340                &FeedbackSender::RequestDocumentMarkers);
 341 }
 342
 343 void FeedbackSender::StopFeedbackCollection() {
 344   if (!timer_.IsRunning())
 345     return;
 346
 347   FlushFeedback();
 348   timer_.Stop();
 349 }
 350
 351 void FeedbackSender::OnURLFetchComplete(const net::URLFetcher* source) {
 352   for (ScopedVector<net::URLFetcher>::iterator sender_it = senders_.begin();
 353        sender_it != senders_.end();
 354        ++sender_it) {
 355     if (*sender_it == source) {
 356       senders_.erase(sender_it);
 357       return;
 358     }
 359   }
 360   delete source;
 361 }
 362
 363 void FeedbackSender::RequestDocumentMarkers() {
 364   // Request document markers from all the renderers that are still alive.
 365   std::set<int> alive_renderers;
 366   for (content::RenderProcessHost::iterator it(
 367            content::RenderProcessHost::AllHostsIterator());
 368        !it.IsAtEnd();
 369        it.Advance()) {
 370     alive_renderers.insert(it.GetCurrentValue()->GetID());
 371     it.GetCurrentValue()->Send(new SpellCheckMsg_RequestDocumentMarkers());
 372   }
 373
 374   // Asynchronously send out the feedback for all the renderers that are no
 375   // longer alive.
 376   std::vector<int> known_renderers = feedback_.GetRendersWithMisspellings();
 377   std::sort(known_renderers.begin(), known_renderers.end());
 378   std::vector<int> dead_renderers =
 379       base::STLSetDifference<std::vector<int> >(known_renderers,
 380                                                 alive_renderers);
 381   for (std::vector<int>::const_iterator it = dead_renderers.begin();
 382        it != dead_renderers.end();
 383        ++it) {
 384     base::ThreadTaskRunnerHandle::Get()->PostTask(
 385         FROM_HERE, base::Bind(&FeedbackSender::OnReceiveDocumentMarkers,
 386                               AsWeakPtr(), *it, std::vector<uint32>()));
 387   }
 388 }
 389
 390 void FeedbackSender::FlushFeedback() {
 391   if (feedback_.Empty())
 392     return;
 393   feedback_.FinalizeAllMisspellings();
 394   SendFeedback(feedback_.GetAllMisspellings(),
 395                renderers_sent_feedback_.empty());
 396   feedback_.Clear();
 397   renderers_sent_feedback_.clear();
 398   session_start_ = base::Time::Now();
 399   timer_.Reset();
 400 }
 401
 402 void FeedbackSender::SendFeedback(const std::vector<Misspelling>& feedback_data,
 403                                   bool is_first_feedback_batch) {
 404   scoped_ptr<base::Value> feedback_value(BuildFeedbackValue(
 405       BuildParams(BuildSuggestionInfo(feedback_data, is_first_feedback_batch),
 406                   language_,
 407                   country_),
 408       api_version_));
 409   std::string feedback;
 410   base::JSONWriter::Write(*feedback_value, &feedback);
 411
 412   // The tests use this identifier to mock the URL fetcher.
 413   static const int kUrlFetcherId = 0;
 414   net::URLFetcher* sender =
 415       net::URLFetcher::Create(kUrlFetcherId, feedback_service_url_,
 416                               net::URLFetcher::POST, this).release();
 417   data_use_measurement::DataUseUserData::AttachToFetcher(
 418       sender, data_use_measurement::DataUseUserData::SPELL_CHECKER);
 419   sender->SetLoadFlags(net::LOAD_DO_NOT_SEND_COOKIES |
 420                        net::LOAD_DO_NOT_SAVE_COOKIES);
 421   sender->SetUploadData("application/json", feedback);
 422   senders_.push_back(sender);
 423
 424   // Request context is NULL in testing.
 425   if (request_context_.get()) {
 426     sender->SetRequestContext(request_context_.get());
 427     sender->Start();
 428   }
 429 }
 430
 431 }  // namespace spellcheck