1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/android/most_visited_sites.h"
7 #include "base/android/jni_android.h"
8 #include "base/android/jni_array.h"
9 #include "base/android/jni_string.h"
10 #include "base/android/scoped_java_ref.h"
11 #include "base/callback.h"
12 #include "base/command_line.h"
13 #include "base/metrics/histogram.h"
14 #include "base/metrics/sparse_histogram.h"
15 #include "base/strings/string_number_conversions.h"
16 #include "base/strings/string_util.h"
17 #include "base/strings/stringprintf.h"
18 #include "base/strings/utf_string_conversions.h"
19 #include "base/time/time.h"
20 #include "chrome/browser/android/popular_sites.h"
21 #include "chrome/browser/history/top_sites_factory.h"
22 #include "chrome/browser/profiles/profile.h"
23 #include "chrome/browser/profiles/profile_android.h"
24 #include "chrome/browser/search/suggestions/suggestions_service_factory.h"
25 #include "chrome/browser/search/suggestions/suggestions_source.h"
26 #include "chrome/browser/sync/profile_sync_service.h"
27 #include "chrome/browser/sync/profile_sync_service_factory.h"
28 #include "chrome/browser/thumbnails/thumbnail_list_source.h"
29 #include "chrome/common/chrome_switches.h"
30 #include "components/history/core/browser/top_sites.h"
31 #include "components/suggestions/suggestions_service.h"
32 #include "components/suggestions/suggestions_utils.h"
33 #include "content/public/browser/browser_thread.h"
34 #include "content/public/browser/url_data_source.h"
35 #include "jni/MostVisitedSites_jni.h"
36 #include "third_party/skia/include/core/SkBitmap.h"
37 #include "ui/gfx/android/java_bitmap.h"
38 #include "ui/gfx/codec/jpeg_codec.h"
41 using base::android::AttachCurrentThread
;
42 using base::android::ConvertJavaStringToUTF8
;
43 using base::android::ScopedJavaGlobalRef
;
44 using base::android::ScopedJavaLocalRef
;
45 using base::android::ToJavaArrayOfStrings
;
46 using content::BrowserThread
;
47 using history::TopSites
;
48 using suggestions::ChromeSuggestion
;
49 using suggestions::SuggestionsProfile
;
50 using suggestions::SuggestionsService
;
51 using suggestions::SuggestionsServiceFactory
;
52 using suggestions::SyncState
;
56 // Total number of tiles displayed.
57 const char kNumTilesHistogramName
[] = "NewTabPage.NumberOfTiles";
58 // Tracking thumbnails.
59 const char kNumLocalThumbnailTilesHistogramName
[] =
60 "NewTabPage.NumberOfThumbnailTiles";
61 const char kNumEmptyTilesHistogramName
[] = "NewTabPage.NumberOfGrayTiles";
62 const char kNumServerTilesHistogramName
[] = "NewTabPage.NumberOfExternalTiles";
64 // Format for tile clicks histogram.
65 const char kOpenedItemHistogramFormat
[] = "NewTabPage.MostVisited.%s";
66 // Format for tile impressions histogram.
67 const char kImpressionHistogramFormat
[] = "NewTabPage.SuggestionsImpression.%s";
68 // Identifiers for the various tile sources.
69 const char kHistogramClientName
[] = "client";
70 const char kHistogramServerName
[] = "server";
71 const char kHistogramServerFormat
[] = "server%d";
72 const char kHistogramPopularName
[] = "popular";
74 const char kPopularSitesFieldTrialName
[] = "NTPPopularSites";
76 scoped_ptr
<SkBitmap
> MaybeFetchLocalThumbnail(
78 const scoped_refptr
<TopSites
>& top_sites
) {
79 DCHECK_CURRENTLY_ON(BrowserThread::DB
);
80 scoped_refptr
<base::RefCountedMemory
> image
;
81 scoped_ptr
<SkBitmap
> bitmap
;
82 if (top_sites
&& top_sites
->GetPageThumbnail(url
, false, &image
))
83 bitmap
.reset(gfx::JPEGCodec::Decode(image
->front(), image
->size()));
87 // Log an event for a given |histogram| at a given element |position|. This
88 // routine exists because regular histogram macros are cached thus can't be used
89 // if the name of the histogram will change at a given call site.
90 void LogHistogramEvent(const std::string
& histogram
,
93 base::HistogramBase
* counter
= base::LinearHistogram::FactoryGet(
98 base::Histogram::kUmaTargetedHistogramFlag
);
100 counter
->Add(position
);
103 // Return the current SyncState for use with the SuggestionsService.
104 SyncState
GetSyncState(Profile
* profile
) {
105 ProfileSyncService
* sync
=
106 ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile
);
108 return SyncState::SYNC_OR_HISTORY_SYNC_DISABLED
;
109 return suggestions::GetSyncState(
110 sync
->CanSyncStart(),
111 sync
->IsSyncActive() && sync
->ConfigurationDone(),
112 sync
->GetActiveDataTypes().Has(syncer::HISTORY_DELETE_DIRECTIVES
));
115 bool ShouldShowPopularSites() {
116 // Note: It's important to query the field trial state first, to ensure that
117 // UMA reports the correct group.
118 const std::string group_name
=
119 base::FieldTrialList::FindFullName(kPopularSitesFieldTrialName
);
120 base::CommandLine
* cmd_line
= base::CommandLine::ForCurrentProcess();
121 if (cmd_line
->HasSwitch(switches::kDisableNTPPopularSites
))
123 if (cmd_line
->HasSwitch(switches::kEnableNTPPopularSites
))
125 return base::StartsWith(group_name
, "Enabled",
126 base::CompareCase::INSENSITIVE_ASCII
);
129 std::string
GetPopularSitesFilename() {
130 return variations::GetVariationParamValue(kPopularSitesFieldTrialName
,
136 MostVisitedSites::MostVisitedSites(Profile
* profile
)
137 : profile_(profile
), num_sites_(0), received_most_visited_sites_(false),
138 received_popular_sites_(false), recorded_uma_(false),
139 num_local_thumbs_(0), num_server_thumbs_(0), num_empty_thumbs_(0),
140 scoped_observer_(this), weak_ptr_factory_(this) {
141 // Register the debugging page for the Suggestions Service and the thumbnails
143 content::URLDataSource::Add(profile_
,
144 new suggestions::SuggestionsSource(profile_
));
145 content::URLDataSource::Add(profile_
, new ThumbnailListSource(profile_
));
147 // Register this class as an observer to the sync service. It is important to
148 // be notified of changes in the sync state such as initialization, sync
149 // being enabled or disabled, etc.
150 ProfileSyncService
* profile_sync_service
=
151 ProfileSyncServiceFactory::GetForProfile(profile_
);
152 if (profile_sync_service
)
153 profile_sync_service
->AddObserver(this);
155 if (ShouldShowPopularSites()) {
156 popular_sites_
.reset(new PopularSites(
158 GetPopularSitesFilename(),
159 profile_
->GetRequestContext(),
160 base::Bind(&MostVisitedSites::OnPopularSitesAvailable
,
161 base::Unretained(this))));
163 received_popular_sites_
= true;
167 MostVisitedSites::~MostVisitedSites() {
168 ProfileSyncService
* profile_sync_service
=
169 ProfileSyncServiceFactory::GetForProfile(profile_
);
170 if (profile_sync_service
&& profile_sync_service
->HasObserver(this))
171 profile_sync_service
->RemoveObserver(this);
174 void MostVisitedSites::Destroy(JNIEnv
* env
, jobject obj
) {
178 void MostVisitedSites::OnLoadingComplete(JNIEnv
* env
, jobject obj
) {
179 RecordThumbnailUMAMetrics();
182 void MostVisitedSites::SetMostVisitedURLsObserver(JNIEnv
* env
,
186 observer_
.Reset(env
, j_observer
);
187 num_sites_
= num_sites
;
189 QueryMostVisitedURLs();
191 scoped_refptr
<history::TopSites
> top_sites
=
192 TopSitesFactory::GetForProfile(profile_
);
194 // TopSites updates itself after a delay. To ensure up-to-date results,
195 // force an update now.
196 top_sites
->SyncWithHistory();
198 // Register as TopSitesObserver so that we can update ourselves when the
200 scoped_observer_
.Add(top_sites
.get());
204 void MostVisitedSites::GetURLThumbnail(JNIEnv
* env
,
207 jobject j_callback_obj
) {
208 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
209 scoped_ptr
<ScopedJavaGlobalRef
<jobject
>> j_callback(
210 new ScopedJavaGlobalRef
<jobject
>());
211 j_callback
->Reset(env
, j_callback_obj
);
213 GURL
url(ConvertJavaStringToUTF8(env
, j_url
));
214 scoped_refptr
<TopSites
> top_sites(TopSitesFactory::GetForProfile(profile_
));
216 BrowserThread::PostTaskAndReplyWithResult(
217 BrowserThread::DB
, FROM_HERE
,
218 base::Bind(&MaybeFetchLocalThumbnail
, url
, top_sites
),
219 base::Bind(&MostVisitedSites::OnLocalThumbnailFetched
,
220 weak_ptr_factory_
.GetWeakPtr(), url
,
221 base::Passed(&j_callback
)));
224 void MostVisitedSites::OnLocalThumbnailFetched(
226 scoped_ptr
<ScopedJavaGlobalRef
<jobject
>> j_callback
,
227 scoped_ptr
<SkBitmap
> bitmap
) {
228 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
230 // A thumbnail is not locally available for |url|. Make sure it is put in
231 // the list to be fetched at the next visit to this site.
232 scoped_refptr
<TopSites
> top_sites(TopSitesFactory::GetForProfile(profile_
));
234 top_sites
->AddForcedURL(url
, base::Time::Now());
235 // Also fetch a remote thumbnail if possible. PopularSites or the
236 // SuggestionsService can supply a thumbnail download URL.
237 SuggestionsService
* suggestions_service
=
238 SuggestionsServiceFactory::GetForProfile(profile_
);
239 if (suggestions_service
) {
240 if (popular_sites_
) {
241 const std::vector
<PopularSites::Site
>& sites
= popular_sites_
->sites();
242 auto it
= std::find_if(sites
.begin(), sites
.end(),
243 [&url
](const PopularSites::Site
& site
) {
244 return site
.url
== url
;
246 if (it
!= sites
.end() && it
->thumbnail_url
.is_valid()) {
247 return suggestions_service
->GetPageThumbnailWithURL(
248 url
, it
->thumbnail_url
,
249 base::Bind(&MostVisitedSites::OnObtainedThumbnail
,
250 weak_ptr_factory_
.GetWeakPtr(), false,
251 base::Passed(&j_callback
)));
254 if (mv_source_
== SUGGESTIONS_SERVICE
) {
255 return suggestions_service
->GetPageThumbnail(
256 url
, base::Bind(&MostVisitedSites::OnObtainedThumbnail
,
257 weak_ptr_factory_
.GetWeakPtr(), false,
258 base::Passed(&j_callback
)));
262 OnObtainedThumbnail(true, j_callback
.Pass(), url
, bitmap
.get());
265 void MostVisitedSites::OnObtainedThumbnail(
266 bool is_local_thumbnail
,
267 scoped_ptr
<ScopedJavaGlobalRef
<jobject
>> j_callback
,
269 const SkBitmap
* bitmap
) {
270 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
271 JNIEnv
* env
= AttachCurrentThread();
272 ScopedJavaLocalRef
<jobject
> j_bitmap
;
274 j_bitmap
= gfx::ConvertToJavaBitmap(bitmap
);
275 if (is_local_thumbnail
) {
278 ++num_server_thumbs_
;
283 Java_ThumbnailCallback_onMostVisitedURLsThumbnailAvailable(
284 env
, j_callback
->obj(), j_bitmap
.obj());
287 void MostVisitedSites::BlacklistUrl(JNIEnv
* env
,
290 GURL
url(ConvertJavaStringToUTF8(env
, j_url
));
292 // Always blacklist in the local TopSites.
293 scoped_refptr
<TopSites
> top_sites
= TopSitesFactory::GetForProfile(profile_
);
295 top_sites
->AddBlacklistedURL(url
);
297 // Only blacklist in the server-side suggestions service if it's active.
298 if (mv_source_
== SUGGESTIONS_SERVICE
) {
299 SuggestionsService
* suggestions_service
=
300 SuggestionsServiceFactory::GetForProfile(profile_
);
301 DCHECK(suggestions_service
);
302 suggestions_service
->BlacklistURL(
303 url
, base::Bind(&MostVisitedSites::OnSuggestionsProfileAvailable
,
304 weak_ptr_factory_
.GetWeakPtr()),
309 void MostVisitedSites::RecordOpenedMostVisitedItem(JNIEnv
* env
,
313 DCHECK_LT(index
, static_cast<int>(tile_sources_
.size()));
314 std::string histogram
= base::StringPrintf(kOpenedItemHistogramFormat
,
315 tile_sources_
[index
].c_str());
316 LogHistogramEvent(histogram
, index
, num_sites_
);
319 void MostVisitedSites::OnStateChanged() {
320 // There have been changes to the sync state. This class cares about a few
321 // (just initialized, enabled/disabled or history sync state changed). Re-run
322 // the query code which will use the proper state.
323 QueryMostVisitedURLs();
327 bool MostVisitedSites::Register(JNIEnv
* env
) {
328 return RegisterNativesImpl(env
);
331 void MostVisitedSites::QueryMostVisitedURLs() {
332 SuggestionsService
* suggestions_service
=
333 SuggestionsServiceFactory::GetForProfile(profile_
);
334 if (suggestions_service
) {
335 // Suggestions service is enabled, initiate a query.
336 suggestions_service
->FetchSuggestionsData(
337 GetSyncState(profile_
),
338 base::Bind(&MostVisitedSites::OnSuggestionsProfileAvailable
,
339 weak_ptr_factory_
.GetWeakPtr()));
341 InitiateTopSitesQuery();
345 void MostVisitedSites::InitiateTopSitesQuery() {
346 scoped_refptr
<TopSites
> top_sites
= TopSitesFactory::GetForProfile(profile_
);
350 top_sites
->GetMostVisitedURLs(
351 base::Bind(&MostVisitedSites::OnMostVisitedURLsAvailable
,
352 weak_ptr_factory_
.GetWeakPtr()),
356 void MostVisitedSites::OnMostVisitedURLsAvailable(
357 const history::MostVisitedURLList
& visited_list
) {
358 std::vector
<base::string16
> titles
;
359 std::vector
<std::string
> urls
;
360 tile_sources_
.clear();
361 int num_tiles
= std::min(static_cast<int>(visited_list
.size()), num_sites_
);
362 for (int i
= 0; i
< num_tiles
; ++i
) {
363 const history::MostVisitedURL
& visited
= visited_list
[i
];
364 if (visited
.url
.is_empty()) {
366 break; // This is the signal that there are no more real visited sites.
368 titles
.push_back(visited
.title
);
369 urls
.push_back(visited
.url
.spec());
370 tile_sources_
.push_back(kHistogramClientName
);
373 received_most_visited_sites_
= true;
374 mv_source_
= TOP_SITES
;
375 AddPopularSites(&titles
, &urls
);
376 NotifyMostVisitedURLsObserver(titles
, urls
);
379 void MostVisitedSites::OnSuggestionsProfileAvailable(
380 const SuggestionsProfile
& suggestions_profile
) {
381 int num_tiles
= suggestions_profile
.suggestions_size();
382 // With no server suggestions, fall back to local Most Visited.
383 if (num_tiles
== 0) {
384 InitiateTopSitesQuery();
387 if (num_sites_
< num_tiles
)
388 num_tiles
= num_sites_
;
390 std::vector
<base::string16
> titles
;
391 std::vector
<std::string
> urls
;
392 tile_sources_
.clear();
393 for (int i
= 0; i
< num_tiles
; ++i
) {
394 const ChromeSuggestion
& suggestion
= suggestions_profile
.suggestions(i
);
395 titles
.push_back(base::UTF8ToUTF16(suggestion
.title()));
396 urls
.push_back(suggestion
.url());
397 std::string tile_source
;
398 if (suggestion
.providers_size() > 0) {
400 base::StringPrintf(kHistogramServerFormat
, suggestion
.providers(0));
402 tile_source
= kHistogramServerName
;
404 tile_sources_
.push_back(tile_source
);
407 received_most_visited_sites_
= true;
408 mv_source_
= SUGGESTIONS_SERVICE
;
409 AddPopularSites(&titles
, &urls
);
410 NotifyMostVisitedURLsObserver(titles
, urls
);
413 void MostVisitedSites::AddPopularSites(std::vector
<base::string16
>* titles
,
414 std::vector
<std::string
>* urls
) {
418 DCHECK_EQ(titles
->size(), urls
->size());
419 DCHECK_EQ(titles
->size(), tile_sources_
.size());
420 DCHECK_LE(static_cast<int>(titles
->size()), num_sites_
);
422 // Collect all non-blacklisted popular suggestions.
423 std::vector
<base::string16
> popular_titles
;
424 std::vector
<std::string
> popular_urls
;
425 scoped_refptr
<TopSites
> top_sites(TopSitesFactory::GetForProfile(profile_
));
426 for (const PopularSites::Site
& popular_site
: popular_sites_
->sites()) {
427 // Skip blacklisted sites.
428 if (top_sites
&& top_sites
->IsBlacklisted(popular_site
.url
))
431 popular_titles
.push_back(popular_site
.title
);
432 popular_urls
.push_back(popular_site
.url
.spec());
433 if (static_cast<int>(popular_titles
.size()) >= num_sites_
)
438 num_sites_
, popular_titles
, popular_urls
, titles
, urls
, &tile_sources_
);
442 void MostVisitedSites::AddPopularSitesImpl(
444 const std::vector
<base::string16
>& popular_titles
,
445 const std::vector
<std::string
>& popular_urls
,
446 std::vector
<base::string16
>* titles
,
447 std::vector
<std::string
>* urls
,
448 std::vector
<std::string
>* tile_sources
) {
449 // Start off with the popular suggestions.
450 std::vector
<base::string16
> new_titles(popular_titles
);
451 std::vector
<std::string
> new_urls(popular_urls
);
452 std::vector
<std::string
> new_tile_sources(new_titles
.size(),
453 kHistogramPopularName
);
455 // Now, go over the personalized suggestions and replace matching popular
456 // suggestions. This is so that when some of the popular suggestions become
457 // personal, they retain their absolute positions.
458 std::vector
<base::string16
> titles_to_insert
;
459 std::vector
<std::string
> urls_to_insert
;
460 std::vector
<std::string
> tile_sources_to_insert
;
461 for (size_t site_index
= 0; site_index
< titles
->size(); site_index
++) {
462 const base::string16
& title
= (*titles
)[site_index
];
463 const std::string
& url
= (*urls
)[site_index
];
464 const std::string
& tile_source
= (*tile_sources
)[site_index
];
465 // See if we already have a matching popular site.
467 for (size_t i
= 0; i
< new_urls
.size(); i
++) {
468 if (new_tile_sources
[i
] == kHistogramPopularName
&&
469 GURL(new_urls
[i
]).host() == GURL(url
).host()) {
470 // We have a matching popular sites suggestion. Replace it with the
471 // actual URL and title.
472 new_titles
[i
] = title
;
474 new_tile_sources
[i
] = tile_source
;
480 titles_to_insert
.push_back(title
);
481 urls_to_insert
.push_back(url
);
482 tile_sources_to_insert
.push_back(tile_source
);
486 // Append personalized suggestions at the end if there's room.
487 size_t num_to_append
=
488 std::min(static_cast<size_t>(num_sites
) - new_titles
.size(),
489 titles_to_insert
.size());
490 new_titles
.insert(new_titles
.end(),
491 titles_to_insert
.end() - num_to_append
,
492 titles_to_insert
.end());
493 new_urls
.insert(new_urls
.end(),
494 urls_to_insert
.end() - num_to_append
,
495 urls_to_insert
.end());
496 new_tile_sources
.insert(new_tile_sources
.end(),
497 tile_sources_to_insert
.end() - num_to_append
,
498 tile_sources_to_insert
.end());
500 // Finally, go over the remaining personalized suggestions and evict popular
501 // suggestions to accommodate them. Do it in reverse order, so the least
502 // important popular suggestions will be evicted.
503 for (size_t i
= titles_to_insert
.size() - num_to_append
; i
> 0; --i
) {
504 const base::string16
& title
= titles_to_insert
[i
- 1];
505 const std::string
& url
= urls_to_insert
[i
- 1];
506 const std::string
& tile_source
= tile_sources_to_insert
[i
- 1];
507 for (size_t insert_i
= new_titles
.size(); insert_i
> 0; --insert_i
) {
508 size_t insert_index
= insert_i
- 1;
509 if (new_tile_sources
[insert_index
] == kHistogramPopularName
) {
510 new_titles
[insert_index
] = title
;
511 new_urls
[insert_index
] = url
;
512 new_tile_sources
[insert_index
] = tile_source
;
518 titles
->swap(new_titles
);
519 urls
->swap(new_urls
);
520 tile_sources
->swap(new_tile_sources
);
523 void MostVisitedSites::NotifyMostVisitedURLsObserver(
524 const std::vector
<base::string16
>& titles
,
525 const std::vector
<std::string
>& urls
) {
526 DCHECK_EQ(titles
.size(), urls
.size());
527 if (received_most_visited_sites_
&& received_popular_sites_
&&
529 RecordImpressionUMAMetrics();
530 UMA_HISTOGRAM_SPARSE_SLOWLY(kNumTilesHistogramName
, titles
.size());
531 recorded_uma_
= true;
533 JNIEnv
* env
= AttachCurrentThread();
534 Java_MostVisitedURLsObserver_onMostVisitedURLsAvailable(
535 env
, observer_
.obj(), ToJavaArrayOfStrings(env
, titles
).obj(),
536 ToJavaArrayOfStrings(env
, urls
).obj());
539 void MostVisitedSites::OnPopularSitesAvailable(bool success
) {
540 received_popular_sites_
= true;
543 LOG(WARNING
) << "Download of popular sites failed";
547 if (observer_
.is_null())
550 std::vector
<std::string
> urls
;
551 std::vector
<std::string
> favicon_urls
;
552 for (const PopularSites::Site
& popular_site
: popular_sites_
->sites()) {
553 urls
.push_back(popular_site
.url
.spec());
554 favicon_urls
.push_back(popular_site
.favicon_url
.spec());
556 JNIEnv
* env
= AttachCurrentThread();
557 Java_MostVisitedURLsObserver_onPopularURLsAvailable(
558 env
, observer_
.obj(), ToJavaArrayOfStrings(env
, urls
).obj(),
559 ToJavaArrayOfStrings(env
, favicon_urls
).obj());
561 QueryMostVisitedURLs();
564 void MostVisitedSites::RecordThumbnailUMAMetrics() {
565 UMA_HISTOGRAM_SPARSE_SLOWLY(kNumLocalThumbnailTilesHistogramName
,
567 num_local_thumbs_
= 0;
568 UMA_HISTOGRAM_SPARSE_SLOWLY(kNumEmptyTilesHistogramName
, num_empty_thumbs_
);
569 num_empty_thumbs_
= 0;
570 UMA_HISTOGRAM_SPARSE_SLOWLY(kNumServerTilesHistogramName
, num_server_thumbs_
);
571 num_server_thumbs_
= 0;
574 void MostVisitedSites::RecordImpressionUMAMetrics() {
575 for (size_t i
= 0; i
< tile_sources_
.size(); i
++) {
576 std::string histogram
= base::StringPrintf(kImpressionHistogramFormat
,
577 tile_sources_
[i
].c_str());
578 LogHistogramEvent(histogram
, static_cast<int>(i
), num_sites_
);
582 void MostVisitedSites::TopSitesLoaded(history::TopSites
* top_sites
) {
585 void MostVisitedSites::TopSitesChanged(history::TopSites
* top_sites
,
586 ChangeReason change_reason
) {
587 if (mv_source_
== TOP_SITES
) {
588 // The displayed suggestions are invalidated.
589 QueryMostVisitedURLs();
593 static jlong
Init(JNIEnv
* env
,
594 const JavaParamRef
<jobject
>& obj
,
595 const JavaParamRef
<jobject
>& jprofile
) {
596 MostVisitedSites
* most_visited_sites
=
597 new MostVisitedSites(ProfileAndroid::FromProfileAndroid(jprofile
));
598 return reinterpret_cast<intptr_t>(most_visited_sites
);