1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/utility/importer/firefox_importer.h"
9 #include "base/file_util.h"
10 #include "base/files/file_enumerator.h"
11 #include "base/memory/scoped_ptr.h"
12 #include "base/message_loop/message_loop.h"
13 #include "base/stl_util.h"
14 #include "base/strings/string_util.h"
15 #include "base/strings/utf_string_conversions.h"
16 #include "chrome/common/importer/firefox_importer_utils.h"
17 #include "chrome/common/importer/firefox_importer_utils.h"
18 #include "chrome/common/importer/imported_bookmark_entry.h"
19 #include "chrome/common/importer/imported_favicon_usage.h"
20 #include "chrome/common/importer/importer_bridge.h"
21 #include "chrome/common/importer/importer_url_row.h"
22 #include "chrome/utility/importer/bookmark_html_reader.h"
23 #include "chrome/utility/importer/favicon_reencode.h"
24 #include "chrome/utility/importer/nss_decryptor.h"
25 #include "components/autofill/core/common/password_form.h"
26 #include "grit/generated_resources.h"
27 #include "sql/connection.h"
28 #include "sql/statement.h"
33 // Original definition is in http://mxr.mozilla.org/firefox/source/toolkit/
34 // components/places/public/nsINavBookmarksService.idl
35 enum BookmarkItemType
{
39 TYPE_DYNAMIC_CONTAINER
= 4
42 // Loads the default bookmarks in the Firefox installed at |app_path|,
43 // and stores their locations in |urls|.
44 void LoadDefaultBookmarks(const base::FilePath
& app_path
,
45 std::set
<GURL
>* urls
) {
46 base::FilePath file
= app_path
.AppendASCII("defaults")
47 .AppendASCII("profile")
48 .AppendASCII("bookmarks.html");
51 std::vector
<ImportedBookmarkEntry
> bookmarks
;
52 bookmark_html_reader::ImportBookmarksFile(base::Callback
<bool(void)>(),
53 base::Callback
<bool(const GURL
&)>(),
57 for (size_t i
= 0; i
< bookmarks
.size(); ++i
)
58 urls
->insert(bookmarks
[i
].url
);
61 // Returns true if |url| has a valid scheme that we allow to import. We
62 // filter out the URL with a unsupported scheme.
63 bool CanImportURL(const GURL
& url
) {
64 // The URL is not valid.
68 // Filter out the URLs with unsupported schemes.
69 const char* const kInvalidSchemes
[] = {"wyciwyg", "place", "about", "chrome"};
70 for (size_t i
= 0; i
< arraysize(kInvalidSchemes
); ++i
) {
71 if (url
.SchemeIs(kInvalidSchemes
[i
]))
80 struct FirefoxImporter::BookmarkItem
{
85 BookmarkItemType type
;
87 base::Time date_added
;
92 FirefoxImporter::FirefoxImporter() {
95 FirefoxImporter::~FirefoxImporter() {
98 void FirefoxImporter::StartImport(
99 const importer::SourceProfile
& source_profile
,
101 ImporterBridge
* bridge
) {
103 source_path_
= source_profile
.source_path
;
104 app_path_
= source_profile
.app_path
;
106 #if defined(OS_POSIX)
107 locale_
= source_profile
.locale
;
110 // The order here is important!
111 bridge_
->NotifyStarted();
112 if ((items
& importer::HOME_PAGE
) && !cancelled()) {
113 bridge_
->NotifyItemStarted(importer::HOME_PAGE
);
114 ImportHomepage(); // Doesn't have a UI item.
115 bridge_
->NotifyItemEnded(importer::HOME_PAGE
);
118 // Note history should be imported before bookmarks because bookmark import
119 // will also import favicons and we store favicon for a URL only if the URL
120 // exist in history or bookmarks.
121 if ((items
& importer::HISTORY
) && !cancelled()) {
122 bridge_
->NotifyItemStarted(importer::HISTORY
);
124 bridge_
->NotifyItemEnded(importer::HISTORY
);
127 if ((items
& importer::FAVORITES
) && !cancelled()) {
128 bridge_
->NotifyItemStarted(importer::FAVORITES
);
130 bridge_
->NotifyItemEnded(importer::FAVORITES
);
132 if ((items
& importer::SEARCH_ENGINES
) && !cancelled()) {
133 bridge_
->NotifyItemStarted(importer::SEARCH_ENGINES
);
134 ImportSearchEngines();
135 bridge_
->NotifyItemEnded(importer::SEARCH_ENGINES
);
137 if ((items
& importer::PASSWORDS
) && !cancelled()) {
138 bridge_
->NotifyItemStarted(importer::PASSWORDS
);
140 bridge_
->NotifyItemEnded(importer::PASSWORDS
);
142 bridge_
->NotifyEnded();
145 void FirefoxImporter::ImportHistory() {
146 base::FilePath file
= source_path_
.AppendASCII("places.sqlite");
147 if (!base::PathExists(file
))
154 // |visit_type| represent the transition type of URLs (typed, click,
155 // redirect, bookmark, etc.) We eliminate some URLs like sub-frames and
156 // redirects, since we don't want them to appear in history.
157 // Firefox transition types are defined in:
158 // toolkit/components/places/public/nsINavHistoryService.idl
159 const char* query
= "SELECT h.url, h.title, h.visit_count, "
160 "h.hidden, h.typed, v.visit_date "
161 "FROM moz_places h JOIN moz_historyvisits v "
162 "ON h.id = v.place_id "
163 "WHERE v.visit_type <= 3";
165 sql::Statement
s(db
.GetUniqueStatement(query
));
167 std::vector
<ImporterURLRow
> rows
;
168 while (s
.Step() && !cancelled()) {
169 GURL
url(s
.ColumnString(0));
171 // Filter out unwanted URLs.
172 if (!CanImportURL(url
))
175 ImporterURLRow
row(url
);
176 row
.title
= s
.ColumnString16(1);
177 row
.visit_count
= s
.ColumnInt(2);
178 row
.hidden
= s
.ColumnInt(3) == 1;
179 row
.typed_count
= s
.ColumnInt(4);
180 row
.last_visit
= base::Time::FromTimeT(s
.ColumnInt64(5)/1000000);
185 if (!rows
.empty() && !cancelled())
186 bridge_
->SetHistoryItems(rows
, importer::VISIT_SOURCE_FIREFOX_IMPORTED
);
189 void FirefoxImporter::ImportBookmarks() {
190 base::FilePath file
= source_path_
.AppendASCII("places.sqlite");
191 if (!base::PathExists(file
))
198 // Get the bookmark folders that we are interested in.
199 int toolbar_folder_id
= -1;
200 int menu_folder_id
= -1;
201 int unsorted_folder_id
= -1;
202 LoadRootNodeID(&db
, &toolbar_folder_id
, &menu_folder_id
, &unsorted_folder_id
);
204 // Load livemark IDs.
205 std::set
<int> livemark_id
;
206 LoadLivemarkIDs(&db
, &livemark_id
);
208 // Load the default bookmarks.
209 std::set
<GURL
> default_urls
;
210 LoadDefaultBookmarks(app_path_
, &default_urls
);
213 GetTopBookmarkFolder(&db
, toolbar_folder_id
, &list
);
214 GetTopBookmarkFolder(&db
, menu_folder_id
, &list
);
215 GetTopBookmarkFolder(&db
, unsorted_folder_id
, &list
);
216 size_t count
= list
.size();
217 for (size_t i
= 0; i
< count
; ++i
)
218 GetWholeBookmarkFolder(&db
, &list
, i
, NULL
);
220 std::vector
<ImportedBookmarkEntry
> bookmarks
;
221 std::vector
<importer::URLKeywordInfo
> url_keywords
;
222 FaviconMap favicon_map
;
224 // TODO(jcampan): http://b/issue?id=1196285 we do not support POST based
225 // keywords yet. We won't include them in the list.
226 std::set
<int> post_keyword_ids
;
227 const char* query
= "SELECT b.id FROM moz_bookmarks b "
228 "INNER JOIN moz_items_annos ia ON ia.item_id = b.id "
229 "INNER JOIN moz_anno_attributes aa ON ia.anno_attribute_id = aa.id "
230 "WHERE aa.name = 'bookmarkProperties/POSTData'";
231 sql::Statement
s(db
.GetUniqueStatement(query
));
236 while (s
.Step() && !cancelled())
237 post_keyword_ids
.insert(s
.ColumnInt(0));
239 for (size_t i
= 0; i
< list
.size(); ++i
) {
240 BookmarkItem
* item
= list
[i
];
242 if (item
->type
== TYPE_FOLDER
) {
243 // Folders are added implicitly on adding children, so we only explicitly
244 // add empty folders.
245 if (!item
->empty_folder
)
247 } else if (item
->type
== TYPE_BOOKMARK
) {
248 // Import only valid bookmarks
249 if (!CanImportURL(item
->url
))
255 // Skip the default bookmarks and unwanted URLs.
256 if (default_urls
.find(item
->url
) != default_urls
.end() ||
257 post_keyword_ids
.find(item
->id
) != post_keyword_ids
.end())
260 // Find the bookmark path by tracing their links to parent folders.
261 std::vector
<base::string16
> path
;
262 BookmarkItem
* child
= item
;
263 bool found_path
= false;
264 bool is_in_toolbar
= false;
265 while (child
->parent
>= 0) {
266 BookmarkItem
* parent
= list
[child
->parent
];
267 if (livemark_id
.find(parent
->id
) != livemark_id
.end()) {
268 // Don't import live bookmarks.
272 if (parent
->id
!= menu_folder_id
) {
273 // To avoid excessive nesting, omit the name for the bookmarks menu
275 path
.insert(path
.begin(), parent
->title
);
278 if (parent
->id
== toolbar_folder_id
)
279 is_in_toolbar
= true;
281 if (parent
->id
== toolbar_folder_id
||
282 parent
->id
== menu_folder_id
||
283 parent
->id
== unsorted_folder_id
) {
284 // We've reached a root node, hooray!
295 ImportedBookmarkEntry entry
;
296 entry
.creation_time
= item
->date_added
;
297 entry
.title
= item
->title
;
298 entry
.url
= item
->url
;
300 entry
.in_toolbar
= is_in_toolbar
;
301 entry
.is_folder
= item
->type
== TYPE_FOLDER
;
303 bookmarks
.push_back(entry
);
305 if (item
->type
== TYPE_BOOKMARK
) {
307 favicon_map
[item
->favicon
].insert(item
->url
);
309 // This bookmark has a keyword, we should import it.
310 if (!item
->keyword
.empty() && item
->url
.is_valid()) {
311 importer::URLKeywordInfo url_keyword_info
;
312 url_keyword_info
.url
= item
->url
;
313 url_keyword_info
.keyword
.assign(base::UTF8ToUTF16(item
->keyword
));
314 url_keyword_info
.display_name
= item
->title
;
315 url_keywords
.push_back(url_keyword_info
);
320 STLDeleteElements(&list
);
322 // Write into profile.
323 if (!bookmarks
.empty() && !cancelled()) {
324 const base::string16
& first_folder_name
=
325 bridge_
->GetLocalizedString(IDS_BOOKMARK_GROUP_FROM_FIREFOX
);
326 bridge_
->AddBookmarks(bookmarks
, first_folder_name
);
328 if (!url_keywords
.empty() && !cancelled()) {
329 bridge_
->SetKeywords(url_keywords
, false);
331 if (!favicon_map
.empty() && !cancelled()) {
332 std::vector
<ImportedFaviconUsage
> favicons
;
333 LoadFavicons(&db
, favicon_map
, &favicons
);
334 bridge_
->SetFavicons(favicons
);
338 void FirefoxImporter::ImportPasswords() {
340 NSSDecryptor decryptor
;
341 if (!decryptor
.Init(source_path_
, source_path_
) &&
342 !decryptor
.Init(app_path_
, source_path_
)) {
346 std::vector
<autofill::PasswordForm
> forms
;
347 base::FilePath source_path
= source_path_
;
348 base::FilePath file
= source_path
.AppendASCII("signons.sqlite");
349 if (base::PathExists(file
)) {
350 // Since Firefox 3.1, passwords are in signons.sqlite db.
351 decryptor
.ReadAndParseSignons(file
, &forms
);
353 // Firefox 3.0 uses signons3.txt to store the passwords.
354 file
= source_path
.AppendASCII("signons3.txt");
355 if (!base::PathExists(file
))
356 file
= source_path
.AppendASCII("signons2.txt");
359 base::ReadFileToString(file
, &content
);
360 decryptor
.ParseSignons(content
, &forms
);
364 for (size_t i
= 0; i
< forms
.size(); ++i
) {
365 bridge_
->SetPasswordForm(forms
[i
]);
370 void FirefoxImporter::ImportSearchEngines() {
371 std::vector
<std::string
> search_engine_data
;
372 GetSearchEnginesXMLData(&search_engine_data
);
374 bridge_
->SetFirefoxSearchEnginesXMLData(search_engine_data
);
377 void FirefoxImporter::ImportHomepage() {
378 GURL home_page
= GetHomepage(source_path_
);
379 if (home_page
.is_valid() && !IsDefaultHomepage(home_page
, app_path_
)) {
380 bridge_
->AddHomePage(home_page
);
384 void FirefoxImporter::GetSearchEnginesXMLData(
385 std::vector
<std::string
>* search_engine_data
) {
386 // TODO(mpawlowski): This may no longer work, search engines are stored in
387 // search.json since Firefox 3.5, not in search.sqlite. XML definitions are
388 // still necessary. http://crbug.com/329175
389 base::FilePath file
= source_path_
.AppendASCII("search.sqlite");
390 if (!base::PathExists(file
))
397 const char* query
= "SELECT engineid FROM engine_data "
398 "WHERE engineid NOT IN "
399 "(SELECT engineid FROM engine_data "
400 "WHERE name='hidden') "
401 "ORDER BY value ASC";
403 sql::Statement
s(db
.GetUniqueStatement(query
));
407 const base::FilePath
searchplugins_path(FILE_PATH_LITERAL("searchplugins"));
408 // Search engine definitions are XMLs stored in two directories. Default
409 // engines are in the app directory (app_path_) and custom engines are
410 // in the profile directory (source_path_).
412 // Since Firefox 21, app_path_ engines are in 'browser' subdirectory:
413 base::FilePath app_path
=
414 app_path_
.AppendASCII("browser").Append(searchplugins_path
);
415 if (!base::PathExists(app_path
)) {
416 // This might be an older Firefox, try old location without the 'browser'
418 app_path
= app_path_
.Append(searchplugins_path
);
421 base::FilePath profile_path
= source_path_
.Append(searchplugins_path
);
423 // Firefox doesn't store a search engine in its sqlite database unless the
424 // user has added a engine. So we get search engines from sqlite db as well
425 // as from the file system.
427 const std::string
kAppPrefix("[app]/");
428 const std::string
kProfilePrefix("[profile]/");
431 std::string
engine(s
.ColumnString(0));
433 // The string contains [app]/<name>.xml or [profile]/<name>.xml where
434 // the [app] and [profile] need to be replaced with the actual app or
436 size_t index
= engine
.find(kAppPrefix
);
437 if (index
!= std::string::npos
) {
439 file
= app_path
.AppendASCII(engine
.substr(index
+ kAppPrefix
.length()));
440 } else if ((index
= engine
.find(kProfilePrefix
)) != std::string::npos
) {
441 // Remove '[profile]/'.
442 file
= profile_path
.AppendASCII(
443 engine
.substr(index
+ kProfilePrefix
.length()));
445 // Looks like absolute path to the file.
446 file
= base::FilePath::FromUTF8Unsafe(engine
);
448 std::string file_data
;
449 base::ReadFileToString(file
, &file_data
);
450 search_engine_data
->push_back(file_data
);
451 } while (s
.Step() && !cancelled());
454 #if defined(OS_POSIX)
455 // Ubuntu-flavored Firefox supports locale-specific search engines via
456 // locale-named subdirectories. They fall back to en-US.
457 // See http://crbug.com/53899
458 // TODO(jshin): we need to make sure our locale code matches that of
460 DCHECK(!locale_
.empty());
461 base::FilePath locale_app_path
= app_path
.AppendASCII(locale_
);
462 base::FilePath default_locale_app_path
= app_path
.AppendASCII("en-US");
463 if (base::DirectoryExists(locale_app_path
))
464 app_path
= locale_app_path
;
465 else if (base::DirectoryExists(default_locale_app_path
))
466 app_path
= default_locale_app_path
;
469 // Get search engine definition from file system.
470 base::FileEnumerator
engines(app_path
, false, base::FileEnumerator::FILES
);
471 for (base::FilePath engine_path
= engines
.Next();
472 !engine_path
.value().empty(); engine_path
= engines
.Next()) {
473 std::string file_data
;
474 base::ReadFileToString(file
, &file_data
);
475 search_engine_data
->push_back(file_data
);
479 void FirefoxImporter::LoadRootNodeID(sql::Connection
* db
,
480 int* toolbar_folder_id
,
482 int* unsorted_folder_id
) {
483 static const char* kToolbarFolderName
= "toolbar";
484 static const char* kMenuFolderName
= "menu";
485 static const char* kUnsortedFolderName
= "unfiled";
487 const char* query
= "SELECT root_name, folder_id FROM moz_bookmarks_roots";
488 sql::Statement
s(db
->GetUniqueStatement(query
));
491 std::string folder
= s
.ColumnString(0);
492 int id
= s
.ColumnInt(1);
493 if (folder
== kToolbarFolderName
)
494 *toolbar_folder_id
= id
;
495 else if (folder
== kMenuFolderName
)
496 *menu_folder_id
= id
;
497 else if (folder
== kUnsortedFolderName
)
498 *unsorted_folder_id
= id
;
502 void FirefoxImporter::LoadLivemarkIDs(sql::Connection
* db
,
503 std::set
<int>* livemark
) {
504 static const char* kFeedAnnotation
= "livemark/feedURI";
507 const char* query
= "SELECT b.item_id "
508 "FROM moz_anno_attributes a "
509 "JOIN moz_items_annos b ON a.id = b.anno_attribute_id "
511 sql::Statement
s(db
->GetUniqueStatement(query
));
512 s
.BindString(0, kFeedAnnotation
);
514 while (s
.Step() && !cancelled())
515 livemark
->insert(s
.ColumnInt(0));
518 void FirefoxImporter::GetTopBookmarkFolder(sql::Connection
* db
,
520 BookmarkList
* list
) {
521 const char* query
= "SELECT b.title "
522 "FROM moz_bookmarks b "
523 "WHERE b.type = 2 AND b.id = ? "
524 "ORDER BY b.position";
525 sql::Statement
s(db
->GetUniqueStatement(query
));
526 s
.BindInt(0, folder_id
);
529 BookmarkItem
* item
= new BookmarkItem
;
530 item
->parent
= -1; // The top level folder has no parent.
531 item
->id
= folder_id
;
532 item
->title
= s
.ColumnString16(0);
533 item
->type
= TYPE_FOLDER
;
535 item
->empty_folder
= true;
536 list
->push_back(item
);
540 void FirefoxImporter::GetWholeBookmarkFolder(sql::Connection
* db
,
543 bool* empty_folder
) {
544 if (position
>= list
->size()) {
549 const char* query
= "SELECT b.id, h.url, COALESCE(b.title, h.title), "
550 "b.type, k.keyword, b.dateAdded, h.favicon_id "
551 "FROM moz_bookmarks b "
552 "LEFT JOIN moz_places h ON b.fk = h.id "
553 "LEFT JOIN moz_keywords k ON k.id = b.keyword_id "
554 "WHERE b.type IN (1,2) AND b.parent = ? "
555 "ORDER BY b.position";
556 sql::Statement
s(db
->GetUniqueStatement(query
));
557 s
.BindInt(0, (*list
)[position
]->id
);
559 BookmarkList temp_list
;
561 BookmarkItem
* item
= new BookmarkItem
;
562 item
->parent
= static_cast<int>(position
);
563 item
->id
= s
.ColumnInt(0);
564 item
->url
= GURL(s
.ColumnString(1));
565 item
->title
= s
.ColumnString16(2);
566 item
->type
= static_cast<BookmarkItemType
>(s
.ColumnInt(3));
567 item
->keyword
= s
.ColumnString(4);
568 item
->date_added
= base::Time::FromTimeT(s
.ColumnInt64(5)/1000000);
569 item
->favicon
= s
.ColumnInt64(6);
570 item
->empty_folder
= true;
572 temp_list
.push_back(item
);
573 if (empty_folder
!= NULL
)
574 *empty_folder
= false;
577 // Appends all items to the list.
578 for (BookmarkList::iterator i
= temp_list
.begin();
579 i
!= temp_list
.end(); ++i
) {
581 // Recursive add bookmarks in sub-folders.
582 if ((*i
)->type
== TYPE_FOLDER
)
583 GetWholeBookmarkFolder(db
, list
, list
->size() - 1, &(*i
)->empty_folder
);
587 void FirefoxImporter::LoadFavicons(
589 const FaviconMap
& favicon_map
,
590 std::vector
<ImportedFaviconUsage
>* favicons
) {
591 const char* query
= "SELECT url, data FROM moz_favicons WHERE id=?";
592 sql::Statement
s(db
->GetUniqueStatement(query
));
597 for (FaviconMap::const_iterator i
= favicon_map
.begin();
598 i
!= favicon_map
.end(); ++i
) {
599 s
.BindInt64(0, i
->first
);
601 ImportedFaviconUsage usage
;
603 usage
.favicon_url
= GURL(s
.ColumnString(0));
604 if (!usage
.favicon_url
.is_valid())
605 continue; // Don't bother importing favicons with invalid URLs.
607 std::vector
<unsigned char> data
;
608 s
.ColumnBlobAsVector(1, &data
);
610 continue; // Data definitely invalid.
612 if (!importer::ReencodeFavicon(&data
[0], data
.size(), &usage
.png_data
))
613 continue; // Unable to decode.
615 usage
.urls
= i
->second
;
616 favicons
->push_back(usage
);