chrome/utility/importer/firefox_importer.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/utility/importer/firefox_importer.h"
   6
   7 #include <set>
   8
   9 #include "base/files/file_enumerator.h"
  10 #include "base/files/file_util.h"
  11 #include "base/json/json_file_value_serializer.h"
  12 #include "base/memory/scoped_ptr.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/stl_util.h"
  15 #include "base/strings/string_util.h"
  16 #include "base/strings/utf_string_conversions.h"
  17 #include "chrome/common/importer/firefox_importer_utils.h"
  18 #include "chrome/common/importer/imported_bookmark_entry.h"
  19 #include "chrome/common/importer/imported_favicon_usage.h"
  20 #include "chrome/common/importer/importer_autofill_form_data_entry.h"
  21 #include "chrome/common/importer/importer_bridge.h"
  22 #include "chrome/common/importer/importer_url_row.h"
  23 #include "chrome/grit/generated_resources.h"
  24 #include "chrome/utility/importer/bookmark_html_reader.h"
  25 #include "chrome/utility/importer/favicon_reencode.h"
  26 #include "chrome/utility/importer/nss_decryptor.h"
  27 #include "components/autofill/core/common/password_form.h"
  28 #include "sql/connection.h"
  29 #include "sql/statement.h"
  30 #include "url/gurl.h"
  31
  32 namespace {
  33
  34 // Original definition is in http://mxr.mozilla.org/firefox/source/toolkit/
  35 //  components/places/public/nsINavBookmarksService.idl
  36 enum BookmarkItemType {
  37   TYPE_BOOKMARK = 1,
  38   TYPE_FOLDER = 2,
  39   TYPE_SEPARATOR = 3,
  40   TYPE_DYNAMIC_CONTAINER = 4
  41 };
  42
  43 // Loads the default bookmarks in the Firefox installed at |app_path|,
  44 // and stores their locations in |urls|.
  45 void LoadDefaultBookmarks(const base::FilePath& app_path,
  46                           std::set<GURL>* urls) {
  47   base::FilePath file = app_path.AppendASCII("defaults")
  48       .AppendASCII("profile")
  49       .AppendASCII("bookmarks.html");
  50   urls->clear();
  51
  52   std::vector<ImportedBookmarkEntry> bookmarks;
  53   std::vector<importer::SearchEngineInfo> search_engines;
  54   bookmark_html_reader::ImportBookmarksFile(base::Callback<bool(void)>(),
  55                                             base::Callback<bool(const GURL&)>(),
  56                                             file,
  57                                             &bookmarks,
  58                                             &search_engines,
  59                                             NULL);
  60   for (size_t i = 0; i < bookmarks.size(); ++i)
  61     urls->insert(bookmarks[i].url);
  62 }
  63
  64 // Returns true if |url| has a valid scheme that we allow to import. We
  65 // filter out the URL with a unsupported scheme.
  66 bool CanImportURL(const GURL& url) {
  67   // The URL is not valid.
  68   if (!url.is_valid())
  69     return false;
  70
  71   // Filter out the URLs with unsupported schemes.
  72   const char* const kInvalidSchemes[] = {"wyciwyg", "place", "about", "chrome"};
  73   for (size_t i = 0; i < arraysize(kInvalidSchemes); ++i) {
  74     if (url.SchemeIs(kInvalidSchemes[i]))
  75       return false;
  76   }
  77
  78   return true;
  79 }
  80
  81 }  // namespace
  82
  83 struct FirefoxImporter::BookmarkItem {
  84   int parent;
  85   int id;
  86   GURL url;
  87   base::string16 title;
  88   BookmarkItemType type;
  89   std::string keyword;
  90   base::Time date_added;
  91   int64 favicon;
  92   bool empty_folder;
  93 };
  94
  95 FirefoxImporter::FirefoxImporter() {
  96 }
  97
  98 FirefoxImporter::~FirefoxImporter() {
  99 }
 100
 101 void FirefoxImporter::StartImport(
 102     const importer::SourceProfile& source_profile,
 103     uint16 items,
 104     ImporterBridge* bridge) {
 105   bridge_ = bridge;
 106   source_path_ = source_profile.source_path;
 107   app_path_ = source_profile.app_path;
 108
 109 #if defined(OS_POSIX)
 110   locale_ = source_profile.locale;
 111 #endif
 112
 113   // The order here is important!
 114   bridge_->NotifyStarted();
 115   if ((items & importer::HOME_PAGE) && !cancelled()) {
 116     bridge_->NotifyItemStarted(importer::HOME_PAGE);
 117     ImportHomepage();  // Doesn't have a UI item.
 118     bridge_->NotifyItemEnded(importer::HOME_PAGE);
 119   }
 120
 121   // Note history should be imported before bookmarks because bookmark import
 122   // will also import favicons and we store favicon for a URL only if the URL
 123   // exist in history or bookmarks.
 124   if ((items & importer::HISTORY) && !cancelled()) {
 125     bridge_->NotifyItemStarted(importer::HISTORY);
 126     ImportHistory();
 127     bridge_->NotifyItemEnded(importer::HISTORY);
 128   }
 129
 130   if ((items & importer::FAVORITES) && !cancelled()) {
 131     bridge_->NotifyItemStarted(importer::FAVORITES);
 132     ImportBookmarks();
 133     bridge_->NotifyItemEnded(importer::FAVORITES);
 134   }
 135   if ((items & importer::SEARCH_ENGINES) && !cancelled()) {
 136     bridge_->NotifyItemStarted(importer::SEARCH_ENGINES);
 137     ImportSearchEngines();
 138     bridge_->NotifyItemEnded(importer::SEARCH_ENGINES);
 139   }
 140   if ((items & importer::PASSWORDS) && !cancelled()) {
 141     bridge_->NotifyItemStarted(importer::PASSWORDS);
 142     ImportPasswords();
 143     bridge_->NotifyItemEnded(importer::PASSWORDS);
 144   }
 145   if ((items & importer::AUTOFILL_FORM_DATA) && !cancelled()) {
 146     bridge_->NotifyItemStarted(importer::AUTOFILL_FORM_DATA);
 147     ImportAutofillFormData();
 148     bridge_->NotifyItemEnded(importer::AUTOFILL_FORM_DATA);
 149   }
 150   bridge_->NotifyEnded();
 151 }
 152
 153 void FirefoxImporter::ImportHistory() {
 154   base::FilePath file = source_path_.AppendASCII("places.sqlite");
 155   if (!base::PathExists(file))
 156     return;
 157
 158   sql::Connection db;
 159   if (!db.Open(file))
 160     return;
 161
 162   // |visit_type| represent the transition type of URLs (typed, click,
 163   // redirect, bookmark, etc.) We eliminate some URLs like sub-frames and
 164   // redirects, since we don't want them to appear in history.
 165   // Firefox transition types are defined in:
 166   //   toolkit/components/places/public/nsINavHistoryService.idl
 167   const char query[] =
 168       "SELECT h.url, h.title, h.visit_count, "
 169       "h.hidden, h.typed, v.visit_date "
 170       "FROM moz_places h JOIN moz_historyvisits v "
 171       "ON h.id = v.place_id "
 172       "WHERE v.visit_type <= 3";
 173
 174   sql::Statement s(db.GetUniqueStatement(query));
 175
 176   std::vector<ImporterURLRow> rows;
 177   while (s.Step() && !cancelled()) {
 178     GURL url(s.ColumnString(0));
 179
 180     // Filter out unwanted URLs.
 181     if (!CanImportURL(url))
 182       continue;
 183
 184     ImporterURLRow row(url);
 185     row.title = s.ColumnString16(1);
 186     row.visit_count = s.ColumnInt(2);
 187     row.hidden = s.ColumnInt(3) == 1;
 188     row.typed_count = s.ColumnInt(4);
 189     row.last_visit = base::Time::FromTimeT(s.ColumnInt64(5)/1000000);
 190
 191     rows.push_back(row);
 192   }
 193
 194   if (!rows.empty() && !cancelled())
 195     bridge_->SetHistoryItems(rows, importer::VISIT_SOURCE_FIREFOX_IMPORTED);
 196 }
 197
 198 void FirefoxImporter::ImportBookmarks() {
 199   base::FilePath file = source_path_.AppendASCII("places.sqlite");
 200   if (!base::PathExists(file))
 201     return;
 202
 203   sql::Connection db;
 204   if (!db.Open(file))
 205     return;
 206
 207   // Get the bookmark folders that we are interested in.
 208   int toolbar_folder_id = -1;
 209   int menu_folder_id = -1;
 210   int unsorted_folder_id = -1;
 211   LoadRootNodeID(&db, &toolbar_folder_id, &menu_folder_id, &unsorted_folder_id);
 212
 213   // Load livemark IDs.
 214   std::set<int> livemark_id;
 215   LoadLivemarkIDs(&db, &livemark_id);
 216
 217   // Load the default bookmarks.
 218   std::set<GURL> default_urls;
 219   LoadDefaultBookmarks(app_path_, &default_urls);
 220
 221   BookmarkList list;
 222   GetTopBookmarkFolder(&db, toolbar_folder_id, &list);
 223   GetTopBookmarkFolder(&db, menu_folder_id, &list);
 224   GetTopBookmarkFolder(&db, unsorted_folder_id, &list);
 225   size_t count = list.size();
 226   for (size_t i = 0; i < count; ++i)
 227     GetWholeBookmarkFolder(&db, &list, i, NULL);
 228
 229   std::vector<ImportedBookmarkEntry> bookmarks;
 230   std::vector<importer::SearchEngineInfo> search_engines;
 231   FaviconMap favicon_map;
 232
 233   // TODO(jcampan): http://b/issue?id=1196285 we do not support POST based
 234   //                keywords yet.  We won't include them in the list.
 235   std::set<int> post_keyword_ids;
 236   const char query[] =
 237       "SELECT b.id FROM moz_bookmarks b "
 238       "INNER JOIN moz_items_annos ia ON ia.item_id = b.id "
 239       "INNER JOIN moz_anno_attributes aa ON ia.anno_attribute_id = aa.id "
 240       "WHERE aa.name = 'bookmarkProperties/POSTData'";
 241   sql::Statement s(db.GetUniqueStatement(query));
 242
 243   if (!s.is_valid())
 244     return;
 245
 246   while (s.Step() && !cancelled())
 247     post_keyword_ids.insert(s.ColumnInt(0));
 248
 249   for (size_t i = 0; i < list.size(); ++i) {
 250     BookmarkItem* item = list[i];
 251
 252     // Folders are added implicitly on adding children, so we only explicitly
 253     // add empty folders.
 254     if (item->type != TYPE_BOOKMARK &&
 255         ((item->type != TYPE_FOLDER) || !item->empty_folder))
 256       continue;
 257
 258     if (CanImportURL(item->url)) {
 259       // Skip the default bookmarks and unwanted URLs.
 260       if (default_urls.find(item->url) != default_urls.end() ||
 261           post_keyword_ids.find(item->id) != post_keyword_ids.end())
 262         continue;
 263
 264       // Find the bookmark path by tracing their links to parent folders.
 265       std::vector<base::string16> path;
 266       BookmarkItem* child = item;
 267       bool found_path = false;
 268       bool is_in_toolbar = false;
 269       while (child->parent >= 0) {
 270         BookmarkItem* parent = list[child->parent];
 271         if (livemark_id.find(parent->id) != livemark_id.end()) {
 272           // Don't import live bookmarks.
 273           break;
 274         }
 275
 276         if (parent->id != menu_folder_id) {
 277           // To avoid excessive nesting, omit the name for the bookmarks menu
 278           // folder.
 279           path.insert(path.begin(), parent->title);
 280         }
 281
 282         if (parent->id == toolbar_folder_id)
 283           is_in_toolbar = true;
 284
 285         if (parent->id == toolbar_folder_id ||
 286             parent->id == menu_folder_id ||
 287             parent->id == unsorted_folder_id) {
 288           // We've reached a root node, hooray!
 289           found_path = true;
 290           break;
 291         }
 292
 293         child = parent;
 294       }
 295
 296       if (!found_path)
 297         continue;
 298
 299       ImportedBookmarkEntry entry;
 300       entry.creation_time = item->date_added;
 301       entry.title = item->title;
 302       entry.url = item->url;
 303       entry.path = path;
 304       entry.in_toolbar = is_in_toolbar;
 305       entry.is_folder = item->type == TYPE_FOLDER;
 306
 307       bookmarks.push_back(entry);
 308     }
 309
 310     if (item->type == TYPE_BOOKMARK) {
 311       if (item->favicon)
 312         favicon_map[item->favicon].insert(item->url);
 313
 314       // Import this bookmark as a search engine if it has a keyword and its URL
 315       // is usable as a search engine URL. (Even if the URL doesn't allow
 316       // substitution, importing as a "search engine" allows users to trigger
 317       // the bookmark by entering its keyword in the omnibox.)
 318       if (item->keyword.empty())
 319         continue;
 320       importer::SearchEngineInfo search_engine_info;
 321       std::string search_engine_url;
 322       if (item->url.is_valid())
 323         search_engine_info.url = base::UTF8ToUTF16(item->url.spec());
 324       else if (bookmark_html_reader::CanImportURLAsSearchEngine(
 325                    item->url,
 326                    &search_engine_url))
 327         search_engine_info.url = base::UTF8ToUTF16(search_engine_url);
 328       else
 329         continue;
 330       search_engine_info.keyword = base::UTF8ToUTF16(item->keyword);
 331       search_engine_info.display_name = item->title;
 332       search_engines.push_back(search_engine_info);
 333     }
 334   }
 335
 336   STLDeleteElements(&list);
 337
 338   // Write into profile.
 339   if (!bookmarks.empty() && !cancelled()) {
 340     const base::string16& first_folder_name =
 341         bridge_->GetLocalizedString(IDS_BOOKMARK_GROUP_FROM_FIREFOX);
 342     bridge_->AddBookmarks(bookmarks, first_folder_name);
 343   }
 344   if (!search_engines.empty() && !cancelled()) {
 345     bridge_->SetKeywords(search_engines, false);
 346   }
 347   if (!favicon_map.empty() && !cancelled()) {
 348     std::vector<ImportedFaviconUsage> favicons;
 349     LoadFavicons(&db, favicon_map, &favicons);
 350     bridge_->SetFavicons(favicons);
 351   }
 352 }
 353
 354 void FirefoxImporter::ImportPasswords() {
 355   // Initializes NSS3.
 356   NSSDecryptor decryptor;
 357   if (!decryptor.Init(source_path_, source_path_) &&
 358       !decryptor.Init(app_path_, source_path_)) {
 359     return;
 360   }
 361
 362   std::vector<autofill::PasswordForm> forms;
 363   base::FilePath source_path = source_path_;
 364   base::FilePath file = source_path.AppendASCII("signons.sqlite");
 365   if (base::PathExists(file)) {
 366     // Since Firefox 3.1, passwords are in signons.sqlite db.
 367     decryptor.ReadAndParseSignons(file, &forms);
 368   } else {
 369     // Firefox 3.0 uses signons3.txt to store the passwords.
 370     file = source_path.AppendASCII("signons3.txt");
 371     if (!base::PathExists(file))
 372       file = source_path.AppendASCII("signons2.txt");
 373
 374     std::string content;
 375     base::ReadFileToString(file, &content);
 376     decryptor.ParseSignons(content, &forms);
 377   }
 378
 379   if (!cancelled()) {
 380     for (size_t i = 0; i < forms.size(); ++i) {
 381       if (!forms[i].username_value.empty() ||
 382           !forms[i].password_value.empty() ||
 383           forms[i].blacklisted_by_user) {
 384         bridge_->SetPasswordForm(forms[i]);
 385       }
 386     }
 387   }
 388 }
 389
 390 void FirefoxImporter::ImportSearchEngines() {
 391   std::vector<std::string> search_engine_data;
 392   GetSearchEnginesXMLData(&search_engine_data);
 393
 394   bridge_->SetFirefoxSearchEnginesXMLData(search_engine_data);
 395 }
 396
 397 void FirefoxImporter::ImportHomepage() {
 398   GURL home_page = GetHomepage(source_path_);
 399   if (home_page.is_valid() && !IsDefaultHomepage(home_page, app_path_)) {
 400     bridge_->AddHomePage(home_page);
 401   }
 402 }
 403
 404 void FirefoxImporter::ImportAutofillFormData() {
 405   base::FilePath file = source_path_.AppendASCII("formhistory.sqlite");
 406   if (!base::PathExists(file))
 407     return;
 408
 409   sql::Connection db;
 410   if (!db.Open(file))
 411     return;
 412
 413   const char query[] =
 414       "SELECT fieldname, value, timesUsed, firstUsed, lastUsed FROM "
 415       "moz_formhistory";
 416
 417   sql::Statement s(db.GetUniqueStatement(query));
 418
 419   std::vector<ImporterAutofillFormDataEntry> form_entries;
 420   while (s.Step() && !cancelled()) {
 421     ImporterAutofillFormDataEntry form_entry;
 422     form_entry.name = s.ColumnString16(0);
 423     form_entry.value = s.ColumnString16(1);
 424     form_entry.times_used = s.ColumnInt(2);
 425     form_entry.first_used = base::Time::FromTimeT(s.ColumnInt64(3) / 1000000);
 426     form_entry.last_used = base::Time::FromTimeT(s.ColumnInt64(4) / 1000000);
 427
 428     // Don't import search bar history.
 429     if (base::UTF16ToUTF8(form_entry.name) == "searchbar-history")
 430       continue;
 431
 432     form_entries.push_back(form_entry);
 433   }
 434
 435   if (!form_entries.empty() && !cancelled())
 436     bridge_->SetAutofillFormData(form_entries);
 437 }
 438
 439 void FirefoxImporter::GetSearchEnginesXMLData(
 440     std::vector<std::string>* search_engine_data) {
 441   base::FilePath file = source_path_.AppendASCII("search.sqlite");
 442   if (!base::PathExists(file)) {
 443     // Since Firefox 3.5, search engines are no longer stored in search.sqlite.
 444     // Instead, search.json is used for storing search engines.
 445     GetSearchEnginesXMLDataFromJSON(search_engine_data);
 446     return;
 447   }
 448
 449   sql::Connection db;
 450   if (!db.Open(file))
 451     return;
 452
 453   const char query[] =
 454       "SELECT engineid FROM engine_data "
 455       "WHERE engineid NOT IN "
 456       "(SELECT engineid FROM engine_data "
 457       "WHERE name='hidden') "
 458       "ORDER BY value ASC";
 459
 460   sql::Statement s(db.GetUniqueStatement(query));
 461   if (!s.is_valid())
 462     return;
 463
 464   const base::FilePath searchplugins_path(FILE_PATH_LITERAL("searchplugins"));
 465   // Search engine definitions are XMLs stored in two directories. Default
 466   // engines are in the app directory (app_path_) and custom engines are
 467   // in the profile directory (source_path_).
 468
 469   // Since Firefox 21, app_path_ engines are in 'browser' subdirectory:
 470   base::FilePath app_path =
 471       app_path_.AppendASCII("browser").Append(searchplugins_path);
 472   if (!base::PathExists(app_path)) {
 473     // This might be an older Firefox, try old location without the 'browser'
 474     // path component:
 475     app_path = app_path_.Append(searchplugins_path);
 476   }
 477
 478   base::FilePath profile_path = source_path_.Append(searchplugins_path);
 479
 480   // Firefox doesn't store a search engine in its sqlite database unless the
 481   // user has added a engine. So we get search engines from sqlite db as well
 482   // as from the file system.
 483   if (s.Step()) {
 484     const std::string kAppPrefix("[app]/");
 485     const std::string kProfilePrefix("[profile]/");
 486     do {
 487       base::FilePath file;
 488       std::string engine(s.ColumnString(0));
 489
 490       // The string contains [app]/<name>.xml or [profile]/<name>.xml where
 491       // the [app] and [profile] need to be replaced with the actual app or
 492       // profile path.
 493       size_t index = engine.find(kAppPrefix);
 494       if (index != std::string::npos) {
 495         // Remove '[app]/'.
 496         file = app_path.AppendASCII(engine.substr(index + kAppPrefix.length()));
 497       } else if ((index = engine.find(kProfilePrefix)) != std::string::npos) {
 498         // Remove '[profile]/'.
 499           file = profile_path.AppendASCII(
 500               engine.substr(index + kProfilePrefix.length()));
 501       } else {
 502         // Looks like absolute path to the file.
 503         file = base::FilePath::FromUTF8Unsafe(engine);
 504       }
 505       std::string file_data;
 506       base::ReadFileToString(file, &file_data);
 507       search_engine_data->push_back(file_data);
 508     } while (s.Step() && !cancelled());
 509   }
 510
 511 #if defined(OS_POSIX)
 512   // Ubuntu-flavored Firefox supports locale-specific search engines via
 513   // locale-named subdirectories. They fall back to en-US.
 514   // See http://crbug.com/53899
 515   // TODO(jshin): we need to make sure our locale code matches that of
 516   // Firefox.
 517   DCHECK(!locale_.empty());
 518   base::FilePath locale_app_path = app_path.AppendASCII(locale_);
 519   base::FilePath default_locale_app_path = app_path.AppendASCII("en-US");
 520   if (base::DirectoryExists(locale_app_path))
 521     app_path = locale_app_path;
 522   else if (base::DirectoryExists(default_locale_app_path))
 523     app_path = default_locale_app_path;
 524 #endif
 525
 526   // Get search engine definition from file system.
 527   base::FileEnumerator engines(app_path, false, base::FileEnumerator::FILES);
 528   for (base::FilePath engine_path = engines.Next();
 529        !engine_path.value().empty(); engine_path = engines.Next()) {
 530     std::string file_data;
 531     base::ReadFileToString(file, &file_data);
 532     search_engine_data->push_back(file_data);
 533   }
 534 }
 535
 536 void FirefoxImporter::GetSearchEnginesXMLDataFromJSON(
 537     std::vector<std::string>* search_engine_data) {
 538   // search-metadata.json contains keywords for search engines. This
 539   // file exists only if the user has set keywords for search engines.
 540   base::FilePath search_metadata_json_file =
 541       source_path_.AppendASCII("search-metadata.json");
 542   JSONFileValueSerializer metadata_serializer(search_metadata_json_file);
 543   scoped_ptr<base::Value> metadata_root(
 544       metadata_serializer.Deserialize(NULL, NULL));
 545   const base::DictionaryValue* search_metadata_root = NULL;
 546   if (metadata_root)
 547     metadata_root->GetAsDictionary(&search_metadata_root);
 548
 549   // search.json contains information about search engines to import.
 550   base::FilePath search_json_file = source_path_.AppendASCII("search.json");
 551   if (!base::PathExists(search_json_file))
 552     return;
 553
 554   JSONFileValueSerializer serializer(search_json_file);
 555   scoped_ptr<base::Value> root(serializer.Deserialize(NULL, NULL));
 556   const base::DictionaryValue* search_root = NULL;
 557   if (!root || !root->GetAsDictionary(&search_root))
 558     return;
 559
 560   const std::string kDirectories("directories");
 561   const base::DictionaryValue* search_directories = NULL;
 562   if (!search_root->GetDictionary(kDirectories, &search_directories))
 563     return;
 564
 565   // Dictionary |search_directories| contains a list of search engines
 566   // (default and installed). The list can be found from key <engines>
 567   // of the dictionary. Key <engines> is a grandchild of key <directories>.
 568   // However, key <engines> parent's key is dynamic which depends on
 569   // operating systems. For example,
 570   //   Ubuntu (for default search engine):
 571   //     /usr/lib/firefox/distribution/searchplugins/locale/en-US
 572   //   Ubuntu (for installed search engines):
 573   //     /home/<username>/.mozilla/firefox/lcd50n4n.default/searchplugins
 574   //   Windows (for default search engine):
 575   //     C:\\Program Files (x86)\\Mozilla Firefox\\browser\\searchplugins
 576   // Therefore, it needs to be retrieved by searching.
 577
 578   for (base::DictionaryValue::Iterator it(*search_directories); !it.IsAtEnd();
 579        it.Advance()) {
 580     // The key of |it| may contains dot (.) which cannot be used as <key>
 581     // for retrieving <engines>. Hence, it is needed to get |it| as dictionary.
 582     // The resulted dictionary can be used for retrieving <engines>.
 583     const std::string kEngines("engines");
 584     const base::DictionaryValue* search_directory = NULL;
 585     if (!it.value().GetAsDictionary(&search_directory))
 586       continue;
 587
 588     const base::ListValue* search_engines = NULL;
 589     if (!search_directory->GetList(kEngines, &search_engines))
 590       continue;
 591
 592     const std::string kFilePath("filePath");
 593     const std::string kHidden("_hidden");
 594     for (size_t i = 0; i < search_engines->GetSize(); ++i) {
 595       const base::DictionaryValue* engine_info = NULL;
 596       if (!search_engines->GetDictionary(i, &engine_info))
 597         continue;
 598
 599       bool is_hidden = false;
 600       std::string file_path;
 601       if (!engine_info->GetBoolean(kHidden, &is_hidden) ||
 602           !engine_info->GetString(kFilePath, &file_path))
 603         continue;
 604
 605       if (!is_hidden) {
 606         const std::string kAppPrefix("[app]/");
 607         const std::string kProfilePrefix("[profile]/");
 608         base::FilePath xml_file = base::FilePath::FromUTF8Unsafe(file_path);
 609
 610         // If |file_path| contains [app] or [profile] then they need to be
 611         // replaced with the actual app or profile path.
 612         size_t index = file_path.find(kAppPrefix);
 613         if (index != std::string::npos) {
 614           // Replace '[app]/' with actual app path.
 615           xml_file = app_path_.AppendASCII("searchplugins").AppendASCII(
 616               file_path.substr(index + kAppPrefix.length()));
 617         } else if ((index = file_path.find(kProfilePrefix)) !=
 618                    std::string::npos) {
 619           // Replace '[profile]/' with actual profile path.
 620           xml_file = source_path_.AppendASCII("searchplugins").AppendASCII(
 621               file_path.substr(index + kProfilePrefix.length()));
 622         }
 623
 624         std::string file_data;
 625         base::ReadFileToString(xml_file, &file_data);
 626
 627         // If a keyword is mentioned for this search engine, then add
 628         // it to the XML string as an <Alias> element and use this updated
 629         // string.
 630         const base::DictionaryValue* search_xml_path = NULL;
 631         if (search_metadata_root && search_metadata_root->HasKey(file_path) &&
 632             search_metadata_root->GetDictionaryWithoutPathExpansion(
 633                 file_path, &search_xml_path)) {
 634           std::string alias;
 635           search_xml_path->GetString("alias", &alias);
 636
 637           // Add <Alias> element as the last child element.
 638           size_t end_of_parent = file_data.find("</SearchPlugin>");
 639           if (end_of_parent != std::string::npos && !alias.empty())
 640             file_data.insert(end_of_parent, "<Alias>" + alias + "</Alias> \n");
 641         }
 642         search_engine_data->push_back(file_data);
 643       }
 644     }
 645   }
 646 }
 647
 648 void FirefoxImporter::LoadRootNodeID(sql::Connection* db,
 649                                       int* toolbar_folder_id,
 650                                       int* menu_folder_id,
 651                                       int* unsorted_folder_id) {
 652   static const char kToolbarFolderName[] = "toolbar";
 653   static const char kMenuFolderName[] = "menu";
 654   static const char kUnsortedFolderName[] = "unfiled";
 655
 656   const char query[] = "SELECT root_name, folder_id FROM moz_bookmarks_roots";
 657   sql::Statement s(db->GetUniqueStatement(query));
 658
 659   while (s.Step()) {
 660     std::string folder = s.ColumnString(0);
 661     int id = s.ColumnInt(1);
 662     if (folder == kToolbarFolderName)
 663       *toolbar_folder_id = id;
 664     else if (folder == kMenuFolderName)
 665       *menu_folder_id = id;
 666     else if (folder == kUnsortedFolderName)
 667       *unsorted_folder_id = id;
 668   }
 669 }
 670
 671 void FirefoxImporter::LoadLivemarkIDs(sql::Connection* db,
 672                                        std::set<int>* livemark) {
 673   static const char kFeedAnnotation[] = "livemark/feedURI";
 674   livemark->clear();
 675
 676   const char query[] =
 677       "SELECT b.item_id "
 678       "FROM moz_anno_attributes a "
 679       "JOIN moz_items_annos b ON a.id = b.anno_attribute_id "
 680       "WHERE a.name = ? ";
 681   sql::Statement s(db->GetUniqueStatement(query));
 682   s.BindString(0, kFeedAnnotation);
 683
 684   while (s.Step() && !cancelled())
 685     livemark->insert(s.ColumnInt(0));
 686 }
 687
 688 void FirefoxImporter::GetTopBookmarkFolder(sql::Connection* db,
 689                                             int folder_id,
 690                                             BookmarkList* list) {
 691   const char query[] =
 692       "SELECT b.title "
 693       "FROM moz_bookmarks b "
 694       "WHERE b.type = 2 AND b.id = ? "
 695       "ORDER BY b.position";
 696   sql::Statement s(db->GetUniqueStatement(query));
 697   s.BindInt(0, folder_id);
 698
 699   if (s.Step()) {
 700     BookmarkItem* item = new BookmarkItem;
 701     item->parent = -1;  // The top level folder has no parent.
 702     item->id = folder_id;
 703     item->title = s.ColumnString16(0);
 704     item->type = TYPE_FOLDER;
 705     item->favicon = 0;
 706     item->empty_folder = true;
 707     list->push_back(item);
 708   }
 709 }
 710
 711 void FirefoxImporter::GetWholeBookmarkFolder(sql::Connection* db,
 712                                               BookmarkList* list,
 713                                               size_t position,
 714                                               bool* empty_folder) {
 715   if (position >= list->size()) {
 716     NOTREACHED();
 717     return;
 718   }
 719
 720   const char query[] =
 721       "SELECT b.id, h.url, COALESCE(b.title, h.title), "
 722       "b.type, k.keyword, b.dateAdded, h.favicon_id "
 723       "FROM moz_bookmarks b "
 724       "LEFT JOIN moz_places h ON b.fk = h.id "
 725       "LEFT JOIN moz_keywords k ON k.id = b.keyword_id "
 726       "WHERE b.type IN (1,2) AND b.parent = ? "
 727       "ORDER BY b.position";
 728   sql::Statement s(db->GetUniqueStatement(query));
 729   s.BindInt(0, (*list)[position]->id);
 730
 731   BookmarkList temp_list;
 732   while (s.Step()) {
 733     BookmarkItem* item = new BookmarkItem;
 734     item->parent = static_cast<int>(position);
 735     item->id = s.ColumnInt(0);
 736     item->url = GURL(s.ColumnString(1));
 737     item->title = s.ColumnString16(2);
 738     item->type = static_cast<BookmarkItemType>(s.ColumnInt(3));
 739     item->keyword = s.ColumnString(4);
 740     item->date_added = base::Time::FromTimeT(s.ColumnInt64(5)/1000000);
 741     item->favicon = s.ColumnInt64(6);
 742     item->empty_folder = true;
 743
 744     temp_list.push_back(item);
 745     if (empty_folder != NULL)
 746       *empty_folder = false;
 747   }
 748
 749   // Appends all items to the list.
 750   for (BookmarkList::iterator i = temp_list.begin();
 751        i != temp_list.end(); ++i) {
 752     list->push_back(*i);
 753     // Recursive add bookmarks in sub-folders.
 754     if ((*i)->type == TYPE_FOLDER)
 755       GetWholeBookmarkFolder(db, list, list->size() - 1, &(*i)->empty_folder);
 756   }
 757 }
 758
 759 void FirefoxImporter::LoadFavicons(
 760     sql::Connection* db,
 761     const FaviconMap& favicon_map,
 762     std::vector<ImportedFaviconUsage>* favicons) {
 763   const char query[] = "SELECT url, data FROM moz_favicons WHERE id=?";
 764   sql::Statement s(db->GetUniqueStatement(query));
 765
 766   if (!s.is_valid())
 767     return;
 768
 769   for (FaviconMap::const_iterator i = favicon_map.begin();
 770        i != favicon_map.end(); ++i) {
 771     s.BindInt64(0, i->first);
 772     if (s.Step()) {
 773       ImportedFaviconUsage usage;
 774
 775       usage.favicon_url = GURL(s.ColumnString(0));
 776       if (!usage.favicon_url.is_valid())
 777         continue;  // Don't bother importing favicons with invalid URLs.
 778
 779       std::vector<unsigned char> data;
 780       s.ColumnBlobAsVector(1, &data);
 781       if (data.empty())
 782         continue;  // Data definitely invalid.
 783
 784       if (!importer::ReencodeFavicon(&data[0], data.size(), &usage.png_data))
 785         continue;  // Unable to decode.
 786
 787       usage.urls = i->second;
 788       favicons->push_back(usage);
 789     }
 790     s.Reset(true);
 791   }
 792 }