src/base/search/searchpluginmanager.cpp

   1 /*
   2  * Bittorrent Client using Qt and libtorrent.
   3  * Copyright (C) 2015, 2018  Vladimir Golovnev <glassez@yandex.ru>
   4  * Copyright (C) 2006  Christophe Dumez <chris@qbittorrent.org>
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version 2
   9  * of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  *
  20  * In addition, as a special exception, the copyright holders give permission to
  21  * link this program with the OpenSSL project's "OpenSSL" library (or with
  22  * modified versions of it that use the same license as the "OpenSSL" library),
  23  * and distribute the linked executables. You must obey the GNU General Public
  24  * License in all respects for all of the code used other than "OpenSSL".  If you
  25  * modify file(s), you may extend this exception to your version of the file(s),
  26  * but you are not obligated to do so. If you do not wish to do so, delete this
  27  * exception statement from your version.
  28  */
  29
  30 #include "searchpluginmanager.h"
  31
  32 #include <memory>
  33
  34 #include <QDir>
  35 #include <QDirIterator>
  36 #include <QDomDocument>
  37 #include <QDomElement>
  38 #include <QDomNode>
  39 #include <QPointer>
  40 #include <QProcess>
  41
  42 #include "base/global.h"
  43 #include "base/logger.h"
  44 #include "base/net/downloadmanager.h"
  45 #include "base/preferences.h"
  46 #include "base/profile.h"
  47 #include "base/utils/bytearray.h"
  48 #include "base/utils/foreignapps.h"
  49 #include "base/utils/fs.h"
  50 #include "searchdownloadhandler.h"
  51 #include "searchhandler.h"
  52
  53 namespace
  54 {
  55     void clearPythonCache(const Path &path)
  56     {
  57         // remove python cache artifacts in `path` and subdirs
  58
  59         PathList dirs = {path};
  60         QDirIterator iter {path.data(), (QDir::AllDirs | QDir::NoDotAndDotDot), QDirIterator::Subdirectories};
  61         while (iter.hasNext())
  62             dirs += Path(iter.next());
  63
  64         for (const Path &dir : asConst(dirs))
  65         {
  66             // python 3: remove "__pycache__" folders
  67             if (dir.filename() == QLatin1String("__pycache__"))
  68             {
  69                 Utils::Fs::removeDirRecursively(dir);
  70                 continue;
  71             }
  72
  73             // python 2: remove "*.pyc" files
  74             const QStringList files = QDir(dir.data()).entryList(QDir::Files);
  75             for (const QString &file : files)
  76             {
  77                 const Path path {file};
  78                 if (path.hasExtension(QLatin1String(".pyc")))
  79                     Utils::Fs::removeFile(path);
  80             }
  81         }
  82     }
  83 }
  84
  85 QPointer<SearchPluginManager> SearchPluginManager::m_instance = nullptr;
  86
  87 SearchPluginManager::SearchPluginManager()
  88     : m_updateUrl(QLatin1String("http://searchplugins.qbittorrent.org/nova3/engines/"))
  89 {
  90     Q_ASSERT(!m_instance); // only one instance is allowed
  91     m_instance = this;
  92
  93     updateNova();
  94     update();
  95 }
  96
  97 SearchPluginManager::~SearchPluginManager()
  98 {
  99     qDeleteAll(m_plugins);
 100 }
 101
 102 SearchPluginManager *SearchPluginManager::instance()
 103 {
 104     if (!m_instance)
 105         m_instance = new SearchPluginManager;
 106     return m_instance;
 107 }
 108
 109 void SearchPluginManager::freeInstance()
 110 {
 111     delete m_instance;
 112 }
 113
 114 QStringList SearchPluginManager::allPlugins() const
 115 {
 116     return m_plugins.keys();
 117 }
 118
 119 QStringList SearchPluginManager::enabledPlugins() const
 120 {
 121     QStringList plugins;
 122     for (const PluginInfo *plugin : asConst(m_plugins))
 123     {
 124         if (plugin->enabled)
 125             plugins << plugin->name;
 126     }
 127
 128     return plugins;
 129 }
 130
 131 QStringList SearchPluginManager::supportedCategories() const
 132 {
 133     QStringList result;
 134     for (const PluginInfo *plugin : asConst(m_plugins))
 135     {
 136         if (plugin->enabled)
 137         {
 138             for (const QString &cat : plugin->supportedCategories)
 139             {
 140                 if (!result.contains(cat))
 141                     result << cat;
 142             }
 143         }
 144     }
 145
 146     return result;
 147 }
 148
 149 QStringList SearchPluginManager::getPluginCategories(const QString &pluginName) const
 150 {
 151     QStringList plugins;
 152     if (pluginName == u"all")
 153         plugins = allPlugins();
 154     else if ((pluginName == u"enabled") || (pluginName == u"multi"))
 155         plugins = enabledPlugins();
 156     else
 157         plugins << pluginName.trimmed();
 158
 159     QSet<QString> categories;
 160     for (const QString &name : asConst(plugins))
 161     {
 162         const PluginInfo *plugin = pluginInfo(name);
 163         if (!plugin) continue; // plugin wasn't found
 164         for (const QString &category : plugin->supportedCategories)
 165             categories << category;
 166     }
 167
 168     return categories.values();
 169 }
 170
 171 PluginInfo *SearchPluginManager::pluginInfo(const QString &name) const
 172 {
 173     return m_plugins.value(name);
 174 }
 175
 176 void SearchPluginManager::enablePlugin(const QString &name, const bool enabled)
 177 {
 178     PluginInfo *plugin = m_plugins.value(name, nullptr);
 179     if (plugin)
 180     {
 181         plugin->enabled = enabled;
 182         // Save to Hard disk
 183         Preferences *const pref = Preferences::instance();
 184         QStringList disabledPlugins = pref->getSearchEngDisabled();
 185         if (enabled)
 186             disabledPlugins.removeAll(name);
 187         else if (!disabledPlugins.contains(name))
 188             disabledPlugins.append(name);
 189         pref->setSearchEngDisabled(disabledPlugins);
 190
 191         emit pluginEnabled(name, enabled);
 192     }
 193 }
 194
 195 // Updates shipped plugin
 196 void SearchPluginManager::updatePlugin(const QString &name)
 197 {
 198     installPlugin(QString::fromLatin1("%1%2.py").arg(m_updateUrl, name));
 199 }
 200
 201 // Install or update plugin from file or url
 202 void SearchPluginManager::installPlugin(const QString &source)
 203 {
 204     clearPythonCache(engineLocation());
 205
 206     if (Net::DownloadManager::hasSupportedScheme(source))
 207     {
 208         using namespace Net;
 209         DownloadManager::instance()->download(DownloadRequest(source).saveToFile(true)
 210                                               , this, &SearchPluginManager::pluginDownloadFinished);
 211     }
 212     else
 213     {
 214         const Path path {source.startsWith(u"file:", Qt::CaseInsensitive) ? QUrl(source).toLocalFile() : source};
 215
 216         QString pluginName = path.filename();
 217         if (pluginName.endsWith(u".py", Qt::CaseInsensitive))
 218         {
 219             pluginName.chop(pluginName.size() - pluginName.lastIndexOf(u'.'));
 220             installPlugin_impl(pluginName, path);
 221         }
 222         else
 223         {
 224             emit pluginInstallationFailed(pluginName, tr("Unknown search engine plugin file format."));
 225         }
 226     }
 227 }
 228
 229 void SearchPluginManager::installPlugin_impl(const QString &name, const Path &path)
 230 {
 231     const PluginVersion newVersion = getPluginVersion(path);
 232     const PluginInfo *plugin = pluginInfo(name);
 233     if (plugin && !(plugin->version < newVersion))
 234     {
 235         LogMsg(tr("Plugin already at version %1, which is greater than %2").arg(plugin->version, newVersion), Log::INFO);
 236         emit pluginUpdateFailed(name, tr("A more recent version of this plugin is already installed."));
 237         return;
 238     }
 239
 240     // Process with install
 241     const Path destPath = pluginPath(name);
 242     const Path backupPath = destPath + ".bak";
 243     bool updated = false;
 244     if (destPath.exists())
 245     {
 246         // Backup in case install fails
 247         Utils::Fs::copyFile(destPath, backupPath);
 248         Utils::Fs::removeFile(destPath);
 249         updated = true;
 250     }
 251     // Copy the plugin
 252     Utils::Fs::copyFile(path, destPath);
 253     // Update supported plugins
 254     update();
 255     // Check if this was correctly installed
 256     if (!m_plugins.contains(name))
 257     {
 258         // Remove broken file
 259         Utils::Fs::removeFile(destPath);
 260         LogMsg(tr("Plugin %1 is not supported.").arg(name), Log::INFO);
 261         if (updated)
 262         {
 263             // restore backup
 264             Utils::Fs::copyFile(backupPath, destPath);
 265             Utils::Fs::removeFile(backupPath);
 266             // Update supported plugins
 267             update();
 268             emit pluginUpdateFailed(name, tr("Plugin is not supported."));
 269         }
 270         else
 271         {
 272             emit pluginInstallationFailed(name, tr("Plugin is not supported."));
 273         }
 274     }
 275     else
 276     {
 277         // Install was successful, remove backup
 278         if (updated)
 279         {
 280             LogMsg(tr("Plugin %1 has been successfully updated.").arg(name), Log::INFO);
 281             Utils::Fs::removeFile(backupPath);
 282         }
 283     }
 284 }
 285
 286 bool SearchPluginManager::uninstallPlugin(const QString &name)
 287 {
 288     clearPythonCache(engineLocation());
 289
 290     // remove it from hard drive
 291     const Path pluginsPath = pluginsLocation();
 292     const QStringList filters {name + QLatin1String(".*")};
 293     const QStringList files = QDir(pluginsPath.data()).entryList(filters, QDir::Files, QDir::Unsorted);
 294     for (const QString &file : files)
 295         Utils::Fs::removeFile(pluginsPath / Path(file));
 296     // Remove it from supported engines
 297     delete m_plugins.take(name);
 298
 299     emit pluginUninstalled(name);
 300     return true;
 301 }
 302
 303 void SearchPluginManager::updateIconPath(PluginInfo *const plugin)
 304 {
 305     if (!plugin) return;
 306
 307     const Path pluginsPath = pluginsLocation();
 308     Path iconPath = pluginsPath / Path(plugin->name + QLatin1String(".png"));
 309     if (iconPath.exists())
 310     {
 311         plugin->iconPath = iconPath;
 312     }
 313     else
 314     {
 315         iconPath = pluginsPath / Path(plugin->name + QLatin1String(".ico"));
 316         if (iconPath.exists())
 317             plugin->iconPath = iconPath;
 318     }
 319 }
 320
 321 void SearchPluginManager::checkForUpdates()
 322 {
 323     // Download version file from update server
 324     using namespace Net;
 325     DownloadManager::instance()->download({m_updateUrl + u"versions.txt"}
 326                                           , this, &SearchPluginManager::versionInfoDownloadFinished);
 327 }
 328
 329 SearchDownloadHandler *SearchPluginManager::downloadTorrent(const QString &siteUrl, const QString &url)
 330 {
 331     return new SearchDownloadHandler {siteUrl, url, this};
 332 }
 333
 334 SearchHandler *SearchPluginManager::startSearch(const QString &pattern, const QString &category, const QStringList &usedPlugins)
 335 {
 336     // No search pattern entered
 337     Q_ASSERT(!pattern.isEmpty());
 338
 339     return new SearchHandler {pattern, category, usedPlugins, this};
 340 }
 341
 342 QString SearchPluginManager::categoryFullName(const QString &categoryName)
 343 {
 344     const QHash<QString, QString> categoryTable
 345     {
 346         {u"all"_qs, tr("All categories")},
 347         {u"movies"_qs, tr("Movies")},
 348         {u"tv"_qs, tr("TV shows")},
 349         {u"music"_qs, tr("Music")},
 350         {u"games"_qs, tr("Games")},
 351         {u"anime"_qs, tr("Anime")},
 352         {u"software"_qs, tr("Software")},
 353         {u"pictures"_qs, tr("Pictures")},
 354         {u"books"_qs, tr("Books")}
 355     };
 356     return categoryTable.value(categoryName);
 357 }
 358
 359 QString SearchPluginManager::pluginFullName(const QString &pluginName)
 360 {
 361     return pluginInfo(pluginName) ? pluginInfo(pluginName)->fullName : QString();
 362 }
 363
 364 Path SearchPluginManager::pluginsLocation()
 365 {
 366     return (engineLocation() / Path(u"engines"_qs));
 367 }
 368
 369 Path SearchPluginManager::engineLocation()
 370 {
 371     static Path location;
 372     if (location.isEmpty())
 373     {
 374         location = specialFolderLocation(SpecialFolder::Data) / Path(u"nova3"_qs);
 375         Utils::Fs::mkpath(location);
 376     }
 377
 378     return location;
 379 }
 380
 381 void SearchPluginManager::versionInfoDownloadFinished(const Net::DownloadResult &result)
 382 {
 383     if (result.status == Net::DownloadStatus::Success)
 384         parseVersionInfo(result.data);
 385     else
 386         emit checkForUpdatesFailed(tr("Update server is temporarily unavailable. %1").arg(result.errorString));
 387 }
 388
 389 void SearchPluginManager::pluginDownloadFinished(const Net::DownloadResult &result)
 390 {
 391     if (result.status == Net::DownloadStatus::Success)
 392     {
 393         const Path filePath = result.filePath;
 394
 395         Path pluginPath {QUrl(result.url).path()};
 396         pluginPath.removeExtension(); // Remove extension
 397         installPlugin_impl(pluginPath.filename(), filePath);
 398         Utils::Fs::removeFile(filePath);
 399     }
 400     else
 401     {
 402         const QString url = result.url;
 403         QString pluginName = url.mid(url.lastIndexOf(u'/') + 1);
 404         pluginName.replace(u".py"_qs, u""_qs, Qt::CaseInsensitive);
 405
 406         if (pluginInfo(pluginName))
 407             emit pluginUpdateFailed(pluginName, tr("Failed to download the plugin file. %1").arg(result.errorString));
 408         else
 409             emit pluginInstallationFailed(pluginName, tr("Failed to download the plugin file. %1").arg(result.errorString));
 410     }
 411 }
 412
 413 // Update nova.py search plugin if necessary
 414 void SearchPluginManager::updateNova()
 415 {
 416     // create nova directory if necessary
 417     const Path enginePath = engineLocation();
 418
 419     QFile packageFile {(enginePath / Path(u"__init__.py"_qs)).data()};
 420     packageFile.open(QIODevice::WriteOnly);
 421     packageFile.close();
 422
 423     Utils::Fs::mkdir(enginePath / Path(u"engines"_qs));
 424
 425     QFile packageFile2 {(enginePath / Path(u"engines/__init__.py"_qs)).data()};
 426     packageFile2.open(QIODevice::WriteOnly);
 427     packageFile2.close();
 428
 429     // Copy search plugin files (if necessary)
 430     const auto updateFile = [&enginePath](const Path &filename, const bool compareVersion)
 431     {
 432         const Path filePathBundled = Path(u":/searchengine/nova3"_qs) / filename;
 433         const Path filePathDisk = enginePath / filename;
 434
 435         if (compareVersion && (getPluginVersion(filePathBundled) <= getPluginVersion(filePathDisk)))
 436             return;
 437
 438         Utils::Fs::removeFile(filePathDisk);
 439         Utils::Fs::copyFile(filePathBundled, filePathDisk);
 440     };
 441
 442     updateFile(Path(u"helpers.py"_qs), true);
 443     updateFile(Path(u"nova2.py"_qs), true);
 444     updateFile(Path(u"nova2dl.py"_qs), true);
 445     updateFile(Path(u"novaprinter.py"_qs), true);
 446     updateFile(Path(u"sgmllib3.py"_qs), false);
 447     updateFile(Path(u"socks.py"_qs), false);
 448 }
 449
 450 void SearchPluginManager::update()
 451 {
 452     QProcess nova;
 453     nova.setProcessEnvironment(QProcessEnvironment::systemEnvironment());
 454
 455     const QStringList params {(engineLocation() / Path(u"/nova2.py"_qs)).toString(), QLatin1String("--capabilities")};
 456     nova.start(Utils::ForeignApps::pythonInfo().executableName, params, QIODevice::ReadOnly);
 457     nova.waitForFinished();
 458
 459     const auto capabilities = QString::fromUtf8(nova.readAll());
 460     QDomDocument xmlDoc;
 461     if (!xmlDoc.setContent(capabilities))
 462     {
 463         qWarning() << "Could not parse Nova search engine capabilities, msg: " << capabilities.toLocal8Bit().data();
 464         qWarning() << "Error: " << nova.readAllStandardError().constData();
 465         return;
 466     }
 467
 468     const QDomElement root = xmlDoc.documentElement();
 469     if (root.tagName() != u"capabilities")
 470     {
 471         qWarning() << "Invalid XML file for Nova search engine capabilities, msg: " << capabilities.toLocal8Bit().data();
 472         return;
 473     }
 474
 475     for (QDomNode engineNode = root.firstChild(); !engineNode.isNull(); engineNode = engineNode.nextSibling())
 476     {
 477         const QDomElement engineElem = engineNode.toElement();
 478         if (!engineElem.isNull())
 479         {
 480             const QString pluginName = engineElem.tagName();
 481
 482             auto plugin = std::make_unique<PluginInfo>();
 483             plugin->name = pluginName;
 484             plugin->version = getPluginVersion(pluginPath(pluginName));
 485             plugin->fullName = engineElem.elementsByTagName(u"name"_qs).at(0).toElement().text();
 486             plugin->url = engineElem.elementsByTagName(u"url"_qs).at(0).toElement().text();
 487
 488             const QStringList categories = engineElem.elementsByTagName(u"categories"_qs).at(0).toElement().text().split(u' ');
 489             for (QString cat : categories)
 490             {
 491                 cat = cat.trimmed();
 492                 if (!cat.isEmpty())
 493                     plugin->supportedCategories << cat;
 494             }
 495
 496             const QStringList disabledEngines = Preferences::instance()->getSearchEngDisabled();
 497             plugin->enabled = !disabledEngines.contains(pluginName);
 498
 499             updateIconPath(plugin.get());
 500
 501             if (!m_plugins.contains(pluginName))
 502             {
 503                 m_plugins[pluginName] = plugin.release();
 504                 emit pluginInstalled(pluginName);
 505             }
 506             else if (m_plugins[pluginName]->version != plugin->version)
 507             {
 508                 delete m_plugins.take(pluginName);
 509                 m_plugins[pluginName] = plugin.release();
 510                 emit pluginUpdated(pluginName);
 511             }
 512         }
 513     }
 514 }
 515
 516 void SearchPluginManager::parseVersionInfo(const QByteArray &info)
 517 {
 518     QHash<QString, PluginVersion> updateInfo;
 519     int numCorrectData = 0;
 520
 521     const QVector<QByteArray> lines = Utils::ByteArray::splitToViews(info, "\n", Qt::SkipEmptyParts);
 522     for (QByteArray line : lines)
 523     {
 524         line = line.trimmed();
 525         if (line.isEmpty()) continue;
 526         if (line.startsWith('#')) continue;
 527
 528         const QVector<QByteArray> list = Utils::ByteArray::splitToViews(line, ":", Qt::SkipEmptyParts);
 529         if (list.size() != 2) continue;
 530
 531         const auto pluginName = QString::fromUtf8(list.first().trimmed());
 532         const PluginVersion version = PluginVersion::tryParse(list.last().trimmed(), {});
 533
 534         if (!version.isValid()) continue;
 535
 536         ++numCorrectData;
 537         if (isUpdateNeeded(pluginName, version))
 538         {
 539             LogMsg(tr("Plugin \"%1\" is outdated, updating to version %2").arg(pluginName, version), Log::INFO);
 540             updateInfo[pluginName] = version;
 541         }
 542     }
 543
 544     if (numCorrectData < lines.size())
 545     {
 546         emit checkForUpdatesFailed(tr("Incorrect update info received for %1 out of %2 plugins.")
 547             .arg(QString::number(lines.size() - numCorrectData), QString::number(lines.size())));
 548     }
 549     else
 550     {
 551         emit checkForUpdatesFinished(updateInfo);
 552     }
 553 }
 554
 555 bool SearchPluginManager::isUpdateNeeded(const QString &pluginName, const PluginVersion newVersion) const
 556 {
 557     const PluginInfo *plugin = pluginInfo(pluginName);
 558     if (!plugin) return true;
 559
 560     PluginVersion oldVersion = plugin->version;
 561     return (newVersion > oldVersion);
 562 }
 563
 564 Path SearchPluginManager::pluginPath(const QString &name)
 565 {
 566     return (pluginsLocation() / Path(name + QLatin1String(".py")));
 567 }
 568
 569 PluginVersion SearchPluginManager::getPluginVersion(const Path &filePath)
 570 {
 571     QFile pluginFile {filePath.data()};
 572     if (!pluginFile.open(QIODevice::ReadOnly | QIODevice::Text))
 573         return {};
 574
 575     while (!pluginFile.atEnd())
 576     {
 577         const auto line = QString::fromUtf8(pluginFile.readLine()).remove(u' ');
 578         if (!line.startsWith(u"#VERSION:", Qt::CaseInsensitive)) continue;
 579
 580         const QString versionStr = line.mid(9);
 581         const PluginVersion version = PluginVersion::tryParse(versionStr, {});
 582         if (version.isValid())
 583             return version;
 584
 585         LogMsg(tr("Search plugin '%1' contains invalid version string ('%2')")
 586             .arg(filePath.filename(), versionStr), Log::MsgType::WARNING);
 587         break;
 588     }
 589
 590     return {};
 591 }