src/base/net/downloadmanager.cpp

   1 /*
   2  * Bittorrent Client using Qt and libtorrent.
   3  * Copyright (C) 2024  Jonathan Ketchker
   4  * Copyright (C) 2015-2024  Vladimir Golovnev <glassez@yandex.ru>
   5  * Copyright (C) 2006  Christophe Dumez <chris@qbittorrent.org>
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; either version 2
  10  * of the License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  20  *
  21  * In addition, as a special exception, the copyright holders give permission to
  22  * link this program with the OpenSSL project's "OpenSSL" library (or with
  23  * modified versions of it that use the same license as the "OpenSSL" library),
  24  * and distribute the linked executables. You must obey the GNU General Public
  25  * License in all respects for all of the code used other than "OpenSSL".  If you
  26  * modify file(s), you may extend this exception to your version of the file(s),
  27  * but you are not obligated to do so. If you do not wish to do so, delete this
  28  * exception statement from your version.
  29  */
  30
  31 #include "downloadmanager.h"
  32
  33 #include <algorithm>
  34
  35 #include <QByteArray>
  36 #include <QDateTime>
  37 #include <QDebug>
  38 #include <QNetworkAccessManager>
  39 #include <QNetworkCookie>
  40 #include <QNetworkCookieJar>
  41 #include <QNetworkProxy>
  42 #include <QNetworkReply>
  43 #include <QNetworkRequest>
  44 #include <QSslError>
  45 #include <QTimer>
  46 #include <QUrl>
  47
  48 #include "base/global.h"
  49 #include "base/logger.h"
  50 #include "base/preferences.h"
  51 #include "downloadhandlerimpl.h"
  52 #include "proxyconfigurationmanager.h"
  53
  54 using namespace std::chrono_literals;
  55
  56 namespace
  57 {
  58     // Disguise as browser to circumvent website blocking
  59     QByteArray getBrowserUserAgent()
  60     {
  61         // Firefox release calendar
  62         // https://whattrainisitnow.com/calendar/
  63         // https://wiki.mozilla.org/index.php?title=Release_Management/Calendar&redirect=no
  64
  65         static QByteArray ret;
  66         if (ret.isEmpty())
  67         {
  68             const std::chrono::time_point baseDate = std::chrono::sys_days(2024y / 04 / 16);
  69             const int baseVersion = 125;
  70
  71             const std::chrono::time_point nowDate = std::chrono::system_clock::now();
  72             const int nowVersion = baseVersion + std::chrono::duration_cast<std::chrono::months>(nowDate - baseDate).count();
  73
  74             QByteArray userAgentTemplate = QByteArrayLiteral("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:%1.0) Gecko/20100101 Firefox/%1.0");
  75             ret = userAgentTemplate.replace("%1", QByteArray::number(nowVersion));
  76         }
  77         return ret;
  78     }
  79 }
  80
  81 class Net::DownloadManager::NetworkCookieJar final : public QNetworkCookieJar
  82 {
  83 public:
  84     explicit NetworkCookieJar(QObject *parent = nullptr)
  85         : QNetworkCookieJar(parent)
  86     {
  87         const QDateTime now = QDateTime::currentDateTime();
  88         QList<QNetworkCookie> cookies = Preferences::instance()->getNetworkCookies();
  89         cookies.removeIf([&now](const QNetworkCookie &cookie)
  90         {
  91             return cookie.isSessionCookie() || (cookie.expirationDate() <= now);
  92         });
  93
  94         setAllCookies(cookies);
  95     }
  96
  97     ~NetworkCookieJar() override
  98     {
  99         const QDateTime now = QDateTime::currentDateTime();
 100         QList<QNetworkCookie> cookies = allCookies();
 101         cookies.removeIf([&now](const QNetworkCookie &cookie)
 102         {
 103             return cookie.isSessionCookie() || (cookie.expirationDate() <= now);
 104         });
 105
 106         Preferences::instance()->setNetworkCookies(cookies);
 107     }
 108
 109     using QNetworkCookieJar::allCookies;
 110     using QNetworkCookieJar::setAllCookies;
 111
 112     QList<QNetworkCookie> cookiesForUrl(const QUrl &url) const override
 113     {
 114         const QDateTime now = QDateTime::currentDateTime();
 115         QList<QNetworkCookie> cookies = QNetworkCookieJar::cookiesForUrl(url);
 116         cookies.removeIf([&now](const QNetworkCookie &cookie)
 117         {
 118             return !cookie.isSessionCookie() && (cookie.expirationDate() <= now);
 119         });
 120
 121         return cookies;
 122     }
 123
 124     bool setCookiesFromUrl(const QList<QNetworkCookie> &cookieList, const QUrl &url) override
 125     {
 126         const QDateTime now = QDateTime::currentDateTime();
 127         QList<QNetworkCookie> cookies = cookieList;
 128         cookies.removeIf([&now](const QNetworkCookie &cookie)
 129         {
 130             return !cookie.isSessionCookie() && (cookie.expirationDate() <= now);
 131         });
 132
 133         return QNetworkCookieJar::setCookiesFromUrl(cookies, url);
 134     }
 135 };
 136
 137 Net::DownloadManager *Net::DownloadManager::m_instance = nullptr;
 138
 139 Net::DownloadManager::DownloadManager(QObject *parent)
 140     : QObject(parent)
 141     , m_networkCookieJar {new NetworkCookieJar(this)}
 142     , m_networkManager {new QNetworkAccessManager(this)}
 143 {
 144     m_networkManager->setCookieJar(m_networkCookieJar);
 145     connect(m_networkManager, &QNetworkAccessManager::sslErrors, this
 146             , [](QNetworkReply *reply, const QList<QSslError> &errors)
 147     {
 148         QStringList errorList;
 149         for (const QSslError &error : errors)
 150             errorList += error.errorString();
 151
 152         QString errorMsg;
 153         if (!Preferences::instance()->isIgnoreSSLErrors())
 154         {
 155             errorMsg = tr("SSL error, URL: \"%1\", errors: \"%2\"");
 156         }
 157         else
 158         {
 159             errorMsg = tr("Ignoring SSL error, URL: \"%1\", errors: \"%2\"");
 160             // Ignore all SSL errors
 161             reply->ignoreSslErrors();
 162         }
 163
 164         LogMsg(errorMsg.arg(reply->url().toString(), errorList.join(u". ")), Log::WARNING);
 165     });
 166
 167     connect(ProxyConfigurationManager::instance(), &ProxyConfigurationManager::proxyConfigurationChanged
 168             , this, &DownloadManager::applyProxySettings);
 169     connect(Preferences::instance(), &Preferences::changed, this, &DownloadManager::applyProxySettings);
 170     applyProxySettings();
 171 }
 172
 173 void Net::DownloadManager::initInstance()
 174 {
 175     if (!m_instance)
 176         m_instance = new DownloadManager;
 177 }
 178
 179 void Net::DownloadManager::freeInstance()
 180 {
 181     delete m_instance;
 182     m_instance = nullptr;
 183 }
 184
 185 Net::DownloadManager *Net::DownloadManager::instance()
 186 {
 187     return m_instance;
 188 }
 189
 190 Net::DownloadHandler *Net::DownloadManager::download(const DownloadRequest &downloadRequest, const bool useProxy)
 191 {
 192     // Process download request
 193     const auto serviceID = ServiceID::fromURL(downloadRequest.url());
 194     const bool isSequentialService = m_sequentialServices.contains(serviceID);
 195
 196     auto *downloadHandler = new DownloadHandlerImpl(this, downloadRequest, useProxy);
 197     connect(downloadHandler, &DownloadHandler::finished, this, [this, serviceID, downloadHandler]
 198     {
 199         if (!downloadHandler->assignedNetworkReply())
 200         {
 201             // DownloadHandler was finished (canceled) before QNetworkReply was assigned,
 202             // so it's still in the queue. Just remove it from there.
 203             m_waitingJobs[serviceID].removeOne(downloadHandler);
 204         }
 205
 206         downloadHandler->deleteLater();
 207     });
 208
 209     if (isSequentialService && m_busyServices.contains(serviceID))
 210     {
 211         m_waitingJobs[serviceID].enqueue(downloadHandler);
 212     }
 213     else
 214     {
 215         qDebug("Downloading %s...", qUtf8Printable(downloadRequest.url()));
 216         if (isSequentialService)
 217             m_busyServices.insert(serviceID);
 218         processRequest(downloadHandler);
 219     }
 220
 221     return downloadHandler;
 222 }
 223
 224 void Net::DownloadManager::registerSequentialService(const Net::ServiceID &serviceID, const std::chrono::seconds delay)
 225 {
 226     m_sequentialServices.insert(serviceID, delay);
 227 }
 228
 229 QList<QNetworkCookie> Net::DownloadManager::cookiesForUrl(const QUrl &url) const
 230 {
 231     return m_networkCookieJar->cookiesForUrl(url);
 232 }
 233
 234 bool Net::DownloadManager::setCookiesFromUrl(const QList<QNetworkCookie> &cookieList, const QUrl &url)
 235 {
 236     return m_networkCookieJar->setCookiesFromUrl(cookieList, url);
 237 }
 238
 239 QList<QNetworkCookie> Net::DownloadManager::allCookies() const
 240 {
 241     return m_networkCookieJar->allCookies();
 242 }
 243
 244 void Net::DownloadManager::setAllCookies(const QList<QNetworkCookie> &cookieList)
 245 {
 246     m_networkCookieJar->setAllCookies(cookieList);
 247 }
 248
 249 bool Net::DownloadManager::deleteCookie(const QNetworkCookie &cookie)
 250 {
 251     return m_networkCookieJar->deleteCookie(cookie);
 252 }
 253
 254 bool Net::DownloadManager::hasSupportedScheme(const QString &url)
 255 {
 256     const QStringList schemes = QNetworkAccessManager().supportedSchemes();
 257     return std::any_of(schemes.cbegin(), schemes.cend(), [&url](const QString &scheme)
 258     {
 259         return url.startsWith((scheme + u':'), Qt::CaseInsensitive);
 260     });
 261 }
 262
 263 void Net::DownloadManager::applyProxySettings()
 264 {
 265     const auto *proxyManager = ProxyConfigurationManager::instance();
 266     const ProxyConfiguration proxyConfig = proxyManager->proxyConfiguration();
 267
 268     m_proxy = QNetworkProxy(QNetworkProxy::NoProxy);
 269
 270     if ((proxyConfig.type == Net::ProxyType::None) || (proxyConfig.type == ProxyType::SOCKS4))
 271         return;
 272
 273     // Proxy enabled
 274     if (proxyConfig.type == ProxyType::SOCKS5)
 275     {
 276         qDebug() << Q_FUNC_INFO << "using SOCKS proxy";
 277         m_proxy.setType(QNetworkProxy::Socks5Proxy);
 278     }
 279     else
 280     {
 281         qDebug() << Q_FUNC_INFO << "using HTTP proxy";
 282         m_proxy.setType(QNetworkProxy::HttpProxy);
 283     }
 284
 285     m_proxy.setHostName(proxyConfig.ip);
 286     m_proxy.setPort(proxyConfig.port);
 287
 288     // Authentication?
 289     if (proxyConfig.authEnabled)
 290     {
 291         qDebug("Proxy requires authentication, authenticating...");
 292         m_proxy.setUser(proxyConfig.username);
 293         m_proxy.setPassword(proxyConfig.password);
 294     }
 295
 296     if (proxyConfig.hostnameLookupEnabled)
 297         m_proxy.setCapabilities(m_proxy.capabilities() | QNetworkProxy::HostNameLookupCapability);
 298     else
 299         m_proxy.setCapabilities(m_proxy.capabilities() & ~QNetworkProxy::HostNameLookupCapability);
 300 }
 301
 302 void Net::DownloadManager::processWaitingJobs(const ServiceID &serviceID)
 303 {
 304     const auto waitingJobsIter = m_waitingJobs.find(serviceID);
 305     if ((waitingJobsIter == m_waitingJobs.end()) || waitingJobsIter.value().isEmpty())
 306     {
 307         // No more waiting jobs for given ServiceID
 308         m_busyServices.remove(serviceID);
 309         return;
 310     }
 311
 312     auto *handler = waitingJobsIter.value().dequeue();
 313     qDebug("Downloading %s...", qUtf8Printable(handler->url()));
 314     processRequest(handler);
 315 }
 316
 317 void Net::DownloadManager::processRequest(DownloadHandlerImpl *downloadHandler)
 318 {
 319     m_networkManager->setProxy((downloadHandler->useProxy() == true) ? m_proxy : QNetworkProxy(QNetworkProxy::NoProxy));
 320
 321     const DownloadRequest downloadRequest = downloadHandler->downloadRequest();
 322     QNetworkRequest request {downloadRequest.url()};
 323
 324     if (downloadRequest.userAgent().isEmpty())
 325         request.setRawHeader("User-Agent", getBrowserUserAgent());
 326     else
 327         request.setRawHeader("User-Agent", downloadRequest.userAgent().toUtf8());
 328
 329     // Spoof HTTP Referer to allow adding torrent link from Torcache/KickAssTorrents
 330     request.setRawHeader("Referer", request.url().toEncoded().data());
 331 #ifdef QT_NO_COMPRESS
 332     // The macro "QT_NO_COMPRESS" defined in QT will disable the zlib related features
 333     // and reply data auto-decompression in QT will also be disabled. But we can support
 334     // gzip encoding and manually decompress the reply data.
 335     request.setRawHeader("Accept-Encoding", "gzip");
 336 #endif
 337     // Qt doesn't support Magnet protocol so we need to handle redirections manually
 338     request.setAttribute(QNetworkRequest::RedirectPolicyAttribute, QNetworkRequest::ManualRedirectPolicy);
 339
 340     request.setTransferTimeout();
 341
 342     QNetworkReply *reply = m_networkManager->get(request);
 343     connect(reply, &QNetworkReply::finished, this, [this, serviceID = ServiceID::fromURL(downloadHandler->url())]
 344     {
 345         QTimer::singleShot(m_sequentialServices.value(serviceID, 0s), this, [this, serviceID] { processWaitingJobs(serviceID); });
 346     });
 347     downloadHandler->assignNetworkReply(reply);
 348 }
 349
 350 Net::DownloadRequest::DownloadRequest(const QString &url)
 351     : m_url {url}
 352 {
 353 }
 354
 355 QString Net::DownloadRequest::url() const
 356 {
 357     return m_url;
 358 }
 359
 360 Net::DownloadRequest &Net::DownloadRequest::url(const QString &value)
 361 {
 362     m_url = value;
 363     return *this;
 364 }
 365
 366 QString Net::DownloadRequest::userAgent() const
 367 {
 368     return m_userAgent;
 369 }
 370
 371 Net::DownloadRequest &Net::DownloadRequest::userAgent(const QString &value)
 372 {
 373     m_userAgent = value;
 374     return *this;
 375 }
 376
 377 qint64 Net::DownloadRequest::limit() const
 378 {
 379     return m_limit;
 380 }
 381
 382 Net::DownloadRequest &Net::DownloadRequest::limit(const qint64 value)
 383 {
 384     m_limit = value;
 385     return *this;
 386 }
 387
 388 bool Net::DownloadRequest::saveToFile() const
 389 {
 390     return m_saveToFile;
 391 }
 392
 393 Net::DownloadRequest &Net::DownloadRequest::saveToFile(const bool value)
 394 {
 395     m_saveToFile = value;
 396     return *this;
 397 }
 398
 399 Path Net::DownloadRequest::destFileName() const
 400 {
 401     return m_destFileName;
 402 }
 403
 404 Net::DownloadRequest &Net::DownloadRequest::destFileName(const Path &value)
 405 {
 406     m_destFileName = value;
 407     return *this;
 408 }
 409
 410 Net::ServiceID Net::ServiceID::fromURL(const QUrl &url)
 411 {
 412     return {url.host(), url.port(80)};
 413 }
 414
 415 std::size_t Net::qHash(const ServiceID &serviceID, const std::size_t seed)
 416 {
 417     return qHashMulti(seed, serviceID.hostName, serviceID.port);
 418 }
 419
 420 bool Net::operator==(const ServiceID &lhs, const ServiceID &rhs)
 421 {
 422     return ((lhs.hostName == rhs.hostName) && (lhs.port == rhs.port));
 423 }