src/base/net/downloadmanager.cpp

   1 /*
   2  * Bittorrent Client using Qt and libtorrent.
   3  * Copyright (C) 2024  Jonathan Ketchker
   4  * Copyright (C) 2015-2024  Vladimir Golovnev <glassez@yandex.ru>
   5  * Copyright (C) 2006  Christophe Dumez <chris@qbittorrent.org>
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; either version 2
  10  * of the License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  20  *
  21  * In addition, as a special exception, the copyright holders give permission to
  22  * link this program with the OpenSSL project's "OpenSSL" library (or with
  23  * modified versions of it that use the same license as the "OpenSSL" library),
  24  * and distribute the linked executables. You must obey the GNU General Public
  25  * License in all respects for all of the code used other than "OpenSSL".  If you
  26  * modify file(s), you may extend this exception to your version of the file(s),
  27  * but you are not obligated to do so. If you do not wish to do so, delete this
  28  * exception statement from your version.
  29  */
  30
  31 #include "downloadmanager.h"
  32
  33 #include <algorithm>
  34
  35 #include <QByteArray>
  36 #include <QDateTime>
  37 #include <QDebug>
  38 #include <QNetworkAccessManager>
  39 #include <QNetworkCookie>
  40 #include <QNetworkCookieJar>
  41 #include <QNetworkProxy>
  42 #include <QNetworkReply>
  43 #include <QNetworkRequest>
  44 #include <QSslError>
  45 #include <QTimer>
  46 #include <QUrl>
  47
  48 #include "base/global.h"
  49 #include "base/logger.h"
  50 #include "base/preferences.h"
  51 #include "downloadhandlerimpl.h"
  52 #include "proxyconfigurationmanager.h"
  53
  54 using namespace std::chrono_literals;
  55
  56 namespace
  57 {
  58     // Disguise as browser to circumvent website blocking
  59     QByteArray getBrowserUserAgent()
  60     {
  61         // Firefox release calendar
  62         // https://whattrainisitnow.com/calendar/
  63         // https://wiki.mozilla.org/index.php?title=Release_Management/Calendar&redirect=no
  64
  65         static QByteArray ret;
  66         if (ret.isEmpty())
  67         {
  68             const std::chrono::time_point baseDate = std::chrono::sys_days(2024y / 04 / 16);
  69             const int baseVersion = 125;
  70
  71             const std::chrono::time_point nowDate = std::chrono::system_clock::now();
  72             const int nowVersion = baseVersion + std::chrono::duration_cast<std::chrono::months>(nowDate - baseDate).count();
  73
  74             QByteArray userAgentTemplate = QByteArrayLiteral("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:%1.0) Gecko/20100101 Firefox/%1.0");
  75             ret = userAgentTemplate.replace("%1", QByteArray::number(nowVersion));
  76         }
  77         return ret;
  78     }
  79 }
  80
  81 class Net::DownloadManager::NetworkCookieJar final : public QNetworkCookieJar
  82 {
  83 public:
  84     explicit NetworkCookieJar(QObject *parent = nullptr)
  85         : QNetworkCookieJar(parent)
  86     {
  87         const QDateTime now = QDateTime::currentDateTime();
  88         QList<QNetworkCookie> cookies = Preferences::instance()->getNetworkCookies();
  89         cookies.removeIf([&now](const QNetworkCookie &cookie)
  90         {
  91             return cookie.isSessionCookie() || (cookie.expirationDate() <= now);
  92         });
  93
  94         setAllCookies(cookies);
  95     }
  96
  97     ~NetworkCookieJar() override
  98     {
  99         const QDateTime now = QDateTime::currentDateTime();
 100         QList<QNetworkCookie> cookies = allCookies();
 101         cookies.removeIf([&now](const QNetworkCookie &cookie)
 102         {
 103             return cookie.isSessionCookie() || (cookie.expirationDate() <= now);
 104         });
 105
 106         Preferences::instance()->setNetworkCookies(cookies);
 107     }
 108
 109     using QNetworkCookieJar::allCookies;
 110     using QNetworkCookieJar::setAllCookies;
 111
 112     QList<QNetworkCookie> cookiesForUrl(const QUrl &url) const override
 113     {
 114         const QDateTime now = QDateTime::currentDateTime();
 115         QList<QNetworkCookie> cookies = QNetworkCookieJar::cookiesForUrl(url);
 116         cookies.removeIf([&now](const QNetworkCookie &cookie)
 117         {
 118             return !cookie.isSessionCookie() && (cookie.expirationDate() <= now);
 119         });
 120
 121         return cookies;
 122     }
 123
 124     bool setCookiesFromUrl(const QList<QNetworkCookie> &cookieList, const QUrl &url) override
 125     {
 126         const QDateTime now = QDateTime::currentDateTime();
 127         QList<QNetworkCookie> cookies = cookieList;
 128         cookies.removeIf([&now](const QNetworkCookie &cookie)
 129         {
 130             return !cookie.isSessionCookie() && (cookie.expirationDate() <= now);
 131         });
 132
 133         return QNetworkCookieJar::setCookiesFromUrl(cookies, url);
 134     }
 135 };
 136
 137 Net::DownloadManager *Net::DownloadManager::m_instance = nullptr;
 138
 139 Net::DownloadManager::DownloadManager(QObject *parent)
 140     : QObject(parent)
 141     , m_networkCookieJar {new NetworkCookieJar(this)}
 142     , m_networkManager {new QNetworkAccessManager(this)}
 143 {
 144     m_networkManager->setCookieJar(m_networkCookieJar);
 145     connect(m_networkManager, &QNetworkAccessManager::sslErrors, this
 146             , [](QNetworkReply *reply, const QList<QSslError> &errors)
 147     {
 148         QStringList errorList;
 149         for (const QSslError &error : errors)
 150             errorList += error.errorString();
 151         LogMsg(tr("Ignoring SSL error, URL: \"%1\", errors: \"%2\"").arg(reply->url().toString(), errorList.join(u". ")), Log::WARNING);
 152
 153         // Ignore all SSL errors
 154         reply->ignoreSslErrors();
 155     });
 156
 157     connect(ProxyConfigurationManager::instance(), &ProxyConfigurationManager::proxyConfigurationChanged
 158             , this, &DownloadManager::applyProxySettings);
 159     connect(Preferences::instance(), &Preferences::changed, this, &DownloadManager::applyProxySettings);
 160     applyProxySettings();
 161 }
 162
 163 void Net::DownloadManager::initInstance()
 164 {
 165     if (!m_instance)
 166         m_instance = new DownloadManager;
 167 }
 168
 169 void Net::DownloadManager::freeInstance()
 170 {
 171     delete m_instance;
 172     m_instance = nullptr;
 173 }
 174
 175 Net::DownloadManager *Net::DownloadManager::instance()
 176 {
 177     return m_instance;
 178 }
 179
 180 Net::DownloadHandler *Net::DownloadManager::download(const DownloadRequest &downloadRequest, const bool useProxy)
 181 {
 182     // Process download request
 183     const auto serviceID = ServiceID::fromURL(downloadRequest.url());
 184     const bool isSequentialService = m_sequentialServices.contains(serviceID);
 185
 186     auto *downloadHandler = new DownloadHandlerImpl(this, downloadRequest, useProxy);
 187     connect(downloadHandler, &DownloadHandler::finished, this, [this, serviceID, downloadHandler]
 188     {
 189         if (!downloadHandler->assignedNetworkReply())
 190         {
 191             // DownloadHandler was finished (canceled) before QNetworkReply was assigned,
 192             // so it's still in the queue. Just remove it from there.
 193             m_waitingJobs[serviceID].removeOne(downloadHandler);
 194         }
 195
 196         downloadHandler->deleteLater();
 197     });
 198
 199     if (isSequentialService && m_busyServices.contains(serviceID))
 200     {
 201         m_waitingJobs[serviceID].enqueue(downloadHandler);
 202     }
 203     else
 204     {
 205         qDebug("Downloading %s...", qUtf8Printable(downloadRequest.url()));
 206         if (isSequentialService)
 207             m_busyServices.insert(serviceID);
 208         processRequest(downloadHandler);
 209     }
 210
 211     return downloadHandler;
 212 }
 213
 214 void Net::DownloadManager::registerSequentialService(const Net::ServiceID &serviceID, const std::chrono::seconds delay)
 215 {
 216     m_sequentialServices.insert(serviceID, delay);
 217 }
 218
 219 QList<QNetworkCookie> Net::DownloadManager::cookiesForUrl(const QUrl &url) const
 220 {
 221     return m_networkCookieJar->cookiesForUrl(url);
 222 }
 223
 224 bool Net::DownloadManager::setCookiesFromUrl(const QList<QNetworkCookie> &cookieList, const QUrl &url)
 225 {
 226     return m_networkCookieJar->setCookiesFromUrl(cookieList, url);
 227 }
 228
 229 QList<QNetworkCookie> Net::DownloadManager::allCookies() const
 230 {
 231     return m_networkCookieJar->allCookies();
 232 }
 233
 234 void Net::DownloadManager::setAllCookies(const QList<QNetworkCookie> &cookieList)
 235 {
 236     m_networkCookieJar->setAllCookies(cookieList);
 237 }
 238
 239 bool Net::DownloadManager::deleteCookie(const QNetworkCookie &cookie)
 240 {
 241     return m_networkCookieJar->deleteCookie(cookie);
 242 }
 243
 244 bool Net::DownloadManager::hasSupportedScheme(const QString &url)
 245 {
 246     const QStringList schemes = QNetworkAccessManager().supportedSchemes();
 247     return std::any_of(schemes.cbegin(), schemes.cend(), [&url](const QString &scheme)
 248     {
 249         return url.startsWith((scheme + u':'), Qt::CaseInsensitive);
 250     });
 251 }
 252
 253 void Net::DownloadManager::applyProxySettings()
 254 {
 255     const auto *proxyManager = ProxyConfigurationManager::instance();
 256     const ProxyConfiguration proxyConfig = proxyManager->proxyConfiguration();
 257
 258     m_proxy = QNetworkProxy(QNetworkProxy::NoProxy);
 259
 260     if ((proxyConfig.type == Net::ProxyType::None) || (proxyConfig.type == ProxyType::SOCKS4))
 261         return;
 262
 263     // Proxy enabled
 264     if (proxyConfig.type == ProxyType::SOCKS5)
 265     {
 266         qDebug() << Q_FUNC_INFO << "using SOCKS proxy";
 267         m_proxy.setType(QNetworkProxy::Socks5Proxy);
 268     }
 269     else
 270     {
 271         qDebug() << Q_FUNC_INFO << "using HTTP proxy";
 272         m_proxy.setType(QNetworkProxy::HttpProxy);
 273     }
 274
 275     m_proxy.setHostName(proxyConfig.ip);
 276     m_proxy.setPort(proxyConfig.port);
 277
 278     // Authentication?
 279     if (proxyConfig.authEnabled)
 280     {
 281         qDebug("Proxy requires authentication, authenticating...");
 282         m_proxy.setUser(proxyConfig.username);
 283         m_proxy.setPassword(proxyConfig.password);
 284     }
 285
 286     if (proxyConfig.hostnameLookupEnabled)
 287         m_proxy.setCapabilities(m_proxy.capabilities() | QNetworkProxy::HostNameLookupCapability);
 288     else
 289         m_proxy.setCapabilities(m_proxy.capabilities() & ~QNetworkProxy::HostNameLookupCapability);
 290 }
 291
 292 void Net::DownloadManager::processWaitingJobs(const ServiceID &serviceID)
 293 {
 294     const auto waitingJobsIter = m_waitingJobs.find(serviceID);
 295     if ((waitingJobsIter == m_waitingJobs.end()) || waitingJobsIter.value().isEmpty())
 296     {
 297         // No more waiting jobs for given ServiceID
 298         m_busyServices.remove(serviceID);
 299         return;
 300     }
 301
 302     auto *handler = waitingJobsIter.value().dequeue();
 303     qDebug("Downloading %s...", qUtf8Printable(handler->url()));
 304     processRequest(handler);
 305 }
 306
 307 void Net::DownloadManager::processRequest(DownloadHandlerImpl *downloadHandler)
 308 {
 309     m_networkManager->setProxy((downloadHandler->useProxy() == true) ? m_proxy : QNetworkProxy(QNetworkProxy::NoProxy));
 310
 311     const DownloadRequest downloadRequest = downloadHandler->downloadRequest();
 312     QNetworkRequest request {downloadRequest.url()};
 313
 314     if (downloadRequest.userAgent().isEmpty())
 315         request.setRawHeader("User-Agent", getBrowserUserAgent());
 316     else
 317         request.setRawHeader("User-Agent", downloadRequest.userAgent().toUtf8());
 318
 319     // Spoof HTTP Referer to allow adding torrent link from Torcache/KickAssTorrents
 320     request.setRawHeader("Referer", request.url().toEncoded().data());
 321 #ifdef QT_NO_COMPRESS
 322     // The macro "QT_NO_COMPRESS" defined in QT will disable the zlib related features
 323     // and reply data auto-decompression in QT will also be disabled. But we can support
 324     // gzip encoding and manually decompress the reply data.
 325     request.setRawHeader("Accept-Encoding", "gzip");
 326 #endif
 327     // Qt doesn't support Magnet protocol so we need to handle redirections manually
 328     request.setAttribute(QNetworkRequest::RedirectPolicyAttribute, QNetworkRequest::ManualRedirectPolicy);
 329
 330     request.setTransferTimeout();
 331
 332     QNetworkReply *reply = m_networkManager->get(request);
 333     connect(reply, &QNetworkReply::finished, this, [this, serviceID = ServiceID::fromURL(downloadHandler->url())]
 334     {
 335         QTimer::singleShot(m_sequentialServices.value(serviceID, 0s), this, [this, serviceID] { processWaitingJobs(serviceID); });
 336     });
 337     downloadHandler->assignNetworkReply(reply);
 338 }
 339
 340 Net::DownloadRequest::DownloadRequest(const QString &url)
 341     : m_url {url}
 342 {
 343 }
 344
 345 QString Net::DownloadRequest::url() const
 346 {
 347     return m_url;
 348 }
 349
 350 Net::DownloadRequest &Net::DownloadRequest::url(const QString &value)
 351 {
 352     m_url = value;
 353     return *this;
 354 }
 355
 356 QString Net::DownloadRequest::userAgent() const
 357 {
 358     return m_userAgent;
 359 }
 360
 361 Net::DownloadRequest &Net::DownloadRequest::userAgent(const QString &value)
 362 {
 363     m_userAgent = value;
 364     return *this;
 365 }
 366
 367 qint64 Net::DownloadRequest::limit() const
 368 {
 369     return m_limit;
 370 }
 371
 372 Net::DownloadRequest &Net::DownloadRequest::limit(const qint64 value)
 373 {
 374     m_limit = value;
 375     return *this;
 376 }
 377
 378 bool Net::DownloadRequest::saveToFile() const
 379 {
 380     return m_saveToFile;
 381 }
 382
 383 Net::DownloadRequest &Net::DownloadRequest::saveToFile(const bool value)
 384 {
 385     m_saveToFile = value;
 386     return *this;
 387 }
 388
 389 Path Net::DownloadRequest::destFileName() const
 390 {
 391     return m_destFileName;
 392 }
 393
 394 Net::DownloadRequest &Net::DownloadRequest::destFileName(const Path &value)
 395 {
 396     m_destFileName = value;
 397     return *this;
 398 }
 399
 400 Net::ServiceID Net::ServiceID::fromURL(const QUrl &url)
 401 {
 402     return {url.host(), url.port(80)};
 403 }
 404
 405 std::size_t Net::qHash(const ServiceID &serviceID, const std::size_t seed)
 406 {
 407     return qHashMulti(seed, serviceID.hostName, serviceID.port);
 408 }
 409
 410 bool Net::operator==(const ServiceID &lhs, const ServiceID &rhs)
 411 {
 412     return ((lhs.hostName == rhs.hostName) && (lhs.port == rhs.port));
 413 }