companion/src/process_sync.cpp

   1 /*
   2  * Copyright (C) OpenTX
   3  *
   4  * Based on code named
   5  *   th9x - http://code.google.com/p/th9x
   6  *   er9x - http://code.google.com/p/er9x
   7  *   gruvin9x - http://code.google.com/p/gruvin9x
   8  *
   9  * License GPLv2: http://www.gnu.org/licenses/gpl-2.0.html
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License version 2 as
  13  * published by the Free Software Foundation.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  */
  20
  21 #include "process_sync.h"
  22
  23 #include <QApplication>
  24 #include <QCryptographicHash>
  25 #include <QElapsedTimer>
  26 #include <QMutexLocker>
  27
  28 #define SYNC_MAX_ERRORS       50  // give up after this many errors per destination
  29
  30 // a flood of log messages can make the UI unresponsive so we'll introduce a dynamic sleep period based on log frequency (values in [us])
  31 #define PAUSE_FACTOR          60UL
  32 #define PAUSE_RECOVERY        (PAUSE_FACTOR / 3 * 2)
  33 #define PAUSE_MINTM           100UL
  34 #define PAUSE_MAXTM           75000UL
  35
  36 #if (QT_VERSION < QT_VERSION_CHECK(5, 5, 0))
  37   #define QtInfoMsg    QtMsgType(4)
  38 #endif
  39
  40 #define PRINT_CREATE(str)     emitProgressMessage((str), QtInfoMsg)
  41 #define PRINT_REPLACE(str)    emitProgressMessage((str), QtWarningMsg)
  42 //#define PRINT_DELETE(str)   emitProgressMessage((str), QtCriticalMsg)  // unused
  43 #define PRINT_ERROR(str)      emitProgressMessage((str), QtFatalMsg)
  44 #define PRINT_SKIP(str)       emitProgressMessage((str), QtDebugMsg)
  45 #define PRINT_INFO(str)       emit progressMessage((str))                // this is always emitted regardless of logLevel option
  46 #define PRINT_SEP()           PRINT_INFO(QString(70, '='))
  47
  48 #ifdef Q_OS_WIN
  49   extern Q_CORE_EXPORT int qt_ntfs_permission_lookup;
  50   #define FILTER_RE_SYNTX     QRegExp::Wildcard
  51 #else
  52   #define FILTER_RE_SYNTX     QRegExp::WildcardUnix
  53 #endif
  54
  55 SyncProcess::SyncProcess(const SyncProcess::SyncOptions & options) :
  56   m_options(options),
  57   m_pauseTime(PAUSE_MINTM),
  58   stopping(false)
  59 {
  60   qRegisterMetaType<SyncProcess::SyncStatus>();
  61
  62   if (m_options.compareType == OVERWR_ALWAYS && (m_options.direction == SYNC_A2B_B2A || m_options.direction == SYNC_B2A_A2B))
  63     m_options.compareType = OVERWR_IF_DIFF;
  64
  65   m_dirFilters = QDir::Filters(m_options.dirFilterFlags);
  66   if (!(m_dirFilters & QDir::Dirs))
  67     m_dirFilters &= ~(QDir::AllDirs);
  68
  69   if (!m_options.includeFilter.isEmpty() && m_options.includeFilter != "*")
  70     m_dirIteratorFilters = m_options.includeFilter.split(',', QString::SkipEmptyParts);
  71
  72   if (!m_options.excludeFilter.isEmpty()) {
  73     for (const QString & f : m_options.excludeFilter.split(',', QString::SkipEmptyParts))
  74       m_excludeFilters.append(QRegExp(f, ((m_dirFilters & QDir::CaseSensitive) ? Qt::CaseSensitive : Qt::CaseInsensitive), FILTER_RE_SYNTX));
  75   }
  76
  77   if (m_options.flags & OPT_DRY_RUN)
  78     testRunStr = tr("[TEST RUN] ");
  79
  80   //qDebug() << m_options;
  81 #ifdef Q_OS_WIN
  82   qt_ntfs_permission_lookup++;  // global enable NTFS permissions checking
  83 #endif
  84 }
  85
  86 SyncProcess::~SyncProcess()
  87 {
  88 #ifdef Q_OS_WIN
  89   qt_ntfs_permission_lookup--;  // global revert NTFS permissions checking
  90 #endif
  91 }
  92
  93 void SyncProcess::stop()
  94 {
  95   QWriteLocker locker(&stopReqMutex);
  96   stopping = true;
  97 }
  98
  99 bool SyncProcess::isStopRequsted()
 100 {
 101   QReadLocker locker(&stopReqMutex);
 102   return stopping;
 103 }
 104
 105 void SyncProcess::run()
 106 {
 107   const QString folderA = (m_options.direction == SYNC_B2A_A2B ? m_options.folderB : m_options.folderA);
 108   const QString folderB = (m_options.direction == SYNC_B2A_A2B ? m_options.folderA : m_options.folderB);
 109   const SyncDirection direction = (m_options.direction == SYNC_B2A_A2B ? SYNC_A2B_B2A : SyncDirection(m_options.direction));
 110   const QString gathering = tr("Gathering file information for %1...");
 111   const QString noFiles = tr("No files found in %1");
 112   int count = 0;
 113
 114   m_stat.clear();
 115   m_startTime = QDateTime::currentDateTime();
 116
 117   emit started();
 118   emit fileCountChanged(0);
 119   emit statusUpdate(m_stat);
 120
 121   if (direction == SYNC_A2B_B2A || direction == SYNC_A2B) {
 122     emit statusMessage(gathering.arg(folderA));
 123     count = getFilesCount(folderA);
 124
 125     if (isStopRequsted())
 126       goto endrun;
 127
 128     if (count) {
 129       m_stat.count = count;
 130       if (m_options.direction == SYNC_A2B_B2A)
 131         count *= 2;  // assume this direction is only 50% of total, exact will be calculated later
 132       emit fileCountChanged(count);
 133       updateDir(folderA, folderB);
 134       if (isStopRequsted())
 135         goto endrun;
 136     }
 137     else {
 138       PRINT_INFO(noFiles.arg(folderA));
 139       PRINT_SEP();
 140     }
 141   }
 142
 143   if (direction == SYNC_A2B_B2A || direction == SYNC_B2A) {
 144     emit statusMessage(gathering.arg(folderB));
 145     count = getFilesCount(folderB);
 146
 147     if (isStopRequsted())
 148       goto endrun;
 149
 150     m_stat.count += count;
 151     emit fileCountChanged(m_stat.count);
 152
 153     if (count) {
 154       updateDir(folderB, folderA);
 155     }
 156     else {
 157       PRINT_INFO(noFiles.arg(folderB));
 158       PRINT_SEP();
 159     }
 160   }
 161
 162   if (!m_stat.count) {
 163     emit statusMessage(tr("Synchronization failed, nothing found to copy."), QtWarningMsg);
 164     emit finished();
 165     return;
 166   }
 167
 168   endrun:
 169   finish();
 170 }
 171
 172 void SyncProcess::finish()
 173 {
 174   const lldiv_t elapsed = lldiv(m_startTime.secsTo(QDateTime::currentDateTime()), 60);
 175   QString endStr = testRunStr;
 176   if (m_stat.index < m_stat.count)
 177     endStr.append(tr("Synchronization aborted at %1 of %2 files.").arg(m_stat.index).arg(m_stat.count));
 178   else
 179     endStr.append(tr("Synchronization finished with %1 files in %2m %3s.").arg(m_stat.count).arg(elapsed.quot).arg(elapsed.rem));
 180   emit statusMessage(endStr);
 181   emit finished();
 182 }
 183
 184 SyncProcess::FileFilterResult SyncProcess::fileFilter(const QFileInfo & fileInfo)
 185 {
 186   // Windows Junctions (mount points) are not detected as links (QTBUG-45344), but that's OK since they're really "hard links."
 187   const bool chkDirLnk = ((m_dirFilters & QDir::NoSymLinks) && !(m_dirFilters & QDir::AllDirs)) || ((m_options.flags & OPT_SKIP_DIR_LINKS) && fileInfo.isDir());
 188   if ((chkDirLnk || ((m_dirFilters & QDir::NoSymLinks) && fileInfo.isFile())) && QFileInfo(fileInfo.absoluteFilePath()).isSymLink())  // MUST create a new QFileInfo here (QTBUG-69001)
 189     return FILE_LINK_IGNORE;
 190
 191   if (m_options.maxFileSize > 0 && fileInfo.isFile() && fileInfo.size() > m_options.maxFileSize)
 192     return FILE_OVERSIZE;
 193
 194   if (!m_excludeFilters.isEmpty() && (!(m_dirFilters & QDir::AllDirs) || fileInfo.isFile())) {
 195     for (QVector<QRegExp>::const_iterator it = m_excludeFilters.constBegin(), end = m_excludeFilters.constEnd(); it != end; ++it) {
 196       if (QRegExp(*it).exactMatch(fileInfo.fileName()))
 197         return FILE_EXCLUDE;
 198     }
 199   }
 200
 201   return FILE_ALLOW;
 202 }
 203
 204 QFileInfoList SyncProcess::dirInfoList(const QString & directory)
 205 {
 206   QDir::Filters flt = m_dirFilters;
 207   if (!(flt & QDir::Dirs) && (m_options.flags & OPT_RECURSIVE))
 208     flt |= ((m_options.dirFilterFlags & QDir::AllDirs) ? QDir::AllDirs : QDir::Dirs);
 209   return QDir(directory).entryInfoList(m_dirIteratorFilters, flt, QDir::Name | QDir::DirsLast);
 210 }
 211
 212 void SyncProcess::pushDirEntries(const QFileInfo & fileInfo, QMutableListIterator<QFileInfo> & it)
 213 {
 214   if ((m_options.flags & OPT_RECURSIVE) && fileInfo.isDir()) {
 215     for (const QFileInfo &fi : dirInfoList(fileInfo.absoluteFilePath()))
 216       it.insert(fi);
 217   }
 218 }
 219
 220 int SyncProcess::getFilesCount(const QString & directory)
 221 {
 222   if (!QFile::exists(directory))
 223     return 0;
 224
 225   int result = 0;
 226   QFileInfoList infoList = dirInfoList(directory);
 227   QMutableListIterator<QFileInfo> it(infoList);
 228   it.toBack();
 229   while (it.hasPrevious() && !isStopRequsted()) {
 230     const QFileInfo fi(it.previous());
 231     it.remove();
 232     if (fileFilter(fi) == FILE_ALLOW) {
 233       pushDirEntries(fi, it);
 234       if (fi.isFile()) {
 235         result++;
 236       }
 237     }
 238     QApplication::processEvents();
 239   }
 240   return result;
 241 }
 242
 243 void SyncProcess::updateDir(const QString & source, const QString & destination)
 244 {
 245   SyncStatus pStat = m_stat;
 246   const QDir srcDir(source), dstDir(destination);
 247   FileFilterResult ffr;
 248   emit statusMessage(testRunStr % tr("Synchronizing: %1\n    To: %2").arg(source, destination));
 249   PRINT_INFO(testRunStr % tr("Starting synchronization:\n  %1 -> %2\n").arg(source, destination));
 250
 251   QFileInfoList infoList = dirInfoList(source);
 252   QMutableListIterator<QFileInfo> it(infoList);
 253   it.toBack();
 254   while (it.hasPrevious() && !isStopRequsted()) {
 255     const QFileInfo fi(it.previous());
 256     it.remove();
 257     if ((ffr = fileFilter(fi)) == FILE_ALLOW) {
 258       pushDirEntries(fi, it);
 259       if ((m_dirFilters & QDir::Dirs) || fi.isFile()) {
 260         updateEntry(fi.filePath(), srcDir, dstDir);
 261         if (fi.isFile())
 262           ++m_stat.index;
 263         emit statusUpdate(m_stat);
 264         if (m_stat.errored - pStat.errored > SYNC_MAX_ERRORS) {
 265           PRINT_ERROR(tr("\nToo many errors, giving up."));
 266           break;
 267         }
 268       }
 269     }
 270     else if (m_options.logLevel == QtDebugMsg) {
 271       switch (ffr) {
 272         case FILE_OVERSIZE:
 273           PRINT_SKIP(tr("Skipping large file: %1 (%2KB)").arg(fi.fileName()).arg(int(fi.size() / 1024)));
 274           break;
 275         case FILE_EXCLUDE:
 276           PRINT_SKIP(tr("Skipping filtered file: %1").arg(fi.fileName()));
 277           break;
 278         case FILE_LINK_IGNORE:
 279           PRINT_SKIP(tr("Skipping linked file: %1").arg(fi.fileName()));
 280           break;
 281         default:
 282           break;
 283       }
 284       // don't count as skipped because these weren't included in the total file count to begin with
 285     }
 286     // throttle if needed
 287     m_pauseTime = qMax(m_pauseTime - PAUSE_RECOVERY, PAUSE_MINTM);
 288     pause();
 289   }
 290
 291   QString endStr = "\n" % testRunStr;
 292   if (isStopRequsted())
 293     endStr.append(tr("Aborted synchronization of:"));
 294   else
 295     endStr.append(tr("Finished synchronizing:"));
 296   endStr.append(QString("\n  %1 -> %2\n  ").arg(source, destination));
 297   endStr.append(tr("Created: %1; Updated: %2; Skipped: %3; Errors: %4;").arg(m_stat.created-pStat.created).arg(m_stat.updated-pStat.updated).arg(m_stat.skipped-pStat.skipped).arg(m_stat.errored-pStat.errored));
 298   PRINT_INFO(endStr);
 299   PRINT_SEP();
 300 }
 301
 302 bool SyncProcess::updateEntry(const QString & entry, const QDir & source, const QDir & destination)
 303 {
 304   const QString srcPath = QDir::toNativeSeparators(source.absoluteFilePath(entry));
 305   const QString destPath = QDir::toNativeSeparators(destination.absoluteFilePath(source.relativeFilePath(entry)));
 306   const QFileInfo sourceInfo(srcPath);
 307   const QFileInfo destInfo(destPath);
 308   static QString lastMkPath;
 309
 310   // check if this is a directory OR if we're copying a file with a path which doesn't exist yet.
 311   if (sourceInfo.isDir() || !destInfo.absoluteDir().exists()) {
 312     const QString mkPath = sourceInfo.isDir() ? destPath : QDir::toNativeSeparators(destInfo.absolutePath());
 313     if (!destination.exists(mkPath)) {
 314       if (mkPath == lastMkPath) {
 315         // we've already tried, and apparently failed, to create this folder... bail out but log as error.
 316         if (!(m_options.flags & OPT_DRY_RUN)) {
 317           ++m_stat.errored;
 318           return false;
 319         }
 320       }
 321       else {
 322         lastMkPath = mkPath;
 323         PRINT_CREATE(tr("Creating directory: %1").arg(mkPath));
 324         if (!(m_options.flags & OPT_DRY_RUN) && !destination.mkpath(mkPath)) {
 325           PRINT_ERROR(tr("Could not create directory: %1").arg(mkPath));
 326             ++m_stat.errored;
 327           return false;
 328         }
 329       }
 330     }
 331     else if (m_dirFilters & QDir::Dirs) {
 332       PRINT_SKIP(tr("Directory exists: %1").arg(mkPath));
 333     }
 334     if (sourceInfo.isDir())
 335       return true;
 336   }
 337
 338   //qDebug() << destPath;
 339   QFile sourceFile(srcPath);
 340   QFile destinationFile(destPath);
 341   const bool destExists = destInfo.exists();
 342   bool checkDate = (m_options.compareType == OVERWR_NEWER_IF_DIFF || m_options.compareType == OVERWR_NEWER_ALWAYS);
 343   bool checkContent = (m_options.compareType == OVERWR_NEWER_IF_DIFF || m_options.compareType == OVERWR_IF_DIFF);
 344   bool existed = false;
 345
 346   if (destExists && checkDate) {
 347     const QDate cmprDate = QDate::currentDate();
 348     if (sourceInfo.lastModified().date() > cmprDate || destInfo.lastModified().date() > cmprDate) {
 349       PRINT_ERROR(tr("At least one of the file modification dates is in the future, error on: %1").arg(srcPath));
 350       ++m_stat.errored;
 351       return false;
 352     }
 353     if (sourceInfo.lastModified() <= destInfo.lastModified()) {
 354       PRINT_SKIP(tr("Skipping older file: %1").arg(srcPath));
 355       ++m_stat.skipped;
 356       return true;
 357     }
 358     checkDate = false;
 359   }
 360
 361   if (destExists && checkContent) {
 362     if (!sourceFile.open(QFile::ReadOnly)) {
 363       PRINT_ERROR(tr("Could not open source file '%1': %2").arg(srcPath, sourceFile.errorString()));
 364       ++m_stat.errored;
 365       return false;
 366     }
 367     if (!destinationFile.open(QFile::ReadOnly)) {
 368       PRINT_ERROR(tr("Could not open destination file '%1': %2").arg(destPath, destinationFile.errorString()));
 369       ++m_stat.errored;
 370       return false;
 371     }
 372
 373     const bool skip = QCryptographicHash::hash(sourceFile.readAll(), QCryptographicHash::Md5) == QCryptographicHash::hash(destinationFile.readAll(), QCryptographicHash::Md5);
 374     sourceFile.close();
 375     destinationFile.close();
 376     if (skip) {
 377       PRINT_SKIP(tr("Skipping identical file: %1").arg(srcPath));
 378       ++m_stat.skipped;
 379       return true;
 380     }
 381     checkContent = false;
 382   }
 383
 384   if (!destExists || (!checkDate && !checkContent)) {
 385     if (destInfo.exists()) {
 386       existed = true;
 387       PRINT_REPLACE(tr("Replacing file: %1").arg(destPath));
 388       if (!(m_options.flags & OPT_DRY_RUN) && !destinationFile.remove()) {
 389         PRINT_ERROR(tr("Could not delete destination file '%1': %2").arg(destPath, destinationFile.errorString()));
 390         ++m_stat.errored;
 391         return false;
 392       }
 393     }
 394     else {
 395       PRINT_CREATE(tr("Creating file: %1").arg(destPath));
 396     }
 397     if (!(m_options.flags & OPT_DRY_RUN) && !sourceFile.copy(destPath)) {
 398       PRINT_ERROR(tr("Copy failed: '%1' to '%2': %3").arg(srcPath, destPath, sourceFile.errorString()));
 399       ++m_stat.errored;
 400       return false;
 401     }
 402
 403     if (existed)
 404       ++m_stat.updated;
 405     else
 406       ++m_stat.created;
 407   }
 408
 409   return true;
 410 }
 411
 412 void SyncProcess::pause()
 413 {
 414   QElapsedTimer tim;
 415   const qint64 exp = m_pauseTime * 1000;
 416   tim.start();
 417   while (tim.nsecsElapsed() < exp && !isStopRequsted())
 418     QApplication::processEvents();
 419 }
 420
 421 void SyncProcess::emitProgressMessage(const QString & text, int type)
 422 {
 423   if (m_options.logLevel == QtDebugMsg || (m_options.logLevel == QtInfoMsg && type > QtDebugMsg) || (type < QtInfoMsg && type >= m_options.logLevel)) {
 424     emit progressMessage(text, type);
 425     m_pauseTime = qMin(m_pauseTime + PAUSE_FACTOR, PAUSE_MAXTM);
 426   }
 427 }