src/PartFileConvert.cpp

   1 //
   2 // This file is part of the aMule Project.
   3 //
   4 // Copyright (c) 2003-2011 aMule Team ( admin@amule.org / http://www.amule.org )
   5 // Copyright (c) 2002-2011 Merkur ( devs@emule-project.net / http://www.emule-project.net )
   6 //
   7 // Any parts of this program derived from the xMule, lMule or eMule project,
   8 // or contributed by third-party developers are copyrighted by their
   9 // respective authors.
  10 //
  11 // This program is free software; you can redistribute it and/or modify
  12 // it under the terms of the GNU General Public License as published by
  13 // the Free Software Foundation; either version 2 of the License, or
  14 // (at your option) any later version.
  15 //
  16 // This program is distributed in the hope that it will be useful,
  17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 // GNU General Public License for more details.
  20 //
  21 // You should have received a copy of the GNU General Public License
  22 // along with this program; if not, write to the Free Software
  23 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
  24 //
  25
  26 #include <wx/app.h>
  27
  28 #include "PartFileConvert.h"
  29
  30 #include "amule.h"
  31 #include "DownloadQueue.h"
  32 #include <common/Format.h>
  33 #include "Logger.h"
  34 #include "PartFile.h"
  35 #include "Preferences.h"
  36 #include "SharedFileList.h"
  37 #include <common/FileFunctions.h>
  38 #include "OtherFunctions.h"
  39 #include "GuiEvents.h"
  40 #include "DataToText.h"
  41
  42 static unsigned s_nextJobId = 0;
  43
  44
  45 struct ConvertJob {
  46         unsigned        id;
  47         CPath           folder;
  48         CPath           filename;
  49         wxString        filehash;
  50         ConvStatus      state;
  51         uint32_t        size;
  52         uint32_t        spaceneeded;
  53         uint8           partmettype;
  54         bool            removeSource;
  55         ConvertJob(const CPath& file, bool deleteSource, ConvStatus status)
  56                 : id(s_nextJobId++), folder(file), state(status), size(0), spaceneeded(0), partmettype(PMT_UNKNOWN), removeSource(deleteSource)
  57                 {}
  58 };
  59
  60 ConvertInfo::ConvertInfo(ConvertJob *job)
  61         : id(job->id),
  62           folder(job->folder), filename(job->filename), filehash(job->filehash),
  63           state(job->state), size(job->size), spaceneeded(job->spaceneeded)
  64 {}
  65
  66
  67 wxThread*               CPartFileConvert::s_convertPfThread = NULL;
  68 std::list<ConvertJob*>  CPartFileConvert::s_jobs;
  69 ConvertJob*             CPartFileConvert::s_pfconverting = NULL;
  70 wxMutex                 CPartFileConvert::s_mutex;
  71
  72
  73 int CPartFileConvert::ScanFolderToAdd(const CPath& folder, bool deletesource)
  74 {
  75         int count = 0;
  76         CDirIterator finder(folder);
  77
  78         CPath file = finder.GetFirstFile(CDirIterator::File, wxT("*.part.met"));
  79         while (file.IsOk()) {
  80                 ConvertToeMule(folder.JoinPaths(file), deletesource);
  81                 file = finder.GetNextFile();
  82                 count++;
  83         }
  84         /* Shareaza
  85         file = finder.GetFirstFile(CDirIterator::File, wxT("*.sd"));
  86         while (!file.IsEmpty()) {
  87                 ConvertToeMule(file, deletesource);
  88                 file = finder.GetNextFile();
  89                 count++;
  90         }
  91         */
  92
  93         file = finder.GetFirstFile(CDirIterator::Dir, wxT("*.*"));
  94         while (file.IsOk()) {
  95                 ScanFolderToAdd(folder.JoinPaths(file), deletesource);
  96
  97                 file = finder.GetNextFile();
  98         }
  99
 100         return count;
 101 }
 102
 103 void CPartFileConvert::ConvertToeMule(const CPath& file, bool deletesource)
 104 {
 105         if (!file.FileExists()) {
 106                 return;
 107         }
 108
 109         ConvertJob* newjob = new ConvertJob(file, deletesource, CONV_QUEUE);
 110
 111         wxMutexLocker lock(s_mutex);
 112
 113         s_jobs.push_back(newjob);
 114
 115         Notify_ConvertUpdateJobInfo(newjob);
 116
 117         StartThread();
 118 }
 119
 120 void CPartFileConvert::StartThread()
 121 {
 122         if (!s_convertPfThread) {
 123                 s_convertPfThread = new CPartFileConvert();
 124
 125                 switch ( s_convertPfThread->Create() ) {
 126                         case wxTHREAD_NO_ERROR:
 127                                 AddDebugLogLineN( logPfConvert, wxT("A new thread has been created.") );
 128                                 break;
 129                         case wxTHREAD_RUNNING:
 130                                 AddDebugLogLineC( logPfConvert, wxT("Error, attempt to create an already running thread!") );
 131                                 break;
 132                         case wxTHREAD_NO_RESOURCE:
 133                                 AddDebugLogLineC( logPfConvert, wxT("Error, attempt to create a thread without resources!") );
 134                                 break;
 135                         default:
 136                                 AddDebugLogLineC( logPfConvert, wxT("Error, unknown error attempting to create a thread!") );
 137                 }
 138
 139                 // The thread shouldn't hog the CPU, as it will already be hogging the HD
 140                 s_convertPfThread->SetPriority(WXTHREAD_MIN_PRIORITY);
 141
 142                 s_convertPfThread->Run();
 143         }
 144 }
 145
 146 void CPartFileConvert::StopThread()
 147 {
 148         if (s_convertPfThread) {
 149                 s_convertPfThread->Delete();
 150         } else {
 151                 return;
 152         }
 153
 154         AddLogLineNS(_("Waiting for partfile convert thread to die..."));
 155         while (s_convertPfThread) {
 156                 wxSleep(1);
 157         }
 158 }
 159
 160 wxThread::ExitCode CPartFileConvert::Entry()
 161 {
 162         int imported = 0;
 163
 164         for (;;)
 165         {
 166                 // search next queued job and start it
 167                 {
 168                         wxMutexLocker lock(s_mutex);
 169                         s_pfconverting = NULL;
 170                         for (std::list<ConvertJob*>::iterator it = s_jobs.begin(); it != s_jobs.end(); ++it) {
 171                                 if ((*it)->state == CONV_QUEUE) {
 172                                         s_pfconverting = *it;
 173                                         break;
 174                                 }
 175                         }
 176                 }
 177
 178                 if (s_pfconverting) {
 179                         {
 180                                 wxMutexLocker lock(s_mutex);
 181                                 s_pfconverting->state = CONV_INPROGRESS;
 182                         }
 183
 184                         Notify_ConvertUpdateJobInfo(s_pfconverting);
 185
 186                         ConvStatus convertResult = performConvertToeMule(s_pfconverting->folder);
 187                         {
 188                                 wxMutexLocker lock(s_mutex);
 189                                 s_pfconverting->state = convertResult;
 190                         }
 191
 192                         if (s_pfconverting->state == CONV_OK) {
 193                                 ++imported;
 194                         }
 195
 196                         Notify_ConvertUpdateJobInfo(s_pfconverting);
 197
 198                         AddLogLineC(CFormat(_("Importing %s: %s")) % s_pfconverting->folder % GetConversionState(s_pfconverting->state));
 199
 200                         if (TestDestroy()) {
 201                                 wxMutexLocker lock(s_mutex);
 202                                 DeleteContents(s_jobs);
 203                                 break;
 204                         }
 205                 } else {
 206                         break; // nothing more to do now
 207                 }
 208         }
 209
 210         // clean up
 211         Notify_ConvertClearInfos();
 212
 213         if (imported) {
 214                 theApp->sharedfiles->PublishNextTurn();
 215         }
 216
 217         AddDebugLogLineN(logPfConvert, wxT("No more jobs on queue, exiting from thread."));
 218
 219         s_convertPfThread = NULL;
 220
 221         return NULL;
 222 }
 223
 224 ConvStatus CPartFileConvert::performConvertToeMule(const CPath& fileName)
 225 {
 226         wxString filepartindex;
 227
 228         CPath folder    = fileName.GetPath();
 229         CPath partfile  = fileName.GetFullName();
 230         CPath newfilename;
 231
 232         CDirIterator finder(folder);
 233
 234         Notify_ConvertUpdateProgressFull(0, _("Reading temp folder"), s_pfconverting->folder.GetPrintable());
 235
 236         filepartindex = partfile.RemoveAllExt().GetRaw();
 237
 238         Notify_ConvertUpdateProgress(4, _("Retrieving basic information from download info file"));
 239
 240         CPartFile* file = new CPartFile();
 241         s_pfconverting->partmettype = file->LoadPartFile(folder, partfile, false, true);
 242
 243         switch (s_pfconverting->partmettype) {
 244                 case PMT_UNKNOWN:
 245                 case PMT_BADFORMAT:
 246                         delete file;
 247                         return CONV_BADFORMAT;
 248         }
 249
 250         CPath oldfile = folder.JoinPaths(partfile.RemoveExt());
 251
 252         {
 253                 wxMutexLocker lock(s_mutex);
 254                 s_pfconverting->size = file->GetFileSize();
 255                 s_pfconverting->filename = file->GetFileName();
 256                 s_pfconverting->filehash = file->GetFileHash().Encode();
 257         }
 258
 259         Notify_ConvertUpdateJobInfo(s_pfconverting);
 260
 261         if (theApp->downloadqueue->GetFileByID(file->GetFileHash())) {
 262                 delete file;
 263                 return CONV_ALREADYEXISTS;
 264         }
 265
 266         if (s_pfconverting->partmettype == PMT_SPLITTED) {
 267                 unsigned fileindex;
 268                 char *ba = new char [PARTSIZE];
 269
 270                 try {
 271                         CFile inputfile;
 272
 273                         // just count
 274                         unsigned maxindex = 0;
 275                         unsigned partfilecount = 0;
 276                         CPath filePath = finder.GetFirstFile(CDirIterator::File, filepartindex + wxT(".*.part"));
 277                         while (filePath.IsOk()) {
 278                                 long l;
 279                                 ++partfilecount;
 280                                 filePath.GetFullName().RemoveExt().GetExt().ToLong(&l);
 281                                 fileindex = (unsigned)l;
 282                                 filePath = finder.GetNextFile();
 283                                 if (fileindex > maxindex) maxindex = fileindex;
 284                         }
 285                         float stepperpart;
 286                         {
 287                                 wxMutexLocker lock(s_mutex);
 288                                 if (partfilecount > 0) {
 289                                         stepperpart = (80.0f / partfilecount);
 290                                         if (maxindex * PARTSIZE <= s_pfconverting->size) {
 291                                                 s_pfconverting->spaceneeded = maxindex * PARTSIZE;
 292                                         } else {
 293                                                 s_pfconverting->spaceneeded = s_pfconverting->size;
 294                                         }
 295                                 } else {
 296                                         stepperpart = 80.0f;
 297                                         s_pfconverting->spaceneeded = 0;
 298                                 }
 299                         }
 300
 301                         Notify_ConvertUpdateJobInfo(s_pfconverting);
 302
 303                         sint64 freespace = CPath::GetFreeSpaceAt(thePrefs::GetTempDir());
 304                         if (freespace != wxInvalidOffset) {
 305                                 if (static_cast<uint64>(freespace) < maxindex * PARTSIZE) {
 306                                         delete file;
 307                                         delete [] ba;
 308                                         return CONV_OUTOFDISKSPACE;
 309                                 }
 310                         }
 311
 312                         // create new partmetfile, and remember the new name
 313                         file->CreatePartFile(true);
 314                         newfilename = file->GetFullName();
 315
 316                         Notify_ConvertUpdateProgress(8, _("Creating destination file"));
 317
 318                         file->m_hpartfile.SetLength( s_pfconverting->spaceneeded );
 319
 320                         unsigned curindex = 0;
 321                         CPath filename = finder.GetFirstFile(CDirIterator::File, filepartindex + wxT(".*.part"));
 322                         while (filename.IsOk()) {
 323                                 // stats
 324                                 ++curindex;
 325
 326                                 Notify_ConvertUpdateProgress(10 + (curindex * stepperpart), CFormat(_("Loading data from old download file (%u of %u)")) % curindex % partfilecount);
 327
 328                                 long l;
 329                                 filename.GetFullName().RemoveExt().GetExt().ToLong(&l);
 330                                 fileindex = (unsigned)l;
 331                                 if (fileindex == 0) {
 332                                         filename = finder.GetNextFile();
 333                                         continue;
 334                                 }
 335
 336                                 uint32 chunkstart = (fileindex - 1) * PARTSIZE;
 337
 338                                 // open, read data of the part-part-file into buffer, close file
 339                                 inputfile.Open(filename, CFile::read);
 340                                 uint64 toReadWrite = std::min<uint64>(PARTSIZE, inputfile.GetLength());
 341                                 inputfile.Read(ba, toReadWrite);
 342                                 inputfile.Close();
 343
 344                                 Notify_ConvertUpdateProgress(10 + (curindex * stepperpart), CFormat(_("Saving data block into new single download file (%u of %u)")) % curindex % partfilecount);
 345
 346                                 // write the buffered data
 347                                 file->m_hpartfile.WriteAt(ba, chunkstart, toReadWrite);
 348
 349                                 filename = finder.GetNextFile();
 350                         }
 351                         delete[] ba;
 352                 } catch (const CSafeIOException& e) {
 353                         AddDebugLogLineC(logPfConvert, wxT("IO error while converting partfiles: ") + e.what());
 354
 355                         delete[] ba;
 356                         file->Delete();
 357                         return CONV_IOERROR;
 358                 }
 359
 360                 file->m_hpartfile.Close();
 361         }
 362         // import an external common format partdownload
 363         else //if (pfconverting->partmettype==PMT_DEFAULTOLD || pfconverting->partmettype==PMT_NEWOLD || Shareaza  )
 364         {
 365                 if (!s_pfconverting->removeSource) {
 366                         wxMutexLocker lock(s_mutex);
 367                         s_pfconverting->spaceneeded = oldfile.GetFileSize();
 368                 }
 369
 370                 Notify_ConvertUpdateJobInfo(s_pfconverting);
 371
 372                 sint64 freespace = CPath::GetFreeSpaceAt(thePrefs::GetTempDir());
 373                 if (freespace == wxInvalidOffset) {
 374                         delete file;
 375                         return CONV_IOERROR;
 376                 } else if (freespace < s_pfconverting->spaceneeded) {
 377                         delete file;
 378                         return CONV_OUTOFDISKSPACE;
 379                 }
 380
 381                 file->CreatePartFile(true);
 382                 newfilename = file->GetFullName();
 383
 384                 file->m_hpartfile.Close();
 385
 386                 bool ret = false;
 387
 388                 Notify_ConvertUpdateProgress(92, _("Copy"));
 389
 390                 CPath::RemoveFile(newfilename.RemoveExt());
 391                 if (!oldfile.FileExists()) {
 392                         // data file does not exist. well, then create a 0 byte big one
 393                         CFile datafile;
 394                         ret = datafile.Create(newfilename.RemoveExt());
 395                 } else if (s_pfconverting->removeSource) {
 396                         ret = CPath::RenameFile(oldfile, newfilename.RemoveExt());
 397                 } else {
 398                         ret = CPath::CloneFile(oldfile, newfilename.RemoveExt(), false);
 399                 }
 400                 if (!ret) {
 401                         file->Delete();
 402                         //delete file;
 403                         return CONV_FAILED;
 404                 }
 405
 406         }
 407
 408         Notify_ConvertUpdateProgress(94, _("Retrieving source downloadfile information"));
 409
 410         CPath::RemoveFile(newfilename);
 411         if (s_pfconverting->removeSource) {
 412                 CPath::RenameFile(folder.JoinPaths(partfile), newfilename);
 413         } else {
 414                 CPath::CloneFile(folder.JoinPaths(partfile), newfilename, false);
 415         }
 416
 417         file->m_hashlist.clear();
 418
 419         if (!file->LoadPartFile(thePrefs::GetTempDir(), file->GetPartMetFileName(), false)) {
 420                 //delete file;
 421                 file->Delete();
 422                 return CONV_BADFORMAT;
 423         }
 424
 425         if (s_pfconverting->partmettype == PMT_NEWOLD || s_pfconverting->partmettype == PMT_SPLITTED) {
 426                 file->SetCompletedSize(file->transferred);
 427                 file->m_iGainDueToCompression = 0;
 428                 file->m_iLostDueToCorruption = 0;
 429         }
 430
 431         Notify_ConvertUpdateProgress(100, _("Adding download and saving new partfile"));
 432
 433         theApp->downloadqueue->AddDownload(file, thePrefs::AddNewFilesPaused(), 0);
 434         file->SavePartFile();
 435
 436         if (file->GetStatus(true) == PS_READY) {
 437                 theApp->sharedfiles->SafeAddKFile(file); // part files are always shared files
 438         }
 439
 440         if (s_pfconverting->removeSource) {
 441                 CPath oldFile = finder.GetFirstFile(CDirIterator::File, filepartindex + wxT(".*"));
 442                 while (oldFile.IsOk()) {
 443                         CPath::RemoveFile(folder.JoinPaths(oldFile));
 444                         oldFile = finder.GetNextFile();
 445                 }
 446
 447                 if (s_pfconverting->partmettype == PMT_SPLITTED) {
 448                         CPath::RemoveDir(folder);
 449                 }
 450         }
 451
 452         return CONV_OK;
 453 }
 454
 455 // Notification handlers
 456
 457 void CPartFileConvert::RemoveJob(unsigned id)
 458 {
 459         wxMutexLocker lock(s_mutex);
 460         for (std::list<ConvertJob*>::iterator it = s_jobs.begin(); it != s_jobs.end(); ++it) {
 461                 if ((*it)->id == id && (*it)->state != CONV_INPROGRESS) {
 462                         ConvertJob *job = *it;
 463                         s_jobs.erase(it);
 464                         Notify_ConvertRemoveJobInfo(id);
 465                         delete job;
 466                         break;
 467                 }
 468         }
 469 }
 470
 471 void CPartFileConvert::RetryJob(unsigned id)
 472 {
 473         wxMutexLocker lock(s_mutex);
 474         for (std::list<ConvertJob*>::iterator it = s_jobs.begin(); it != s_jobs.end(); ++it) {
 475                 if ((*it)->id == id && (*it)->state != CONV_INPROGRESS && (*it)->state != CONV_OK) {
 476                         (*it)->state = CONV_QUEUE;
 477                         Notify_ConvertUpdateJobInfo(*it);
 478                         StartThread();
 479                         break;
 480                 }
 481         }
 482 }
 483
 484 void CPartFileConvert::ReaddAllJobs()
 485 {
 486         wxMutexLocker lock(s_mutex);
 487         for (std::list<ConvertJob*>::iterator it = s_jobs.begin(); it != s_jobs.end(); ++it) {
 488                 Notify_ConvertUpdateJobInfo(*it);
 489         }
 490 }