src/PartFileConvert.cpp

   1 //
   2 // This file is part of the aMule Project.
   3 //
   4 // Copyright (c) 2003-2008 aMule Team ( admin@amule.org / http://www.amule.org )
   5 // Copyright (c) 2002-2008 Merkur ( devs@emule-project.net / http://www.emule-project.net )
   6 //
   7 // Any parts of this program derived from the xMule, lMule or eMule project,
   8 // or contributed by third-party developers are copyrighted by their
   9 // respective authors.
  10 //
  11 // This program is free software; you can redistribute it and/or modify
  12 // it under the terms of the GNU General Public License as published by
  13 // the Free Software Foundation; either version 2 of the License, or
  14 // (at your option) any later version.
  15 //
  16 // This program is distributed in the hope that it will be useful,
  17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 // GNU General Public License for more details.
  20 //
  21 // You should have received a copy of the GNU General Public License
  22 // along with this program; if not, write to the Free Software
  23 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
  24 //
  25
  26 #include <wx/app.h>
  27
  28 #include "PartFileConvert.h"
  29
  30 #include "amule.h"
  31 #include "DownloadQueue.h"
  32 #include <common/Format.h>
  33 #include "Logger.h"
  34 #include "PartFile.h"
  35 #include "Preferences.h"
  36 #include "SharedFileList.h"
  37 #include <common/FileFunctions.h>
  38 #include "OtherFunctions.h"
  39 #include "GuiEvents.h"
  40 #include "DataToText.h"
  41
  42 static unsigned s_nextJobId = 0;
  43
  44
  45 struct ConvertJob {
  46         unsigned        id;
  47         CPath           folder;
  48         CPath           filename;
  49         wxString        filehash;
  50         int             format;
  51         ConvStatus      state;
  52         uint32_t        size;
  53         uint32_t        spaceneeded;
  54         uint8           partmettype;
  55         bool            removeSource;
  56         ConvertJob()    { id=s_nextJobId++; size=0; spaceneeded=0; partmettype=PMT_UNKNOWN; removeSource=true; }
  57 };
  58
  59 ConvertInfo::ConvertInfo(ConvertJob *job)
  60         : id(job->id),
  61           folder(job->folder), filename(job->filename), filehash(job->filehash),
  62           state(job->state), size(job->size), spaceneeded(job->spaceneeded)
  63 {}
  64
  65
  66 wxThread*               CPartFileConvert::s_convertPfThread = NULL;
  67 std::list<ConvertJob*>  CPartFileConvert::s_jobs;
  68 ConvertJob*             CPartFileConvert::s_pfconverting = NULL;
  69 wxMutex                 CPartFileConvert::s_mutex;
  70
  71
  72 int CPartFileConvert::ScanFolderToAdd(const CPath& folder, bool deletesource)
  73 {
  74         int count = 0;
  75         CDirIterator finder(folder);
  76
  77         CPath file = finder.GetFirstFile(CDirIterator::File, wxT("*.part.met"));
  78         while (file.IsOk()) {
  79                 ConvertToeMule(folder.JoinPaths(file), deletesource);
  80                 file = finder.GetNextFile();
  81                 count++;
  82         }
  83         /* Shareaza
  84         file = finder.GetFirstFile(CDirIterator::File, wxT("*.sd"));
  85         while (!file.IsEmpty()) {
  86                 ConvertToeMule(file, deletesource);
  87                 file = finder.GetNextFile();
  88                 count++;
  89         }
  90         */
  91
  92         file = finder.GetFirstFile(CDirIterator::Dir, wxT("*.*"));
  93         while (file.IsOk()) {
  94                 ScanFolderToAdd(folder.JoinPaths(file), deletesource);
  95
  96                 file = finder.GetNextFile();
  97         }
  98
  99         return count;
 100 }
 101
 102 void CPartFileConvert::ConvertToeMule(const CPath& file, bool deletesource)
 103 {
 104         if (!file.FileExists()) {
 105                 return;
 106         }
 107
 108         ConvertJob* newjob = new ConvertJob();
 109         newjob->folder = file;
 110         newjob->removeSource = deletesource;
 111         newjob->state = CONV_QUEUE;
 112
 113         wxMutexLocker lock(s_mutex);
 114
 115         s_jobs.push_back(newjob);
 116
 117         Notify_ConvertUpdateJobInfo(newjob);
 118
 119         StartThread();
 120 }
 121
 122 void CPartFileConvert::StartThread()
 123 {
 124         if (!s_convertPfThread) {
 125                 s_convertPfThread = new CPartFileConvert();
 126
 127                 switch ( s_convertPfThread->Create() ) {
 128                         case wxTHREAD_NO_ERROR:
 129                                 AddDebugLogLineM( false, logPfConvert, wxT("A new thread has been created.") );
 130                                 break;
 131                         case wxTHREAD_RUNNING:
 132                                 AddDebugLogLineM( true, logPfConvert, wxT("Error, attempt to create an already running thread!") );
 133                                 break;
 134                         case wxTHREAD_NO_RESOURCE:
 135                                 AddDebugLogLineM( true, logPfConvert, wxT("Error, attempt to create a thread without resources!") );
 136                                 break;
 137                         default:
 138                                 AddDebugLogLineM( true, logPfConvert, wxT("Error, unknown error attempting to create a thread!") );
 139                 }
 140
 141                 // The thread shouldn't hog the CPU, as it will already be hogging the HD
 142                 s_convertPfThread->SetPriority(WXTHREAD_MIN_PRIORITY);
 143
 144                 s_convertPfThread->Run();
 145         }
 146 }
 147
 148 void CPartFileConvert::StopThread()
 149 {
 150         if (s_convertPfThread) {
 151                 s_convertPfThread->Delete();
 152         } else {
 153                 return;
 154         }
 155
 156         AddLogLineNS(_("Waiting for partfile convert thread to die..."));
 157         while (s_convertPfThread) {
 158                 wxSleep(1);
 159         }
 160 }
 161
 162 wxThread::ExitCode CPartFileConvert::Entry()
 163 {
 164         int imported = 0;
 165
 166         for (;;)
 167         {
 168                 // search next queued job and start it
 169                 {
 170                         wxMutexLocker lock(s_mutex);
 171                         s_pfconverting = NULL;
 172                         for (std::list<ConvertJob*>::iterator it = s_jobs.begin(); it != s_jobs.end(); ++it) {
 173                                 if ((*it)->state == CONV_QUEUE) {
 174                                         s_pfconverting = *it;
 175                                         break;
 176                                 }
 177                         }
 178                 }
 179
 180                 if (s_pfconverting) {
 181                         {
 182                                 wxMutexLocker lock(s_mutex);
 183                                 s_pfconverting->state = CONV_INPROGRESS;
 184                         }
 185
 186                         Notify_ConvertUpdateJobInfo(s_pfconverting);
 187
 188                         ConvStatus convertResult = performConvertToeMule(s_pfconverting->folder);
 189                         {
 190                                 wxMutexLocker lock(s_mutex);
 191                                 s_pfconverting->state = convertResult;
 192                         }
 193
 194                         if (s_pfconverting->state == CONV_OK) {
 195                                 ++imported;
 196                         }
 197
 198                         Notify_ConvertUpdateJobInfo(s_pfconverting);
 199
 200                         AddLogLineM(true, CFormat(_("Importing %s: %s")) % s_pfconverting->folder % GetConversionState(s_pfconverting->state));
 201
 202                         if (TestDestroy()) {
 203                                 wxMutexLocker lock(s_mutex);
 204                                 DeleteContents(s_jobs);
 205                                 break;
 206                         }
 207                 } else {
 208                         break; // nothing more to do now
 209                 }
 210         }
 211
 212         // clean up
 213         Notify_ConvertClearInfos();
 214
 215         if (imported) {
 216                 theApp->sharedfiles->PublishNextTurn();
 217         }
 218
 219         AddDebugLogLineM(false, logPfConvert, wxT("No more jobs on queue, exiting from thread."));
 220
 221         s_convertPfThread = NULL;
 222
 223         return NULL;
 224 }
 225
 226 ConvStatus CPartFileConvert::performConvertToeMule(const CPath& fileName)
 227 {
 228         wxString filepartindex, buffer;
 229         unsigned fileindex;
 230
 231         CPath folder    = fileName.GetPath();
 232         CPath partfile  = fileName.GetFullName();
 233         CPath newfilename;
 234
 235         CDirIterator finder(folder);
 236
 237         Notify_ConvertUpdateProgressFull(0, _("Reading temp folder"), s_pfconverting->folder.GetPrintable());
 238
 239         filepartindex = partfile.RemoveAllExt().GetRaw();
 240
 241         Notify_ConvertUpdateProgress(4, _("Retrieving basic information from download info file"));
 242
 243         CPartFile* file = new CPartFile();
 244         s_pfconverting->partmettype = file->LoadPartFile(folder, partfile, false, true);
 245
 246         switch (s_pfconverting->partmettype) {
 247                 case PMT_UNKNOWN:
 248                 case PMT_BADFORMAT:
 249                         delete file;
 250                         return CONV_BADFORMAT;
 251         }
 252
 253         CPath oldfile = folder.JoinPaths(partfile.RemoveExt());
 254
 255         {
 256                 wxMutexLocker lock(s_mutex);
 257                 s_pfconverting->size = file->GetFileSize();
 258                 s_pfconverting->filename = file->GetFileName();
 259                 s_pfconverting->filehash = file->GetFileHash().Encode();
 260         }
 261
 262         Notify_ConvertUpdateJobInfo(s_pfconverting);
 263
 264         if (theApp->downloadqueue->GetFileByID(file->GetFileHash())) {
 265                 delete file;
 266                 return CONV_ALREADYEXISTS;
 267         }
 268
 269         if (s_pfconverting->partmettype == PMT_SPLITTED) {
 270                 char *ba = new char [PARTSIZE];
 271
 272                 try {
 273                         CFile inputfile;
 274
 275                         // just count
 276                         unsigned maxindex = 0;
 277                         unsigned partfilecount = 0;
 278                         CPath filePath = finder.GetFirstFile(CDirIterator::File, filepartindex + wxT(".*.part"));
 279                         while (filePath.IsOk()) {
 280                                 long l;
 281                                 ++partfilecount;
 282                                 filePath.GetFullName().RemoveExt().GetExt().ToLong(&l);
 283                                 fileindex = (unsigned)l;
 284                                 filePath = finder.GetNextFile();
 285                                 if (fileindex > maxindex) maxindex = fileindex;
 286                         }
 287                         float stepperpart;
 288                         {
 289                                 wxMutexLocker lock(s_mutex);
 290                                 if (partfilecount > 0) {
 291                                         stepperpart = (80.0f / partfilecount);
 292                                         if (maxindex * PARTSIZE <= s_pfconverting->size) {
 293                                                 s_pfconverting->spaceneeded = maxindex * PARTSIZE;
 294                                         } else {
 295                                                 s_pfconverting->spaceneeded = s_pfconverting->size;
 296                                         }
 297                                 } else {
 298                                         stepperpart = 80.0f;
 299                                         s_pfconverting->spaceneeded = 0;
 300                                 }
 301                         }
 302
 303                         Notify_ConvertUpdateJobInfo(s_pfconverting);
 304
 305                         sint64 freespace = CPath::GetFreeSpaceAt(thePrefs::GetTempDir());
 306                         if (freespace != wxInvalidOffset) {
 307                                 if (static_cast<uint64>(freespace) < maxindex * PARTSIZE) {
 308                                         delete file;
 309                                         delete [] ba;
 310                                         return CONV_OUTOFDISKSPACE;
 311                                 }
 312                         }
 313
 314                         // create new partmetfile, and remember the new name
 315                         file->CreatePartFile();
 316                         newfilename = file->GetFullName();
 317
 318                         Notify_ConvertUpdateProgress(8, _("Creating destination file"));
 319
 320                         file->m_hpartfile.SetLength( s_pfconverting->spaceneeded );
 321
 322                         unsigned curindex = 0;
 323                         CPath filename = finder.GetFirstFile(CDirIterator::File, filepartindex + wxT(".*.part"));
 324                         while (filename.IsOk()) {
 325                                 // stats
 326                                 ++curindex;
 327                                 buffer = wxString::Format(_("Loading data from old download file (%u of %u)"), curindex, partfilecount);
 328
 329                                 Notify_ConvertUpdateProgress(10 + (curindex * stepperpart), buffer);
 330
 331                                 long l;
 332                                 filename.GetFullName().RemoveExt().GetExt().ToLong(&l);
 333                                 fileindex = (unsigned)l;
 334                                 if (fileindex == 0) {
 335                                         filename = finder.GetNextFile();
 336                                         continue;
 337                                 }
 338
 339                                 uint32 chunkstart = (fileindex - 1) * PARTSIZE;
 340
 341                                 // open, read data of the part-part-file into buffer, close file
 342                                 inputfile.Open(filename, CFile::read);
 343                                 uint64 toReadWrite = std::min<uint64>(PARTSIZE, inputfile.GetLength());
 344                                 inputfile.Read(ba, toReadWrite);
 345                                 inputfile.Close();
 346
 347                                 buffer = wxString::Format(_("Saving data block into new single download file (%u of %u)"), curindex, partfilecount);
 348
 349                                 Notify_ConvertUpdateProgress(10 + (curindex * stepperpart), buffer);
 350
 351                                 // write the buffered data
 352                                 file->m_hpartfile.WriteAt(ba, chunkstart, toReadWrite);
 353
 354                                 filename = finder.GetNextFile();
 355                         }
 356                         delete[] ba;
 357                 } catch (const CSafeIOException& e) {
 358                         AddDebugLogLineM(true, logPfConvert, wxT("IO error while converting partfiles: ") + e.what());
 359
 360                         delete[] ba;
 361                         file->Delete();
 362                         return CONV_IOERROR;
 363                 }
 364
 365                 file->m_hpartfile.Close();
 366         }
 367         // import an external common format partdownload
 368         else //if (pfconverting->partmettype==PMT_DEFAULTOLD || pfconverting->partmettype==PMT_NEWOLD || Shareaza  )
 369         {
 370                 if (!s_pfconverting->removeSource) {
 371                         wxMutexLocker lock(s_mutex);
 372                         s_pfconverting->spaceneeded = oldfile.GetFileSize();
 373                 }
 374
 375                 Notify_ConvertUpdateJobInfo(s_pfconverting);
 376
 377                 sint64 freespace = CPath::GetFreeSpaceAt(thePrefs::GetTempDir());
 378                 if (freespace == wxInvalidOffset) {
 379                         delete file;
 380                         return CONV_IOERROR;
 381                 } else if (freespace < s_pfconverting->spaceneeded) {
 382                         delete file;
 383                         return CONV_OUTOFDISKSPACE;
 384                 }
 385
 386                 file->CreatePartFile();
 387                 newfilename = file->GetFullName();
 388
 389                 file->m_hpartfile.Close();
 390
 391                 bool ret = false;
 392
 393                 Notify_ConvertUpdateProgress(92, _("Copy"));
 394
 395                 CPath::RemoveFile(newfilename.RemoveExt());
 396                 if (!oldfile.FileExists()) {
 397                         // data file does not exist. well, then create a 0 byte big one
 398                         CFile datafile;
 399                         ret = datafile.Create(newfilename.RemoveExt());
 400                 } else if (s_pfconverting->removeSource) {
 401                         ret = CPath::RenameFile(oldfile, newfilename.RemoveExt());
 402                 } else {
 403                         ret = CPath::CloneFile(oldfile, newfilename.RemoveExt(), false);
 404                 }
 405                 if (!ret) {
 406                         file->Delete();
 407                         //delete file;
 408                         return CONV_FAILED;
 409                 }
 410
 411         }
 412
 413         Notify_ConvertUpdateProgress(94, _("Retrieving source downloadfile information"));
 414
 415         CPath::RemoveFile(newfilename);
 416         if (s_pfconverting->removeSource) {
 417                 CPath::RenameFile(folder.JoinPaths(partfile), newfilename);
 418         } else {
 419                 CPath::CloneFile(folder.JoinPaths(partfile), newfilename, false);
 420         }
 421
 422         file->m_hashlist.clear();
 423
 424         if (!file->LoadPartFile(thePrefs::GetTempDir(), file->GetPartMetFileName(), false)) {
 425                 //delete file;
 426                 file->Delete();
 427                 return CONV_BADFORMAT;
 428         }
 429
 430         if (s_pfconverting->partmettype == PMT_NEWOLD || s_pfconverting->partmettype == PMT_SPLITTED) {
 431                 file->SetCompletedSize(file->transferred);
 432                 file->m_iGainDueToCompression = 0;
 433                 file->m_iLostDueToCorruption = 0;
 434         }
 435
 436         Notify_ConvertUpdateProgress(100, _("Adding download and saving new partfile"));
 437
 438         theApp->downloadqueue->AddDownload(file, thePrefs::AddNewFilesPaused(), 0);
 439         file->SavePartFile();
 440
 441         if (file->GetStatus(true) == PS_READY) {
 442                 theApp->sharedfiles->SafeAddKFile(file); // part files are always shared files
 443         }
 444
 445         if (s_pfconverting->removeSource) {
 446                 CPath oldFile = finder.GetFirstFile(CDirIterator::File, filepartindex + wxT(".*"));
 447                 while (oldFile.IsOk()) {
 448                         CPath::RemoveFile(folder.JoinPaths(oldFile));
 449                         oldFile = finder.GetNextFile();
 450                 }
 451
 452                 if (s_pfconverting->partmettype == PMT_SPLITTED) {
 453                         CPath::RemoveDir(folder);
 454                 }
 455         }
 456
 457         return CONV_OK;
 458 }
 459
 460 // Notification handlers
 461
 462 void CPartFileConvert::RemoveJob(unsigned id)
 463 {
 464         wxMutexLocker lock(s_mutex);
 465         for (std::list<ConvertJob*>::iterator it = s_jobs.begin(); it != s_jobs.end(); ++it) {
 466                 if ((*it)->id == id && (*it)->state != CONV_INPROGRESS) {
 467                         ConvertJob *job = *it;
 468                         s_jobs.erase(it);
 469                         Notify_ConvertRemoveJobInfo(id);
 470                         delete job;
 471                         break;
 472                 }
 473         }
 474 }
 475
 476 void CPartFileConvert::RetryJob(unsigned id)
 477 {
 478         wxMutexLocker lock(s_mutex);
 479         for (std::list<ConvertJob*>::iterator it = s_jobs.begin(); it != s_jobs.end(); ++it) {
 480                 if ((*it)->id == id && (*it)->state != CONV_INPROGRESS && (*it)->state != CONV_OK) {
 481                         (*it)->state = CONV_QUEUE;
 482                         Notify_ConvertUpdateJobInfo(*it);
 483                         StartThread();
 484                         break;
 485                 }
 486         }
 487 }
 488
 489 void CPartFileConvert::ReaddAllJobs()
 490 {
 491         wxMutexLocker lock(s_mutex);
 492         for (std::list<ConvertJob*>::iterator it = s_jobs.begin(); it != s_jobs.end(); ++it) {
 493                 Notify_ConvertUpdateJobInfo(*it);
 494         }
 495 }