src/PartFileConvert.cpp

   1 //
   2 // This file is part of the aMule Project.
   3 //
   4 // Copyright (c) 2003-2008 aMule Team ( admin@amule.org / http://www.amule.org )
   5 // Copyright (c) 2002-2008 Merkur ( devs@emule-project.net / http://www.emule-project.net )
   6 //
   7 // Any parts of this program derived from the xMule, lMule or eMule project,
   8 // or contributed by third-party developers are copyrighted by their
   9 // respective authors.
  10 //
  11 // This program is free software; you can redistribute it and/or modify
  12 // it under the terms of the GNU General Public License as published by
  13 // the Free Software Foundation; either version 2 of the License, or
  14 // (at your option) any later version.
  15 //
  16 // This program is distributed in the hope that it will be useful,
  17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 // GNU General Public License for more details.
  20 //
  21 // You should have received a copy of the GNU General Public License
  22 // along with this program; if not, write to the Free Software
  23 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
  24 //
  25
  26 #include <wx/app.h>
  27
  28 #include "PartFileConvert.h"
  29
  30 #include "amule.h"
  31 #include "DownloadQueue.h"
  32 #include <common/Format.h>
  33 #include "Logger.h"
  34 #include "PartFile.h"
  35 #include "Preferences.h"
  36 #include "SharedFileList.h"
  37 #include <common/FileFunctions.h>
  38 #include "OtherFunctions.h"
  39 #include "GuiEvents.h"
  40 #include "DataToText.h"
  41
  42 static unsigned s_nextJobId = 0;
  43
  44
  45 struct ConvertJob {
  46         unsigned        id;
  47         CPath           folder;
  48         CPath           filename;
  49         wxString        filehash;
  50         int             format;
  51         ConvStatus      state;
  52         uint32_t        size;
  53         uint32_t        spaceneeded;
  54         uint8           partmettype;
  55         bool            removeSource;
  56         ConvertJob()    { id=s_nextJobId++; size=0; spaceneeded=0; partmettype=PMT_UNKNOWN; removeSource=true; }
  57 };
  58
  59 ConvertInfo::ConvertInfo(ConvertJob *job)
  60         : id(job->id),
  61           folder(job->folder), filename(job->filename), filehash(job->filehash),
  62           state(job->state), size(job->size), spaceneeded(job->spaceneeded)
  63 {}
  64
  65
  66 wxThread*               CPartFileConvert::s_convertPfThread = NULL;
  67 std::list<ConvertJob*>  CPartFileConvert::s_jobs;
  68 ConvertJob*             CPartFileConvert::s_pfconverting = NULL;
  69 wxMutex                 CPartFileConvert::s_mutex;
  70
  71
  72 int CPartFileConvert::ScanFolderToAdd(const CPath& folder, bool deletesource)
  73 {
  74         int count = 0;
  75         CDirIterator finder(folder);
  76
  77         CPath file = finder.GetFirstFile(CDirIterator::File, wxT("*.part.met"));
  78         while (file.IsOk()) {
  79                 ConvertToeMule(folder.JoinPaths(file), deletesource);
  80                 file = finder.GetNextFile();
  81                 count++;
  82         }
  83         /* Shareaza
  84         file = finder.GetFirstFile(CDirIterator::File, wxT("*.sd"));
  85         while (!file.IsEmpty()) {
  86                 ConvertToeMule(file, deletesource);
  87                 file = finder.GetNextFile();
  88                 count++;
  89         }
  90         */
  91
  92         file = finder.GetFirstFile(CDirIterator::Dir, wxT("*.*"));
  93         while (file.IsOk()) {
  94                 ScanFolderToAdd(folder.JoinPaths(file), deletesource);
  95
  96                 file = finder.GetNextFile();
  97         }
  98
  99         return count;
 100 }
 101
 102 void CPartFileConvert::ConvertToeMule(const CPath& file, bool deletesource)
 103 {
 104         if (!file.FileExists()) {
 105                 return;
 106         }
 107
 108         ConvertJob* newjob = new ConvertJob();
 109         newjob->folder = file;
 110         newjob->removeSource = deletesource;
 111         newjob->state = CONV_QUEUE;
 112
 113         wxMutexLocker lock(s_mutex);
 114
 115         s_jobs.push_back(newjob);
 116
 117         Notify_ConvertUpdateJobInfo(newjob);
 118
 119         StartThread();
 120 }
 121
 122 void CPartFileConvert::StartThread()
 123 {
 124         if (!s_convertPfThread) {
 125                 s_convertPfThread = new CPartFileConvert();
 126
 127                 switch ( s_convertPfThread->Create() ) {
 128                         case wxTHREAD_NO_ERROR:
 129                                 AddDebugLogLineM( false, logPfConvert, wxT("A new thread has been created.") );
 130                                 break;
 131                         case wxTHREAD_RUNNING:
 132                                 AddDebugLogLineM( true, logPfConvert, wxT("Error, attempt to create an already running thread!") );
 133                                 break;
 134                         case wxTHREAD_NO_RESOURCE:
 135                                 AddDebugLogLineM( true, logPfConvert, wxT("Error, attempt to create a thread without resources!") );
 136                                 break;
 137                         default:
 138                                 AddDebugLogLineM( true, logPfConvert, wxT("Error, unknown error attempting to create a thread!") );
 139                 }
 140
 141                 // The thread shouldn't hog the CPU, as it will already be hogging the HD
 142                 s_convertPfThread->SetPriority(WXTHREAD_MIN_PRIORITY);
 143
 144                 s_convertPfThread->Run();
 145         }
 146 }
 147
 148 void CPartFileConvert::StopThread()
 149 {
 150         if (s_convertPfThread) {
 151                 s_convertPfThread->Delete();
 152         } else {
 153                 return;
 154         }
 155
 156         AddLogLineNS(_("Waiting for partfile convert thread to die..."));
 157         while (s_convertPfThread) {
 158                 wxSleep(1);
 159         }
 160 }
 161
 162 wxThread::ExitCode CPartFileConvert::Entry()
 163 {
 164         int imported = 0;
 165
 166         for (;;)
 167         {
 168                 // search next queued job and start it
 169                 {
 170                         wxMutexLocker lock(s_mutex);
 171                         s_pfconverting = NULL;
 172                         for (std::list<ConvertJob*>::iterator it = s_jobs.begin(); it != s_jobs.end(); ++it) {
 173                                 s_pfconverting = *it;
 174                                 if (s_pfconverting->state == CONV_QUEUE) {
 175                                         break;
 176                                 } else {
 177                                         s_pfconverting = NULL;
 178                                 }
 179                         }
 180                 }
 181
 182                 if (s_pfconverting) {
 183                         {
 184                                 wxMutexLocker lock(s_mutex);
 185                                 s_pfconverting->state = CONV_INPROGRESS;
 186                         }
 187
 188                         Notify_ConvertUpdateJobInfo(s_pfconverting);
 189
 190                         ConvStatus convertResult = performConvertToeMule(s_pfconverting->folder);
 191                         {
 192                                 wxMutexLocker lock(s_mutex);
 193                                 s_pfconverting->state = convertResult;
 194                         }
 195
 196                         if (s_pfconverting->state == CONV_OK) {
 197                                 ++imported;
 198                         }
 199
 200                         if (TestDestroy()) {
 201                                 wxMutexLocker lock(s_mutex);
 202                                 DeleteContents(s_jobs);
 203                                 break;
 204                         }
 205
 206                         Notify_ConvertUpdateJobInfo(s_pfconverting);
 207
 208                         AddLogLineM(true, CFormat(_("Importing %s: %s")) % s_pfconverting->folder % GetConversionState(s_pfconverting->state));
 209                 } else {
 210                         break; // nothing more to do now
 211                 }
 212         }
 213
 214         // clean up
 215         Notify_ConvertClearInfos();
 216
 217         if (imported) {
 218                 theApp->sharedfiles->PublishNextTurn();
 219         }
 220
 221         AddDebugLogLineM(false, logPfConvert, wxT("No more jobs on queue, exiting from thread."));
 222
 223         s_convertPfThread = NULL;
 224
 225         return NULL;
 226 }
 227
 228 ConvStatus CPartFileConvert::performConvertToeMule(const CPath& fileName)
 229 {
 230         wxString filepartindex, buffer;
 231         unsigned fileindex;
 232
 233         CPath folder    = fileName.GetPath();
 234         CPath partfile  = fileName.GetFullName();
 235         CPath newfilename;
 236
 237         CDirIterator finder(folder);
 238
 239         Notify_ConvertUpdateProgressFull(0, _("Reading temp folder"), s_pfconverting->folder.GetPrintable());
 240
 241         filepartindex = partfile.RemoveAllExt().GetRaw();
 242
 243         Notify_ConvertUpdateProgress(4, _("Retrieving basic information from download info file"));
 244
 245         CPartFile* file = new CPartFile();
 246         s_pfconverting->partmettype = file->LoadPartFile(folder, partfile, false, true);
 247
 248         switch (s_pfconverting->partmettype) {
 249                 case PMT_UNKNOWN:
 250                 case PMT_BADFORMAT:
 251                         delete file;
 252                         return CONV_BADFORMAT;
 253         }
 254
 255         CPath oldfile = folder.JoinPaths(partfile.RemoveExt());
 256
 257         {
 258                 wxMutexLocker lock(s_mutex);
 259                 s_pfconverting->size = file->GetFileSize();
 260                 s_pfconverting->filename = file->GetFileName();
 261                 s_pfconverting->filehash = file->GetFileHash().Encode();
 262         }
 263
 264         Notify_ConvertUpdateJobInfo(s_pfconverting);
 265
 266         if (theApp->downloadqueue->GetFileByID(file->GetFileHash())) {
 267                 delete file;
 268                 return CONV_ALREADYEXISTS;
 269         }
 270
 271         if (s_pfconverting->partmettype == PMT_SPLITTED) {
 272                 char *ba = new char [PARTSIZE];
 273
 274                 try {
 275                         CFile inputfile;
 276
 277                         // just count
 278                         unsigned maxindex = 0;
 279                         unsigned partfilecount = 0;
 280                         CPath filePath = finder.GetFirstFile(CDirIterator::File, filepartindex + wxT(".*.part"));
 281                         while (filePath.IsOk()) {
 282                                 long l;
 283                                 ++partfilecount;
 284                                 filePath.GetFullName().RemoveExt().GetExt().ToLong(&l);
 285                                 fileindex = (unsigned)l;
 286                                 filePath = finder.GetNextFile();
 287                                 // GonoszTopi - why the hell does eMule need this??
 288                                 //if (fileindex == 0) continue;
 289                                 if (fileindex > maxindex) maxindex = fileindex;
 290                         }
 291                         float stepperpart;
 292                         {
 293                                 wxMutexLocker lock(s_mutex);
 294                                 if (partfilecount > 0) {
 295                                         stepperpart = (80.0f / partfilecount);
 296                                         if (maxindex * PARTSIZE <= s_pfconverting->size) {
 297                                                 s_pfconverting->spaceneeded = maxindex * PARTSIZE;
 298                                         } else {
 299                                                 s_pfconverting->spaceneeded = ((s_pfconverting->size / PARTSIZE) * PARTSIZE) + (s_pfconverting->size % PARTSIZE);
 300                                         }
 301                                 } else {
 302                                         stepperpart = 80.0f;
 303                                         s_pfconverting->spaceneeded = 0;
 304                                 }
 305                         }
 306
 307                         Notify_ConvertUpdateJobInfo(s_pfconverting);
 308
 309                         sint64 freespace = CPath::GetFreeSpaceAt(thePrefs::GetTempDir());
 310                         if (freespace != wxInvalidOffset) {
 311                                 if (static_cast<uint64>(freespace) < maxindex * PARTSIZE) {
 312                                         delete file;
 313                                         delete [] ba;
 314                                         return CONV_OUTOFDISKSPACE;
 315                                 }
 316                         }
 317
 318                         // create new partmetfile, and remember the new name
 319                         file->CreatePartFile();
 320                         newfilename = file->GetFullName();
 321
 322                         Notify_ConvertUpdateProgress(8, _("Creating destination file"));
 323
 324                         file->m_hpartfile.SetLength( s_pfconverting->spaceneeded );
 325
 326                         unsigned curindex = 0;
 327                         CPath filename = finder.GetFirstFile(CDirIterator::File, filepartindex + wxT(".*.part"));
 328                         while (filename.IsOk()) {
 329                                 // stats
 330                                 ++curindex;
 331                                 buffer = wxString::Format(_("Loading data from old download file (%u of %u)"), curindex, partfilecount);
 332
 333                                 Notify_ConvertUpdateProgress(10 + (curindex * stepperpart), buffer);
 334
 335                                 long l;
 336                                 filename.GetFullName().RemoveExt().GetExt().ToLong(&l);
 337                                 fileindex = (unsigned)l;
 338                                 if (fileindex == 0) {
 339                                         filename = finder.GetNextFile();
 340                                         continue;
 341                                 }
 342
 343                                 uint32 chunkstart = (fileindex - 1) * PARTSIZE;
 344
 345                                 // open, read data of the part-part-file into buffer, close file
 346                                 inputfile.Open(filename, CFile::read);
 347                                 uint64 toReadWrite = std::min<uint64>(PARTSIZE, inputfile.GetLength());
 348                                 inputfile.Read(ba, toReadWrite);
 349                                 inputfile.Close();
 350
 351                                 buffer = wxString::Format(_("Saving data block into new single download file (%u of %u)"), curindex, partfilecount);
 352
 353                                 Notify_ConvertUpdateProgress(10 + (curindex * stepperpart), buffer);
 354
 355                                 // write the buffered data
 356                                 file->m_hpartfile.WriteAt(ba, chunkstart, toReadWrite);
 357
 358                                 filename = finder.GetNextFile();
 359                         }
 360                         delete[] ba;
 361                 } catch (const CSafeIOException& e) {
 362                         AddDebugLogLineM(true, logPfConvert, wxT("IO error while converting partfiles: ") + e.what());
 363
 364                         delete[] ba;
 365                         file->Delete();
 366                         return CONV_IOERROR;
 367                 }
 368
 369                 file->m_hpartfile.Close();
 370         }
 371         // import an external common format partdownload
 372         else //if (pfconverting->partmettype==PMT_DEFAULTOLD || pfconverting->partmettype==PMT_NEWOLD || Shareaza  )
 373         {
 374                 if (!s_pfconverting->removeSource) {
 375                         wxMutexLocker lock(s_mutex);
 376                         s_pfconverting->spaceneeded = oldfile.GetFileSize();
 377                 }
 378
 379                 Notify_ConvertUpdateJobInfo(s_pfconverting);
 380
 381                 sint64 freespace = CPath::GetFreeSpaceAt(thePrefs::GetTempDir());
 382                 if (freespace == wxInvalidOffset) {
 383                         delete file;
 384                         return CONV_IOERROR;
 385                 } else if (!s_pfconverting->removeSource && (freespace < s_pfconverting->spaceneeded)) {
 386                         delete file;
 387                         return CONV_OUTOFDISKSPACE;
 388                 }
 389
 390                 file->CreatePartFile();
 391                 newfilename = file->GetFullName();
 392
 393                 file->m_hpartfile.Close();
 394
 395                 bool ret = false;
 396
 397                 Notify_ConvertUpdateProgress(92, _("Copy"));
 398
 399                 CPath::RemoveFile(newfilename.RemoveExt());
 400                 if (!oldfile.FileExists()) {
 401                         // data file does not exist. well, then create a 0 byte big one
 402                         CFile datafile;
 403                         ret = datafile.Create(newfilename.RemoveExt());
 404                 } else if (s_pfconverting->removeSource) {
 405                         ret = CPath::RenameFile(oldfile, newfilename.RemoveExt());
 406                 } else {
 407                         ret = CPath::CloneFile(oldfile, newfilename.RemoveExt(), false);
 408                 }
 409                 if (!ret) {
 410                         file->Delete();
 411                         //delete file;
 412                         return CONV_FAILED;
 413                 }
 414
 415         }
 416
 417         Notify_ConvertUpdateProgress(94, _("Retrieving source downloadfile information"));
 418
 419         CPath::RemoveFile(newfilename);
 420         if (s_pfconverting->removeSource) {
 421                 CPath::RenameFile(folder.JoinPaths(partfile), newfilename);
 422         } else {
 423                 CPath::CloneFile(folder.JoinPaths(partfile), newfilename, false);
 424         }
 425
 426         file->m_hashlist.clear();
 427
 428         if (!file->LoadPartFile(thePrefs::GetTempDir(), file->GetPartMetFileName(), false)) {
 429                 //delete file;
 430                 file->Delete();
 431                 return CONV_BADFORMAT;
 432         }
 433
 434         if (s_pfconverting->partmettype == PMT_NEWOLD || s_pfconverting->partmettype == PMT_SPLITTED ) {
 435                 file->SetCompletedSize(file->transferred);
 436                 file->m_iGainDueToCompression = 0;
 437                 file->m_iLostDueToCorruption = 0;
 438         }
 439
 440         Notify_ConvertUpdateProgress(100, _("Adding download and saving new partfile"));
 441
 442         theApp->downloadqueue->AddDownload(file, thePrefs::AddNewFilesPaused(), 0);
 443         file->SavePartFile();
 444
 445         if (file->GetStatus(true) == PS_READY) {
 446                 theApp->sharedfiles->SafeAddKFile(file); // part files are always shared files
 447         }
 448
 449         if (s_pfconverting->removeSource) {
 450                 CPath oldFile = finder.GetFirstFile(CDirIterator::File, filepartindex + wxT(".*"));
 451                 while (oldFile.IsOk()) {
 452                         CPath::RemoveFile(folder.JoinPaths(oldFile));
 453                         oldFile = finder.GetNextFile();
 454                 }
 455
 456                 if (s_pfconverting->partmettype == PMT_SPLITTED) {
 457                         CPath::RemoveDir(folder);
 458                 }
 459         }
 460
 461         return CONV_OK;
 462 }
 463
 464 // Notification handlers
 465
 466 void CPartFileConvert::RemoveJob(unsigned id)
 467 {
 468         wxMutexLocker lock(s_mutex);
 469         for (std::list<ConvertJob*>::iterator it = s_jobs.begin(); it != s_jobs.end(); ++it) {
 470                 if ((*it)->id == id && (*it)->state != CONV_INPROGRESS) {
 471                         ConvertJob *job = *it;
 472                         s_jobs.erase(it);
 473                         Notify_ConvertRemoveJobInfo(id);
 474                         delete job;
 475                         break;
 476                 }
 477         }
 478 }
 479
 480 void CPartFileConvert::RetryJob(unsigned id)
 481 {
 482         wxMutexLocker lock(s_mutex);
 483         for (std::list<ConvertJob*>::iterator it = s_jobs.begin(); it != s_jobs.end(); ++it) {
 484                 if ((*it)->id == id && (*it)->state != CONV_INPROGRESS && (*it)->state != CONV_OK) {
 485                         (*it)->state = CONV_QUEUE;
 486                         Notify_ConvertUpdateJobInfo(*it);
 487                         StartThread();
 488                         break;
 489                 }
 490         }
 491 }
 492
 493 void CPartFileConvert::ReaddAllJobs()
 494 {
 495         wxMutexLocker lock(s_mutex);
 496         for (std::list<ConvertJob*>::iterator it = s_jobs.begin(); it != s_jobs.end(); ++it) {
 497                 Notify_ConvertUpdateJobInfo(*it);
 498         }
 499 }