Upstream tarball 9882
[amule.git] / src / ThreadTasks.cpp
blobb3ab5a528c62031610fd70d91620e8e4dba61178
1 //
2 // This file is part of the aMule Project.
3 //
4 // Copyright (c) 2006-2008 Mikkel Schubert ( xaignar@amule.org / http:://www.amule.org )
5 // Copyright (c) 2003-2008 aMule Team ( admin@amule.org / http://www.amule.org )
6 // Copyright (c) 2002-2008 Merkur ( devs@emule-project.net / http://www.emule-project.net )
7 //
8 // Any parts of this program derived from the xMule, lMule or eMule project,
9 // or contributed by third-party developers are copyrighted by their
10 // respective authors.
12 // This program is free software; you can redistribute it and/or modify
13 // it under the terms of the GNU General Public License as published by
14 // the Free Software Foundation; either version 2 of the License, or
15 // (at your option) any later version.
17 // This program is distributed in the hope that it will be useful,
18 // but WITHOUT ANY WARRANTY; without even the implied warranty of
19 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 // GNU General Public License for more details.
21 //
22 // You should have received a copy of the GNU General Public License
23 // along with this program; if not, write to the Free Software
24 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
28 #include <wx/app.h> // Needed for wxTheApp
30 #include "ThreadTasks.h" // Interface declarations
31 #include "PartFile.h" // Needed for CPartFile
32 #include "Logger.h" // Needed for Add(Debug)LogLineM
33 #include <common/Format.h> // Needed for CFormat
34 #include "amule.h" // Needed for theApp
35 #include "KnownFileList.h" // Needed for theApp->knownfiles
36 #include "Preferences.h" // Needed for thePrefs
37 #include "ScopedPtr.h" // Needed for CScopedPtr and CScopedArray
38 #include "PlatformSpecific.h" // Needed for CanFSHandleSpecialChars
40 #ifdef HAVE_CONFIG_H
41 # include "config.h"
42 #endif
44 //! This hash represents the value for an empty MD4 hashing
45 const byte g_emptyMD4Hash[16] = {
46 0x31, 0xD6, 0xCF, 0xE0, 0xD1, 0x6A, 0xE9, 0x31,
47 0xB7, 0x3C, 0x59, 0xD7, 0xE0, 0xC0, 0x89, 0xC0 };
50 ////////////////////////////////////////////////////////////
51 // CHashingTask
53 CHashingTask::CHashingTask(const CPath& path, const CPath& filename, const CPartFile* part)
54 // GetPrintable is used to improve the readability of the log.
55 : CThreadTask(wxT("Hashing"), path.JoinPaths(filename).GetPrintable(), (part ? ETP_High : ETP_Normal)),
56 m_path(path),
57 m_filename(filename),
58 m_toHash((EHashes)(EH_MD4 | EH_AICH)),
59 m_owner(part)
61 // We can only create the AICH hashset if the file is a knownfile or
62 // if the partfile is complete, since the MD4 hashset is checked first,
63 // so that the AICH hashset only gets assigned if the MD4 hashset
64 // matches what we expected. Due to the rareity of post-completion
65 // corruptions, this gives us a nice speedup in most cases.
66 if (part && !part->GetGapList().empty()) {
67 m_toHash = EH_MD4;
72 CHashingTask::CHashingTask(const CKnownFile* toAICHHash)
73 // GetPrintable is used to improve the readability of the log.
74 : CThreadTask(wxT("AICH Hashing"), toAICHHash->GetFilePath().JoinPaths(toAICHHash->GetFileName()).GetPrintable(), ETP_Low),
75 m_path(toAICHHash->GetFilePath()),
76 m_filename(toAICHHash->GetFileName()),
77 m_toHash(EH_AICH),
78 m_owner(toAICHHash)
83 void CHashingTask::Entry()
85 CFileAutoClose file;
87 CPath fullPath = m_path.JoinPaths(m_filename);
88 if (!file.Open(fullPath, CFile::read)) {
89 AddDebugLogLineM(true, logHasher,
90 CFormat(wxT("Warning, failed to open file, skipping: %s")) % fullPath);
91 return;
94 uint64 fileLength = 0;
95 try {
96 fileLength = file.GetLength();
97 } catch (const CIOFailureException&) {
98 AddDebugLogLineM(true, logHasher,
99 CFormat(wxT("Warning, failed to retrieve file-length, skipping: %s")) % fullPath);
100 return;
103 if (fileLength > MAX_FILE_SIZE) {
104 AddDebugLogLineM(true, logHasher,
105 CFormat(wxT("Warning, file is larger than supported size, skipping: %s")) % fullPath);
106 return;
107 } else if (fileLength == 0) {
108 if (m_owner) {
109 // It makes no sense to try to hash empty partfiles ...
110 wxFAIL;
111 } else {
112 // Zero-size partfiles should be hashed, but not zero-sized shared-files.
113 AddDebugLogLineM( true, logHasher,
114 CFormat(wxT("Warning, 0-size file, skipping: %s")) % fullPath);
117 return;
120 // For thread-safety, results are passed via a temporary file object.
121 CScopedPtr<CKnownFile> knownfile(new CKnownFile());
122 knownfile->m_filePath = m_path;
123 knownfile->SetFileName(m_filename);
124 knownfile->SetFileSize(fileLength);
125 knownfile->m_lastDateChanged = CPath::GetModificationTime(fullPath);
126 knownfile->m_AvailPartFrequency.insert(
127 knownfile->m_AvailPartFrequency.begin(),
128 knownfile->GetPartCount(), 0);
130 if ((m_toHash & EH_MD4) && (m_toHash & EH_AICH)) {
131 knownfile->GetAICHHashset()->FreeHashSet();
132 AddDebugLogLineM( false, logHasher, CFormat(
133 _("Starting to create MD4 and AICH hash for file: %s")) %
134 m_filename );
135 } else if ((m_toHash & EH_MD4)) {
136 AddDebugLogLineM( false, logHasher, CFormat(
137 _("Starting to create MD4 hash for file: %s")) % m_filename );
138 } else if ((m_toHash & EH_AICH)) {
139 knownfile->GetAICHHashset()->FreeHashSet();
140 AddDebugLogLineM( false, logHasher, CFormat(
141 _("Starting to create AICH hash for file: %s")) % m_filename );
142 } else {
143 wxCHECK_RET(0, (CFormat(wxT("No hashes requested for file, skipping: %s"))
144 % m_filename).GetString());
148 // This loops creates the part-hashes, loop-de-loop.
149 try {
150 for (uint16 part = 0; part < knownfile->GetPartCount() && !TestDestroy(); part++) {
151 if (CreateNextPartHash(file, part, knownfile.get(), m_toHash) == false) {
152 AddDebugLogLineM(true, logHasher,
153 CFormat(wxT("Error while hashing file, skipping: %s"))
154 % m_filename);
156 return;
159 } catch (const CSafeIOException& e) {
160 AddDebugLogLineM(true, logHasher, wxT("IO exception while hashing file: ") + e.what());
161 return;
164 if ((m_toHash & EH_MD4) && !TestDestroy()) {
165 // If the file is < PARTSIZE, then the filehash is that one hash,
166 // otherwise, the filehash is the hash of the parthashes
167 if ( knownfile->m_hashlist.size() == 1 ) {
168 knownfile->m_abyFileHash = knownfile->m_hashlist[0];
169 knownfile->m_hashlist.clear();
170 } else if ( knownfile->m_hashlist.size() ) {
171 CMD4Hash hash;
172 knownfile->CreateHashFromHashlist(knownfile->m_hashlist, &hash);
173 knownfile->m_abyFileHash = hash;
174 } else {
175 // This should not happen!
176 wxFAIL;
180 // Did we create a AICH hashset?
181 if ((m_toHash & EH_AICH) && !TestDestroy()) {
182 CAICHHashSet* AICHHashSet = knownfile->GetAICHHashset();
184 AICHHashSet->ReCalculateHash(false);
185 if (AICHHashSet->VerifyHashTree(true) ) {
186 AICHHashSet->SetStatus(AICH_HASHSETCOMPLETE);
187 if (!AICHHashSet->SaveHashSet()) {
188 AddDebugLogLineM( true, logHasher,
189 CFormat(wxT("Warning, failed to save AICH hashset for file: %s"))
190 % m_filename );
195 if ((m_toHash == EH_AICH) && !TestDestroy()) {
196 CHashingEvent evt(MULE_EVT_AICH_HASHING, knownfile.release(), m_owner);
198 wxPostEvent(wxTheApp, evt);
199 } else if (!TestDestroy()) {
200 CHashingEvent evt(MULE_EVT_HASHING, knownfile.release(), m_owner);
202 wxPostEvent(wxTheApp, evt);
207 bool CHashingTask::CreateNextPartHash(CFileAutoClose& file, uint16 part, CKnownFile* owner, EHashes toHash)
209 wxCHECK_MSG(!file.Eof(), false, wxT("Unexpected EOF in CreateNextPartHash"));
211 const uint64 offset = part * PARTSIZE;
212 // We'll read at most PARTSIZE bytes per cycle
213 const uint64 partLength = owner->GetPartSize(part);
215 CMD4Hash hash;
216 CMD4Hash* md4Hash = ((toHash & EH_MD4) ? &hash : NULL);
217 CAICHHashTree* aichHash = NULL;
219 // Setup for AICH hashing
220 if (toHash & EH_AICH) {
221 aichHash = owner->GetAICHHashset()->m_pHashTree.FindHash(offset, partLength);
224 owner->CreateHashFromFile(file, offset, partLength, md4Hash, aichHash);
226 if (toHash & EH_MD4) {
227 // Store the md4 hash
228 owner->m_hashlist.push_back(hash);
230 // This is because of the ed2k implementation for parts. A 2 * PARTSIZE
231 // file i.e. will have 3 parts (see CKnownFile::SetFileSize for comments).
232 // So we have to create the hash for the 0-size data, which will be the default
233 // md4 hash for null data: 31D6CFE0D16AE931B73C59D7E0C089C0
234 if ((partLength == PARTSIZE) && file.Eof()) {
235 owner->m_hashlist.push_back(CMD4Hash(g_emptyMD4Hash));
239 return true;
243 void CHashingTask::OnLastTask()
245 if (GetType() == wxT("Hashing")) {
246 // To prevent rehashing in case of crashes, we
247 // explicity save the list of hashed files here.
248 theApp->knownfiles->Save();
250 // Make sure the AICH-hashes are up to date.
251 CThreadScheduler::AddTask(new CAICHSyncTask());
256 ////////////////////////////////////////////////////////////
257 // CAICHSyncTask
259 CAICHSyncTask::CAICHSyncTask()
260 : CThreadTask(wxT("AICH Syncronizing"), wxEmptyString, ETP_Low)
265 void CAICHSyncTask::Entry()
267 ConvertToKnown2ToKnown264();
269 AddDebugLogLineM( false, logAICHThread, wxT("Syncronization thread started.") );
271 // We collect all masterhashs which we find in the known2.met and store them in a list
272 std::list<CAICHHash> hashlist;
273 const CPath fullpath = CPath(theApp->ConfigDir + KNOWN2_MET_FILENAME);
275 CFile file;
276 if (!file.Open(fullpath, (fullpath.FileExists() ? CFile::read_write : CFile::write))) {
277 AddDebugLogLineM( true, logAICHThread, wxT("Error, failed to open 'known2_64.met' file!") );
278 return;
281 uint32 nLastVerifiedPos = 0;
282 try {
283 if (file.Eof()) {
284 file.WriteUInt8(KNOWN2_MET_VERSION);
285 } else {
286 if (file.ReadUInt8() != KNOWN2_MET_VERSION) {
287 throw CEOFException(wxT("Invalid met-file header found, removing file."));
290 uint64 nExistingSize = file.GetLength();
291 while (file.GetPosition() < nExistingSize) {
292 // Read the next hash
293 hashlist.push_back(CAICHHash(&file));
295 uint32 nHashCount = file.ReadUInt32();
296 if (file.GetPosition() + nHashCount * CAICHHash::GetHashSize() > nExistingSize){
297 throw CEOFException(wxT("Hashlist ends past end of file."));
300 // skip the rest of this hashset
301 nLastVerifiedPos = file.Seek(nHashCount * HASHSIZE, wxFromCurrent);
304 } catch (const CEOFException&) {
305 AddDebugLogLineM(true, logAICHThread, wxT("Hashlist corrupted, truncating file."));
306 file.SetLength(nLastVerifiedPos);
307 } catch (const CIOFailureException& e) {
308 AddDebugLogLineM(true, logAICHThread, wxT("IO failure while reading hashlist (Aborting): ") + e.what());
310 return;
313 AddDebugLogLineM( false, logAICHThread, wxT("Masterhashes of known files have been loaded.") );
315 // Now we check that all files which are in the sharedfilelist have a
316 // corresponding hash in our list. Those how don't are queued for hashing.
317 theApp->sharedfiles->CheckAICHHashes(hashlist);
321 bool CAICHSyncTask::ConvertToKnown2ToKnown264()
323 // converting known2.met to known2_64.met to support large files
324 // changing hashcount from uint16 to uint32
326 const CPath oldfullpath = CPath(theApp->ConfigDir + OLD_KNOWN2_MET_FILENAME);
327 const CPath newfullpath = CPath(theApp->ConfigDir + KNOWN2_MET_FILENAME);
329 if (newfullpath.FileExists() || !oldfullpath.FileExists()) {
330 // In this case, there is nothing that we need to do.
331 return false;
334 CFile oldfile;
335 CFile newfile;
337 if (!oldfile.Open(oldfullpath, CFile::read)) {
338 AddDebugLogLineM(true, logAICHThread, wxT("Failed to open 'known2.met' file."));
340 // else -> known2.met also doesn't exists, so nothing to convert
341 return false;
345 if (!newfile.Open(newfullpath, CFile::write_excl)) {
346 AddDebugLogLineM(true, logAICHThread, wxT("Failed to create 'known2_64.met' file."));
348 return false;
351 AddLogLineM(false, CFormat(_("Converting old AICH hashsets in '%s' to 64b in '%s'."))
352 % OLD_KNOWN2_MET_FILENAME % KNOWN2_MET_FILENAME);
354 try {
355 newfile.WriteUInt8(KNOWN2_MET_VERSION);
357 while (newfile.GetPosition() < oldfile.GetLength()) {
358 CAICHHash aichHash(&oldfile);
359 uint32 nHashCount = oldfile.ReadUInt16();
361 CScopedArray<byte> buffer(nHashCount * CAICHHash::GetHashSize());
363 oldfile.Read(buffer.get(), nHashCount * CAICHHash::GetHashSize());
364 newfile.Write(aichHash.GetRawHash(), CAICHHash::GetHashSize());
365 newfile.WriteUInt32(nHashCount);
366 newfile.Write(buffer.get(), nHashCount * CAICHHash::GetHashSize());
368 newfile.Flush();
369 } catch (const CEOFException& e) {
370 AddDebugLogLineM(true, logAICHThread, wxT("Error reading old 'known2.met' file.") + e.what());
371 return false;
372 } catch (const CIOFailureException& e) {
373 AddDebugLogLineM(true, logAICHThread, wxT("IO error while converting 'known2.met' file: ") + e.what());
374 return false;
377 // FIXME LARGE FILES (uncomment)
378 //DeleteFile(oldfullpath);
380 return true;
385 ////////////////////////////////////////////////////////////
386 // CCompletionTask
389 CCompletionTask::CCompletionTask(const CPartFile* file)
390 // GetPrintable is used to improve the readability of the log.
391 : CThreadTask(wxT("Completing"), file->GetFullName().GetPrintable(), ETP_High),
392 m_filename(file->GetFileName()),
393 m_metPath(file->GetFullName()),
394 m_category(file->GetCategory()),
395 m_owner(file),
396 m_error(false)
398 wxASSERT(m_filename.IsOk());
399 wxASSERT(m_metPath.IsOk());
400 wxASSERT(m_owner);
404 void CCompletionTask::Entry()
406 CPath targetPath;
409 #ifndef AMULE_DAEMON
410 // Prevent the preference values from changing underneeth us.
411 wxMutexGuiLocker guiLock;
412 #else
413 //#warning Thread-safety needed
414 #endif
416 targetPath = theApp->glob_prefs->GetCategory(m_category)->path;
417 if (!targetPath.DirExists()) {
418 targetPath = thePrefs::GetIncomingDir();
422 CPath dstName = m_filename.Cleanup(true, !PlatformSpecific::CanFSHandleSpecialChars(targetPath));
424 // Avoid empty filenames ...
425 if (!dstName.IsOk()) {
426 dstName = CPath(wxT("Unknown"));
429 if (m_filename != dstName) {
430 AddDebugLogLineM(true, logPartFile, CFormat(_("WARNING: The filename '%s' is invalid and has been renamed to '%s'."))
431 % m_filename % dstName);
434 // Avoid saving to an already existing filename
435 CPath newName = targetPath.JoinPaths(dstName);
436 for (unsigned count = 0; newName.FileExists(); ++count) {
437 wxString postfix = wxString::Format(wxT("(%u)"), count);
439 newName = targetPath.JoinPaths(dstName.AddPostfix(postfix));
442 if (newName != targetPath.JoinPaths(dstName)) {
443 AddDebugLogLineM(true, logPartFile, CFormat(_("WARNING: The file '%s' already exists, new file renamed to '%s'."))
444 % dstName % newName.GetFullName());
447 // Move will handle dirs on the same partition, otherwise copy is needed.
448 CPath partfilename = m_metPath.RemoveExt();
449 if (!CPath::RenameFile(partfilename, newName)) {
450 if (!CPath::CloneFile(partfilename, newName, true)) {
451 m_error = true;
452 return;
455 if (!CPath::RemoveFile(partfilename)) {
456 AddDebugLogLineM(true, logPartFile, CFormat(_("WARNING: Could not remove original '%s' after creating backup"))
457 % partfilename);
461 // Removes the various other data-files
462 const wxChar* otherMetExt[] = { wxT(""), PARTMET_BAK_EXT, wxT(".seeds"), NULL };
463 for (size_t i = 0; otherMetExt[i]; ++i) {
464 CPath toRemove = m_metPath.AppendExt(otherMetExt[i]);
466 if (toRemove.FileExists()) {
467 if (!CPath::RemoveFile(toRemove)) {
468 AddDebugLogLineM(true, logPartFile, CFormat(_("WARNING: Failed to delete %s")) % toRemove);
473 m_newName = newName;
477 void CCompletionTask::OnExit()
479 // Notify the app that the completion has finished for this file.
480 CCompletionEvent evt(m_error, m_owner, m_newName);
482 wxPostEvent(wxTheApp, evt);
487 ////////////////////////////////////////////////////////////
488 // CAllocateFileTask
490 #ifdef HAVE_FALLOCATE
491 # define _GNU_SOURCE
492 # ifdef HAVE_FCNTL_H
493 # include <fcntl.h>
494 # endif
495 # include <linux/falloc.h>
496 #elif defined HAVE_SYS_FALLOCATE
497 # include <sys/syscall.h>
498 # include <sys/types.h>
499 # include <unistd.h>
500 #elif defined HAVE_POSIX_FALLOCATE
501 # define _XOPEN_SOURCE 600
502 # include <stdlib.h>
503 # ifdef HAVE_FCNTL_H
504 # include <fcntl.h>
505 # endif
506 #endif
507 #include <stdlib.h>
508 #include <errno.h>
510 CAllocateFileTask::CAllocateFileTask(CPartFile *file, bool pause)
511 // GetPrintable is used to improve the readability of the log.
512 : CThreadTask(wxT("Allocating"), file->GetFullName().RemoveExt().GetPrintable(), ETP_High),
513 m_file(file), m_pause(pause), m_result(ENOSYS)
515 wxASSERT(file != NULL);
518 void CAllocateFileTask::Entry()
520 if (m_file->GetFileSize() == 0) {
521 m_result = 0;
522 return;
525 uint64_t minFree = thePrefs::IsCheckDiskspaceEnabled() ? thePrefs::GetMinFreeDiskSpace() : 0;
526 int64_t freeSpace = CPath::GetFreeSpaceAt(thePrefs::GetTempDir());
528 // Don't even try to allocate, if there's no space to complete the operation.
529 if (freeSpace != wxInvalidOffset) {
530 if ((uint64_t)freeSpace < m_file->GetFileSize() + minFree) {
531 m_result = ENOSPC;
532 return;
536 CFile file;
537 file.Open(m_file->GetFullName().RemoveExt(), CFile::read_write);
539 #ifdef __WXMSW__
540 try {
541 // File is already created as non-sparse, so we only need to set the length.
542 // This will fail to allocate the file e.g. under wine on linux/ext3,
543 // but works with NTFS and FAT32.
544 file.Seek(m_file->GetFileSize() - 1, wxFromStart);
545 file.WriteUInt8(0);
546 file.Close();
547 m_result = 0;
548 } catch (const CSafeIOException&) {
549 m_result = errno;
551 #else
552 // Use kernel level routines if possible
553 # ifdef HAVE_FALLOCATE
554 m_result = fallocate(file.fd(), 0, 0, m_file->GetFileSize());
555 # elif defined HAVE_SYS_FALLOCATE
556 m_result = syscall(SYS_fallocate, file.fd(), 0, (loff_t)0, (loff_t)m_file->GetFileSize());
557 if (m_result == -1) {
558 m_result = errno;
560 # elif defined HAVE_POSIX_FALLOCATE
561 // otherwise use glibc implementation, if available
562 m_result = posix_fallocate(file.fd(), 0, m_file->GetFileSize());
563 # endif
565 if (m_result != 0 && m_result != ENOSPC) {
566 // If everything else fails, use slow-and-dirty method of allocating the file: write the whole file with zeroes.
567 # define BLOCK_SIZE 1048576 /* Write 1 MB blocks */
568 void *zero = calloc(1, BLOCK_SIZE);
569 if (zero != NULL) {
570 try {
571 uint64_t size = m_file->GetFileSize();
572 for (; size >= BLOCK_SIZE; size -= BLOCK_SIZE) {
573 file.Write(zero, BLOCK_SIZE);
575 if (size > 0) {
576 file.Write(zero, size);
578 file.Close();
579 m_result = 0;
580 } catch (const CSafeIOException&) {
581 m_result = errno;
583 free(zero);
584 } else {
585 m_result = ENOMEM;
589 #endif
590 if (file.IsOpened()) {
591 file.Close();
595 void CAllocateFileTask::OnExit()
597 // Notify the app that the preallocation has finished for this file.
598 CAllocFinishedEvent evt(m_file, m_pause, m_result);
600 wxPostEvent(wxTheApp, evt);
605 ////////////////////////////////////////////////////////////
606 // CHashingEvent
608 DEFINE_LOCAL_EVENT_TYPE(MULE_EVT_HASHING)
609 DEFINE_LOCAL_EVENT_TYPE(MULE_EVT_AICH_HASHING)
611 CHashingEvent::CHashingEvent(wxEventType type, CKnownFile* result, const CKnownFile* owner)
612 : wxEvent(-1, type),
613 m_owner(owner),
614 m_result(result)
619 wxEvent* CHashingEvent::Clone() const
621 return new CHashingEvent(GetEventType(), m_result, m_owner);
625 const CKnownFile* CHashingEvent::GetOwner() const
627 return m_owner;
631 CKnownFile* CHashingEvent::GetResult() const
633 return m_result;
639 ////////////////////////////////////////////////////////////
640 // CCompletionEvent
642 DEFINE_LOCAL_EVENT_TYPE(MULE_EVT_FILE_COMPLETED)
645 CCompletionEvent::CCompletionEvent(bool errorOccured, const CPartFile* owner, const CPath& fullPath)
646 : wxEvent(-1, MULE_EVT_FILE_COMPLETED),
647 m_fullPath(fullPath),
648 m_owner(owner),
649 m_error(errorOccured)
654 wxEvent* CCompletionEvent::Clone() const
656 return new CCompletionEvent(m_error, m_owner, m_fullPath);
660 bool CCompletionEvent::ErrorOccured() const
662 return m_error;
666 const CPartFile* CCompletionEvent::GetOwner() const
668 return m_owner;
672 const CPath& CCompletionEvent::GetFullPath() const
674 return m_fullPath;
678 ////////////////////////////////////////////////////////////
679 // CAllocFinishedEvent
681 DEFINE_LOCAL_EVENT_TYPE(MULE_EVT_ALLOC_FINISHED)
683 wxEvent *CAllocFinishedEvent::Clone() const
685 return new CAllocFinishedEvent(m_file, m_pause, m_result);
688 // File_checked_for_headers