Upstream tarball 9690
[amule.git] / src / ThreadTasks.cpp
blobc5abe0d6389e71e0c2831b77b73f8ec7fb0f2596
1 //
2 // This file is part of the aMule Project.
3 //
4 // Copyright (c) 2006-2008 Mikkel Schubert ( xaignar@amule.org / http:://www.amule.org )
5 // Copyright (c) 2003-2008 aMule Team ( admin@amule.org / http://www.amule.org )
6 // Copyright (c) 2002-2008 Merkur ( devs@emule-project.net / http://www.emule-project.net )
7 //
8 // Any parts of this program derived from the xMule, lMule or eMule project,
9 // or contributed by third-party developers are copyrighted by their
10 // respective authors.
12 // This program is free software; you can redistribute it and/or modify
13 // it under the terms of the GNU General Public License as published by
14 // the Free Software Foundation; either version 2 of the License, or
15 // (at your option) any later version.
17 // This program is distributed in the hope that it will be useful,
18 // but WITHOUT ANY WARRANTY; without even the implied warranty of
19 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 // GNU General Public License for more details.
21 //
22 // You should have received a copy of the GNU General Public License
23 // along with this program; if not, write to the Free Software
24 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
28 #include <wx/app.h> // Needed for wxTheApp
30 #include "ThreadTasks.h" // Interface declarations
31 #include "PartFile.h" // Needed for CPartFile
32 #include "Logger.h" // Needed for Add(Debug)LogLineM
33 #include <common/Format.h> // Needed for CFormat
34 #include "amule.h" // Needed for theApp
35 #include "KnownFileList.h" // Needed for theApp->knownfiles
36 #include "Preferences.h" // Needed for thePrefs
37 #include "ScopedPtr.h" // Needed for CScopedPtr and CScopedArray
38 #include "PlatformSpecific.h" // Needed for CanFSHandleSpecialChars
41 //! This hash represents the value for an empty MD4 hashing
42 const byte g_emptyMD4Hash[16] = {
43 0x31, 0xD6, 0xCF, 0xE0, 0xD1, 0x6A, 0xE9, 0x31,
44 0xB7, 0x3C, 0x59, 0xD7, 0xE0, 0xC0, 0x89, 0xC0 };
47 ////////////////////////////////////////////////////////////
48 // CHashingTask
50 CHashingTask::CHashingTask(const CPath& path, const CPath& filename, const CPartFile* part)
51 // GetPrintable is used to improve the readability of the log.
52 : CThreadTask(wxT("Hashing"), path.JoinPaths(filename).GetPrintable(), (part ? ETP_High : ETP_Normal)),
53 m_path(path),
54 m_filename(filename),
55 m_toHash((EHashes)(EH_MD4 | EH_AICH)),
56 m_owner(part)
58 // We can only create the AICH hashset if the file is a knownfile or
59 // if the partfile is complete, since the MD4 hashset is checked first,
60 // so that the AICH hashset only gets assigned if the MD4 hashset
61 // matches what we expected. Due to the rareity of post-completion
62 // corruptions, this gives us a nice speedup in most cases.
63 if (part && !part->GetGapList().empty()) {
64 m_toHash = EH_MD4;
69 CHashingTask::CHashingTask(const CKnownFile* toAICHHash)
70 // GetPrintable is used to improve the readability of the log.
71 : CThreadTask(wxT("AICH Hashing"), toAICHHash->GetFilePath().JoinPaths(toAICHHash->GetFileName()).GetPrintable(), ETP_Low),
72 m_path(toAICHHash->GetFilePath()),
73 m_filename(toAICHHash->GetFileName()),
74 m_toHash(EH_AICH),
75 m_owner(toAICHHash)
80 void CHashingTask::Entry()
82 CFileAutoClose file;
84 CPath fullPath = m_path.JoinPaths(m_filename);
85 if (!file.Open(fullPath, CFile::read)) {
86 AddDebugLogLineM(true, logHasher,
87 CFormat(wxT("Warning, failed to open file, skipping: %s")) % fullPath);
88 return;
91 uint64 fileLength = 0;
92 try {
93 fileLength = file.GetLength();
94 } catch (const CIOFailureException&) {
95 AddDebugLogLineM(true, logHasher,
96 CFormat(wxT("Warning, failed to retrieve file-length, skipping: %s")) % fullPath);
97 return;
100 if (fileLength > MAX_FILE_SIZE) {
101 AddDebugLogLineM(true, logHasher,
102 CFormat(wxT("Warning, file is larger than supported size, skipping: %s")) % fullPath);
103 return;
104 } else if (fileLength == 0) {
105 if (m_owner) {
106 // It makes no sense to try to hash empty partfiles ...
107 wxFAIL;
108 } else {
109 // Zero-size partfiles should be hashed, but not zero-sized shared-files.
110 AddDebugLogLineM( true, logHasher,
111 CFormat(wxT("Warning, 0-size file, skipping: %s")) % fullPath);
114 return;
117 // For thread-safety, results are passed via a temporary file object.
118 CScopedPtr<CKnownFile> knownfile(new CKnownFile());
119 knownfile->m_filePath = m_path;
120 knownfile->SetFileName(m_filename);
121 knownfile->SetFileSize(fileLength);
122 knownfile->m_lastDateChanged = CPath::GetModificationTime(fullPath);
123 knownfile->m_AvailPartFrequency.insert(
124 knownfile->m_AvailPartFrequency.begin(),
125 knownfile->GetPartCount(), 0);
127 if ((m_toHash & EH_MD4) && (m_toHash & EH_AICH)) {
128 knownfile->GetAICHHashset()->FreeHashSet();
129 AddDebugLogLineM( false, logHasher, CFormat(
130 _("Starting to create MD4 and AICH hash for file: %s")) %
131 m_filename );
132 } else if ((m_toHash & EH_MD4)) {
133 AddDebugLogLineM( false, logHasher, CFormat(
134 _("Starting to create MD4 hash for file: %s")) % m_filename );
135 } else if ((m_toHash & EH_AICH)) {
136 knownfile->GetAICHHashset()->FreeHashSet();
137 AddDebugLogLineM( false, logHasher, CFormat(
138 _("Starting to create AICH hash for file: %s")) % m_filename );
139 } else {
140 wxCHECK_RET(0, (CFormat(wxT("No hashes requested for file, skipping: %s"))
141 % m_filename).GetString());
145 // This loops creates the part-hashes, loop-de-loop.
146 try {
147 for (uint16 part = 0; part < knownfile->GetPartCount() && !TestDestroy(); part++) {
148 if (CreateNextPartHash(file, part, knownfile.get(), m_toHash) == false) {
149 AddDebugLogLineM(true, logHasher,
150 CFormat(wxT("Error while hashing file, skipping: %s"))
151 % m_filename);
153 return;
156 } catch (const CSafeIOException& e) {
157 AddDebugLogLineM(true, logHasher, wxT("IO exception while hashing file: ") + e.what());
158 return;
161 if ((m_toHash & EH_MD4) && !TestDestroy()) {
162 // If the file is < PARTSIZE, then the filehash is that one hash,
163 // otherwise, the filehash is the hash of the parthashes
164 if ( knownfile->m_hashlist.size() == 1 ) {
165 knownfile->m_abyFileHash = knownfile->m_hashlist[0];
166 knownfile->m_hashlist.clear();
167 } else if ( knownfile->m_hashlist.size() ) {
168 CMD4Hash hash;
169 knownfile->CreateHashFromHashlist(knownfile->m_hashlist, &hash);
170 knownfile->m_abyFileHash = hash;
171 } else {
172 // This should not happen!
173 wxFAIL;
177 // Did we create a AICH hashset?
178 if ((m_toHash & EH_AICH) && !TestDestroy()) {
179 CAICHHashSet* AICHHashSet = knownfile->GetAICHHashset();
181 AICHHashSet->ReCalculateHash(false);
182 if (AICHHashSet->VerifyHashTree(true) ) {
183 AICHHashSet->SetStatus(AICH_HASHSETCOMPLETE);
184 if (!AICHHashSet->SaveHashSet()) {
185 AddDebugLogLineM( true, logHasher,
186 CFormat(wxT("Warning, failed to save AICH hashset for file: %s"))
187 % m_filename );
192 if ((m_toHash == EH_AICH) && !TestDestroy()) {
193 CHashingEvent evt(MULE_EVT_AICH_HASHING, knownfile.release(), m_owner);
195 wxPostEvent(wxTheApp, evt);
196 } else if (!TestDestroy()) {
197 CHashingEvent evt(MULE_EVT_HASHING, knownfile.release(), m_owner);
199 wxPostEvent(wxTheApp, evt);
204 bool CHashingTask::CreateNextPartHash(CFileAutoClose& file, uint16 part, CKnownFile* owner, EHashes toHash)
206 wxCHECK_MSG(!file.Eof(), false, wxT("Unexpected EOF in CreateNextPartHash"));
208 const uint64 offset = part * PARTSIZE;
209 // We'll read at most PARTSIZE bytes per cycle
210 const uint64 partLength = owner->GetPartSize(part);
212 CMD4Hash hash;
213 CMD4Hash* md4Hash = ((toHash & EH_MD4) ? &hash : NULL);
214 CAICHHashTree* aichHash = NULL;
216 // Setup for AICH hashing
217 if (toHash & EH_AICH) {
218 aichHash = owner->GetAICHHashset()->m_pHashTree.FindHash(offset, partLength);
221 owner->CreateHashFromFile(file, offset, partLength, md4Hash, aichHash);
223 if (toHash & EH_MD4) {
224 // Store the md4 hash
225 owner->m_hashlist.push_back(hash);
227 // This is because of the ed2k implementation for parts. A 2 * PARTSIZE
228 // file i.e. will have 3 parts (see CKnownFile::SetFileSize for comments).
229 // So we have to create the hash for the 0-size data, which will be the default
230 // md4 hash for null data: 31D6CFE0D16AE931B73C59D7E0C089C0
231 if ((partLength == PARTSIZE) && file.Eof()) {
232 owner->m_hashlist.push_back(CMD4Hash(g_emptyMD4Hash));
236 return true;
240 void CHashingTask::OnLastTask()
242 if (GetType() == wxT("Hashing")) {
243 // To prevent rehashing in case of crashes, we
244 // explicity save the list of hashed files here.
245 theApp->knownfiles->Save();
247 // Make sure the AICH-hashes are up to date.
248 CThreadScheduler::AddTask(new CAICHSyncTask());
253 ////////////////////////////////////////////////////////////
254 // CAICHSyncTask
256 CAICHSyncTask::CAICHSyncTask()
257 : CThreadTask(wxT("AICH Syncronizing"), wxEmptyString, ETP_Low)
262 void CAICHSyncTask::Entry()
264 ConvertToKnown2ToKnown264();
266 AddDebugLogLineM( false, logAICHThread, wxT("Syncronization thread started.") );
268 // We collect all masterhashs which we find in the known2.met and store them in a list
269 std::list<CAICHHash> hashlist;
270 const CPath fullpath = CPath(theApp->ConfigDir + KNOWN2_MET_FILENAME);
272 CFile file;
273 if (!file.Open(fullpath, (fullpath.FileExists() ? CFile::read_write : CFile::write))) {
274 AddDebugLogLineM( true, logAICHThread, wxT("Error, failed to open 'known2_64.met' file!") );
275 return;
278 uint32 nLastVerifiedPos = 0;
279 try {
280 if (file.Eof()) {
281 file.WriteUInt8(KNOWN2_MET_VERSION);
282 } else {
283 if (file.ReadUInt8() != KNOWN2_MET_VERSION) {
284 throw CEOFException(wxT("Invalid met-file header found, removing file."));
287 uint64 nExistingSize = file.GetLength();
288 while (file.GetPosition() < nExistingSize) {
289 // Read the next hash
290 hashlist.push_back(CAICHHash(&file));
292 uint32 nHashCount = file.ReadUInt32();
293 if (file.GetPosition() + nHashCount * CAICHHash::GetHashSize() > nExistingSize){
294 throw CEOFException(wxT("Hashlist ends past end of file."));
297 // skip the rest of this hashset
298 nLastVerifiedPos = file.Seek(nHashCount * HASHSIZE, wxFromCurrent);
301 } catch (const CEOFException&) {
302 AddDebugLogLineM(true, logAICHThread, wxT("Hashlist corrupted, truncating file."));
303 file.SetLength(nLastVerifiedPos);
304 } catch (const CIOFailureException& e) {
305 AddDebugLogLineM(true, logAICHThread, wxT("IO failure while reading hashlist (Aborting): ") + e.what());
307 return;
310 AddDebugLogLineM( false, logAICHThread, wxT("Masterhashes of known files have been loaded.") );
312 // Now we check that all files which are in the sharedfilelist have a
313 // corresponding hash in our list. Those how don't are queued for hashing.
314 theApp->sharedfiles->CheckAICHHashes(hashlist);
318 bool CAICHSyncTask::ConvertToKnown2ToKnown264()
320 // converting known2.met to known2_64.met to support large files
321 // changing hashcount from uint16 to uint32
323 const CPath oldfullpath = CPath(theApp->ConfigDir + OLD_KNOWN2_MET_FILENAME);
324 const CPath newfullpath = CPath(theApp->ConfigDir + KNOWN2_MET_FILENAME);
326 if (newfullpath.FileExists() || !oldfullpath.FileExists()) {
327 // In this case, there is nothing that we need to do.
328 return false;
331 CFile oldfile;
332 CFile newfile;
334 if (!oldfile.Open(oldfullpath, CFile::read)) {
335 AddDebugLogLineM(true, logAICHThread, wxT("Failed to open 'known2.met' file."));
337 // else -> known2.met also doesn't exists, so nothing to convert
338 return false;
342 if (!newfile.Open(newfullpath, CFile::write_excl)) {
343 AddDebugLogLineM(true, logAICHThread, wxT("Failed to create 'known2_64.met' file."));
345 return false;
348 AddLogLineM(false, CFormat(_("Converting old AICH hashsets in '%s' to 64b in '%s'."))
349 % OLD_KNOWN2_MET_FILENAME % KNOWN2_MET_FILENAME);
351 try {
352 newfile.WriteUInt8(KNOWN2_MET_VERSION);
354 while (newfile.GetPosition() < oldfile.GetLength()) {
355 CAICHHash aichHash(&oldfile);
356 uint32 nHashCount = oldfile.ReadUInt16();
358 CScopedArray<byte> buffer(nHashCount * CAICHHash::GetHashSize());
360 oldfile.Read(buffer.get(), nHashCount * CAICHHash::GetHashSize());
361 newfile.Write(aichHash.GetRawHash(), CAICHHash::GetHashSize());
362 newfile.WriteUInt32(nHashCount);
363 newfile.Write(buffer.get(), nHashCount * CAICHHash::GetHashSize());
365 newfile.Flush();
366 } catch (const CEOFException& e) {
367 AddDebugLogLineM(true, logAICHThread, wxT("Error reading old 'known2.met' file.") + e.what());
368 return false;
369 } catch (const CIOFailureException& e) {
370 AddDebugLogLineM(true, logAICHThread, wxT("IO error while converting 'known2.met' file: ") + e.what());
371 return false;
374 // FIXME LARGE FILES (uncomment)
375 //DeleteFile(oldfullpath);
377 return true;
382 ////////////////////////////////////////////////////////////
383 // CCompletionTask
386 CCompletionTask::CCompletionTask(const CPartFile* file)
387 // GetPrintable is used to improve the readability of the log.
388 : CThreadTask(wxT("Completing"), file->GetFullName().GetPrintable(), ETP_High),
389 m_filename(file->GetFileName()),
390 m_metPath(file->GetFullName()),
391 m_category(file->GetCategory()),
392 m_owner(file),
393 m_error(false)
395 wxASSERT(m_filename.IsOk());
396 wxASSERT(m_metPath.IsOk());
397 wxASSERT(m_owner);
401 void CCompletionTask::Entry()
403 CPath targetPath;
406 #ifndef AMULE_DAEMON
407 // Prevent the preference values from changing underneeth us.
408 wxMutexGuiLocker guiLock;
409 #else
410 //#warning Thread-safety needed
411 #endif
413 targetPath = theApp->glob_prefs->GetCategory(m_category)->path;
414 if (!targetPath.DirExists()) {
415 targetPath = thePrefs::GetIncomingDir();
419 CPath dstName = m_filename.Cleanup(true, !PlatformSpecific::CanFSHandleSpecialChars(targetPath));
421 // Avoid empty filenames ...
422 if (!dstName.IsOk()) {
423 dstName = CPath(wxT("Unknown"));
426 if (m_filename != dstName) {
427 AddDebugLogLineM(true, logPartFile, CFormat(_("WARNING: The filename '%s' is invalid and has been renamed to '%s'."))
428 % m_filename % dstName);
431 // Avoid saving to an already existing filename
432 CPath newName = targetPath.JoinPaths(dstName);
433 for (unsigned count = 0; newName.FileExists(); ++count) {
434 wxString postfix = wxString::Format(wxT("(%u)"), count);
436 newName = targetPath.JoinPaths(dstName.AddPostfix(postfix));
439 if (newName != targetPath.JoinPaths(dstName)) {
440 AddDebugLogLineM(true, logPartFile, CFormat(_("WARNING: The file '%s' already exists, new file renamed to '%s'."))
441 % dstName % newName.GetFullName());
444 // Move will handle dirs on the same partition, otherwise copy is needed.
445 CPath partfilename = m_metPath.RemoveExt();
446 if (!CPath::RenameFile(partfilename, newName)) {
447 if (!CPath::CloneFile(partfilename, newName, true)) {
448 m_error = true;
449 return;
452 if (!CPath::RemoveFile(partfilename)) {
453 AddDebugLogLineM(true, logPartFile, CFormat(_("WARNING: Could not remove original '%s' after creating backup"))
454 % partfilename);
458 // Removes the various other data-files
459 const wxChar* otherMetExt[] = { wxT(""), PARTMET_BAK_EXT, wxT(".seeds"), NULL };
460 for (size_t i = 0; otherMetExt[i]; ++i) {
461 CPath toRemove = m_metPath.AppendExt(otherMetExt[i]);
463 if (toRemove.FileExists()) {
464 if (!CPath::RemoveFile(toRemove)) {
465 AddDebugLogLineM(true, logPartFile, CFormat(_("WARNING: Failed to delete %s")) % toRemove);
470 m_newName = newName;
474 void CCompletionTask::OnExit()
476 // Notify the app that the completion has finished for this file.
477 CCompletionEvent evt(m_error, m_owner, m_newName);
479 wxPostEvent(wxTheApp, evt);
484 ////////////////////////////////////////////////////////////
485 // CAllocateFileTask
487 #ifdef HAVE_FALLOCATE
488 # include <linux/falloc.h>
489 #elif defined HAVE_SYS_FALLOCATE
490 # include <sys/syscall.h>
491 # include <sys/types.h>
492 # include <unistd.h>
493 #elif defined HAVE_POSIX_FALLOCATE
494 # define _XOPEN_SOURCE 600
495 # include <stdlib.h>
496 # ifdef HAVE_FCNTL_H
497 # include <fcntl.h>
498 # endif
499 #endif
500 #include <stdlib.h>
501 #include <errno.h>
503 CAllocateFileTask::CAllocateFileTask(CPartFile *file, bool pause)
504 // GetPrintable is used to improve the readability of the log.
505 : CThreadTask(wxT("Allocating"), file->GetFullName().RemoveExt().GetPrintable(), ETP_High),
506 m_file(file), m_pause(pause), m_result(ENOSYS)
508 wxASSERT(file != NULL);
511 void CAllocateFileTask::Entry()
513 if (m_file->GetFileSize() == 0) {
514 m_result = 0;
515 return;
518 uint64_t minFree = thePrefs::IsCheckDiskspaceEnabled() ? thePrefs::GetMinFreeDiskSpace() : 0;
519 int64_t freeSpace = CPath::GetFreeSpaceAt(thePrefs::GetTempDir());
521 // Don't even try to allocate, if there's no space to complete the operation.
522 if (freeSpace != wxInvalidOffset) {
523 if ((uint64_t)freeSpace < m_file->GetFileSize() + minFree) {
524 m_result = ENOSPC;
525 return;
529 CFile file;
530 file.Open(m_file->GetFullName().RemoveExt(), CFile::read_write);
532 #ifdef __WXMSW__
533 try {
534 // File is already created as non-sparse, so we only need to set the length.
535 // This will fail to allocate the file e.g. under wine on linux/ext3,
536 // but works with NTFS and FAT32.
537 file.Seek(m_file->GetFileSize() - 1, wxFromStart);
538 file.WriteUInt8(0);
539 file.Close();
540 m_result = 0;
541 } catch (const CSafeIOException&) {
542 m_result = errno;
544 #else
545 // Use kernel level routines if possible
546 # ifdef HAVE_FALLOCATE
547 m_result = fallocate(file.fd(), 0, 0, m_file->GetFileSize());
548 # elif defined HAVE_SYS_FALLOCATE
549 m_result = syscall(SYS_fallocate, file.fd(), 0, (loff_t)0, (loff_t)m_file->GetFileSize());
550 if (m_result == -1) {
551 m_result = errno;
553 # elif defined HAVE_POSIX_FALLOCATE
554 // otherwise use glibc implementation, if available
555 m_result = posix_fallocate(file.fd(), 0, m_file->GetFileSize());
556 # endif
558 if (m_result != 0 && m_result != ENOSPC) {
559 // If everything else fails, use slow-and-dirty method of allocating the file: write the whole file with zeroes.
560 # define BLOCK_SIZE 1048576 /* Write 1 MB blocks */
561 void *zero = calloc(1, BLOCK_SIZE);
562 if (zero != NULL) {
563 try {
564 uint64_t size = m_file->GetFileSize();
565 for (; size >= BLOCK_SIZE; size -= BLOCK_SIZE) {
566 file.Write(zero, BLOCK_SIZE);
568 if (size > 0) {
569 file.Write(zero, size);
571 file.Close();
572 m_result = 0;
573 } catch (const CSafeIOException&) {
574 m_result = errno;
576 free(zero);
577 } else {
578 m_result = ENOMEM;
582 #endif
583 if (file.IsOpened()) {
584 file.Close();
588 void CAllocateFileTask::OnExit()
590 // Notify the app that the preallocation has finished for this file.
591 CAllocFinishedEvent evt(m_file, m_pause, m_result);
593 wxPostEvent(wxTheApp, evt);
598 ////////////////////////////////////////////////////////////
599 // CHashingEvent
601 DEFINE_LOCAL_EVENT_TYPE(MULE_EVT_HASHING)
602 DEFINE_LOCAL_EVENT_TYPE(MULE_EVT_AICH_HASHING)
604 CHashingEvent::CHashingEvent(wxEventType type, CKnownFile* result, const CKnownFile* owner)
605 : wxEvent(-1, type),
606 m_owner(owner),
607 m_result(result)
612 wxEvent* CHashingEvent::Clone() const
614 return new CHashingEvent(GetEventType(), m_result, m_owner);
618 const CKnownFile* CHashingEvent::GetOwner() const
620 return m_owner;
624 CKnownFile* CHashingEvent::GetResult() const
626 return m_result;
632 ////////////////////////////////////////////////////////////
633 // CCompletionEvent
635 DEFINE_LOCAL_EVENT_TYPE(MULE_EVT_FILE_COMPLETED)
638 CCompletionEvent::CCompletionEvent(bool errorOccured, const CPartFile* owner, const CPath& fullPath)
639 : wxEvent(-1, MULE_EVT_FILE_COMPLETED),
640 m_fullPath(fullPath),
641 m_owner(owner),
642 m_error(errorOccured)
647 wxEvent* CCompletionEvent::Clone() const
649 return new CCompletionEvent(m_error, m_owner, m_fullPath);
653 bool CCompletionEvent::ErrorOccured() const
655 return m_error;
659 const CPartFile* CCompletionEvent::GetOwner() const
661 return m_owner;
665 const CPath& CCompletionEvent::GetFullPath() const
667 return m_fullPath;
671 ////////////////////////////////////////////////////////////
672 // CAllocFinishedEvent
674 DEFINE_LOCAL_EVENT_TYPE(MULE_EVT_ALLOC_FINISHED)
676 wxEvent *CAllocFinishedEvent::Clone() const
678 return new CAllocFinishedEvent(m_file, m_pause, m_result);
681 // File_checked_for_headers