src/DownloadQueue.cpp

   1 //
   2 // This file is part of the aMule Project.
   3 //
   4 // Copyright (c) 2003-2008 aMule Team ( admin@amule.org / http://www.amule.org )
   5 // Copyright (c) 2002-2008 Merkur ( devs@emule-project.net / http://www.emule-project.net )
   6 //
   7 // Any parts of this program derived from the xMule, lMule or eMule project,
   8 // or contributed by third-party developers are copyrighted by their
   9 // respective authors.
  10 //
  11 // This program is free software; you can redistribute it and/or modify
  12 // it under the terms of the GNU General Public License as published by
  13 // the Free Software Foundation; either version 2 of the License, or
  14 // (at your option) any later version.
  15 //
  16 // This program is distributed in the hope that it will be useful,
  17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 // GNU General Public License for more details.
  20 //
  21 // You should have received a copy of the GNU General Public License
  22 // along with this program; if not, write to the Free Software
  23 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
  24 //
  25
  26 #include "DownloadQueue.h"      // Interface declarations
  27
  28 #include <protocol/Protocols.h>
  29 #include <protocol/kad/Constants.h>
  30 #include <common/Macros.h>
  31 #include <common/MenuIDs.h>
  32 #include <common/Constants.h>
  33
  34 #include <wx/utils.h>
  35
  36 #include "Server.h"             // Needed for CServer
  37 #include "Packet.h"             // Needed for CPacket
  38 #include "MemFile.h"            // Needed for CMemFile
  39 #include "ClientList.h"         // Needed for CClientList
  40 #include "updownclient.h"       // Needed for CUpDownClient
  41 #include "ServerList.h"         // Needed for CServerList
  42 #include "ServerConnect.h"      // Needed for CServerConnect
  43 #include "ED2KLink.h"           // Needed for CED2KFileLink
  44 #include "SearchList.h"         // Needed for CSearchFile
  45 #include "SharedFileList.h"     // Needed for CSharedFileList
  46 #include "PartFile.h"           // Needed for CPartFile
  47 #include "Preferences.h"        // Needed for thePrefs
  48 #include "amule.h"              // Needed for theApp
  49 #include "AsyncDNS.h"           // Needed for CAsyncDNS
  50 #include "Statistics.h"         // Needed for theStats
  51 #include "Logger.h"
  52 #include <common/Format.h>      // Needed for CFormat
  53 #include "IPFilter.h"
  54 #include <common/FileFunctions.h>       // Needed for CDirIterator
  55 #include "GuiEvents.h"          // Needed for Notify_*
  56 #include "UserEvents.h"
  57 #include "MagnetURI.h"          // Needed for CMagnetED2KConverter
  58 #include "ScopedPtr.h"          // Needed for CScopedPtr
  59 #include "PlatformSpecific.h"   // Needed for CanFSHandleLargeFiles
  60
  61 #include "kademlia/kademlia/Kademlia.h"
  62
  63 #include <string>                       // Do_not_auto_remove (mingw-gcc-3.4.5)
  64
  65
  66 // Max. file IDs per UDP packet
  67 // ----------------------------
  68 // 576 - 30 bytes of header (28 for UDP, 2 for "E3 9A" edonkey proto) = 546 bytes
  69 // 546 / 16 = 34
  70
  71
  72 #define MAX_FILES_PER_UDP_PACKET        31      // 2+16*31 = 498 ... is still less than 512 bytes!!
  73 #define MAX_REQUESTS_PER_SERVER         35
  74
  75
  76 CDownloadQueue::CDownloadQueue()
  77 // Needs to be recursive that that is can own an observer assigned to itself
  78         : m_mutex( wxMUTEX_RECURSIVE )
  79 {
  80         m_datarate = 0;
  81         m_udpserver = 0;
  82         m_lastsorttime = 0;
  83         m_lastudpsearchtime = 0;
  84         m_lastudpstattime = 0;
  85         m_udcounter = 0;
  86         m_nLastED2KLinkCheck = 0;
  87         m_dwNextTCPSrcReq = 0;
  88         m_cRequestsSentToServer = 0;
  89         m_lastDiskCheck = 0;
  90         SetLastKademliaFileRequest();
  91 }
  92
  93
  94 CDownloadQueue::~CDownloadQueue()
  95 {
  96         if ( !m_filelist.empty() ) {
  97                 for ( unsigned int i = 0; i < m_filelist.size(); i++ ) {
  98                         AddLogLineNS(CFormat(_("Saving PartFile %u of %u")) % (i + 1) % m_filelist.size());
  99                         delete m_filelist[i];
 100                 }
 101                 AddLogLineNS(_("All PartFiles Saved."));
 102         }
 103 }
 104
 105
 106 void CDownloadQueue::LoadMetFiles(const CPath& path)
 107 {
 108         AddLogLineNS(CFormat(_("Loading temp files from %s.")) % path.GetPrintable());
 109
 110         std::vector<CPath> files;
 111
 112         // Locate part-files to be loaded
 113         CDirIterator TempDir(path);
 114         CPath fileName = TempDir.GetFirstFile(CDirIterator::File, wxT("*.part.met"));
 115         while (fileName.IsOk()) {
 116                 files.push_back(path.JoinPaths(fileName));
 117
 118                 fileName = TempDir.GetNextFile();
 119         }
 120
 121         // Loading in order makes it easier to figure which
 122         // file is broken in case of crashes, or the like.
 123         std::sort(files.begin(), files.end());
 124
 125         // Load part-files
 126         for ( size_t i = 0; i < files.size(); i++ ) {
 127                 AddLogLineNS(CFormat(_("Loading PartFile %u of %u")) % (i + 1) % files.size());
 128                 fileName = files[i].GetFullName();
 129                 CPartFile *toadd = new CPartFile();
 130                 bool result = toadd->LoadPartFile(path, fileName) != 0;
 131                 if (!result) {
 132                         // Try from backup
 133                         result = toadd->LoadPartFile(path, fileName, true) != 0;
 134                 }
 135                 if (result && !IsFileExisting(toadd->GetFileHash())) {
 136                         {
 137                                 wxMutexLocker lock(m_mutex);
 138                                 m_filelist.push_back(toadd);
 139                         }
 140                         NotifyObservers(EventType(EventType::INSERTED, toadd));
 141                         Notify_DownloadCtrlAddFile(toadd);
 142                 } else {
 143                         wxString msg;
 144                         if (result) {
 145                                 msg << CFormat(wxT("WARNING: Duplicate partfile with hash '%s' found, skipping: %s"))
 146                                         % toadd->GetFileHash().Encode() % fileName;
 147                         } else {
 148                                 // If result is false, then reading of both the primary and the backup .met failed
 149                                 AddLogLineM(false,
 150                                         _("ERROR: Failed to load backup file. Search http://forum.amule.org for .part.met recovery solutions."));
 151                                 msg << CFormat(wxT("ERROR: Failed to load PartFile '%s'")) % fileName;
 152                         }
 153                         AddLogLineCS(msg);
 154
 155                         // Delete the partfile object in the end.
 156                         delete toadd;
 157                 }
 158         }
 159         AddLogLineNS(_("All PartFiles Loaded."));
 160
 161         if ( GetFileCount() == 0 ) {
 162                 AddLogLineM(false, _("No part files found"));
 163         } else {
 164                 AddLogLineM(false, wxString::Format(wxPLURAL("Found %u part file", "Found %u part files", GetFileCount()), GetFileCount()) );
 165
 166                 DoSortByPriority();
 167                 CheckDiskspace( path );
 168         }
 169 }
 170
 171
 172 uint16 CDownloadQueue::GetFileCount() const
 173 {
 174         wxMutexLocker lock( m_mutex );
 175
 176         return m_filelist.size();
 177 }
 178
 179
 180 CServer* CDownloadQueue::GetUDPServer() const
 181 {
 182         wxMutexLocker lock( m_mutex );
 183
 184         return m_udpserver;
 185 }
 186
 187
 188 void CDownloadQueue::SetUDPServer( CServer* server )
 189 {
 190         wxMutexLocker lock( m_mutex );
 191
 192         m_udpserver = server;
 193 }
 194
 195
 196 void CDownloadQueue::SaveSourceSeeds()
 197 {
 198         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
 199                 GetFileByIndex( i )->SaveSourceSeeds();
 200         }
 201 }
 202
 203
 204 void CDownloadQueue::LoadSourceSeeds()
 205 {
 206         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
 207                 GetFileByIndex( i )->LoadSourceSeeds();
 208         }
 209 }
 210
 211
 212 void CDownloadQueue::AddSearchToDownload(CSearchFile* toadd, uint8 category)
 213 {
 214         if ( IsFileExisting(toadd->GetFileHash()) ) {
 215                 return;
 216         }
 217
 218         if (toadd->GetFileSize() > OLD_MAX_FILE_SIZE) {
 219                 if (!PlatformSpecific::CanFSHandleLargeFiles(thePrefs::GetTempDir())) {
 220                         AddLogLineM(true, _("Filesystem for Temp directory cannot handle large files."));
 221                         return;
 222                 } else if (!PlatformSpecific::CanFSHandleLargeFiles(theApp->glob_prefs->GetCatPath(category))) {
 223                         AddLogLineM(true, _("Filesystem for Incoming directory cannot handle large files."));
 224                         return;
 225                 }
 226         }
 227
 228         CPartFile* newfile = NULL;
 229         try {
 230                 newfile = new CPartFile(toadd);
 231         } catch (const CInvalidPacket& WXUNUSED(e)) {
 232                 AddDebugLogLineM(true, logDownloadQueue, wxT("Search-result contained invalid tags, could not add"));
 233         }
 234
 235         if ( newfile && newfile->GetStatus() != PS_ERROR ) {
 236                 AddDownload( newfile, thePrefs::AddNewFilesPaused(), category );
 237                 // Add any possible sources
 238                 if (toadd->GetClientID() && toadd->GetClientPort()) {
 239                         CMemFile sources(1+4+2);
 240                         sources.WriteUInt8(1);
 241                         sources.WriteUInt32(toadd->GetClientID());
 242                         sources.WriteUInt16(toadd->GetClientPort());
 243                         sources.Reset();
 244                         newfile->AddSources(sources, toadd->GetClientServerIP(), toadd->GetClientServerPort(), SF_SEARCH_RESULT, false);
 245                 }
 246                 for (std::list<CSearchFile::ClientStruct>::const_iterator it = toadd->GetClients().begin(); it != toadd->GetClients().end(); ++it) {
 247                         CMemFile sources(1+4+2);
 248                         sources.WriteUInt8(1);
 249                         sources.WriteUInt32(it->m_ip);
 250                         sources.WriteUInt16(it->m_port);
 251                         sources.Reset();
 252                         newfile->AddSources(sources, it->m_serverIP, it->m_serverPort, SF_SEARCH_RESULT, false);
 253                 }
 254         } else {
 255                 delete newfile;
 256         }
 257 }
 258
 259
 260 struct SFindBestPF
 261 {
 262         void operator()(CPartFile* file) {
 263                 // Check if we should filter out other categories
 264                 if ((m_category != -1) && (file->GetCategory() != m_category)) {
 265                         return;
 266                 } else if (file->GetStatus() != PS_PAUSED) {
 267                         return;
 268                 }
 269
 270                 if (!m_result || (file->GetDownPriority() > m_result->GetDownPriority())) {
 271                         m_result = file;
 272                 }
 273         }
 274
 275         //! The category to look for, or -1 if any category is good
 276         int m_category;
 277         //! If any acceptable files are found, this variable store their pointer
 278         CPartFile* m_result;
 279 };
 280
 281
 282 void CDownloadQueue::StartNextFile(CPartFile* oldfile)
 283 {
 284         if ( thePrefs::StartNextFile() ) {
 285                 SFindBestPF visitor = { -1, NULL };
 286
 287                 {
 288                         wxMutexLocker lock(m_mutex);
 289
 290                         if (thePrefs::StartNextFileSame()) {
 291                                 // Get a download in the same category
 292                                 visitor.m_category = oldfile->GetCategory();
 293
 294                                 visitor = std::for_each(m_filelist.begin(), m_filelist.end(), visitor);
 295                         }
 296
 297                         if (visitor.m_result == NULL) {
 298                                 // Get a download, regardless of category
 299                                 visitor.m_category = -1;
 300
 301                                 visitor = std::for_each(m_filelist.begin(), m_filelist.end(), visitor);
 302                         }
 303                 }
 304
 305                 if (visitor.m_result) {
 306                         visitor.m_result->ResumeFile();
 307                 }
 308         }
 309 }
 310
 311
 312 void CDownloadQueue::AddDownload(CPartFile* file, bool paused, uint8 category)
 313 {
 314         wxCHECK_RET(!IsFileExisting(file->GetFileHash()), wxT("Adding duplicate part-file"));
 315
 316         if (file->GetStatus(true) == PS_ALLOCATING) {
 317                 file->PauseFile();
 318         } else if (paused && GetFileCount()) {
 319                 file->StopFile();
 320         }
 321
 322         {
 323                 wxMutexLocker lock(m_mutex);
 324                 m_filelist.push_back( file );
 325                 DoSortByPriority();
 326         }
 327
 328         NotifyObservers( EventType( EventType::INSERTED, file ) );
 329         if (category < theApp->glob_prefs->GetCatCount()) {
 330                 file->SetCategory(category);
 331         } else {
 332                 AddDebugLogLineM( false, logDownloadQueue, wxT("Tried to add download into invalid category.") );
 333         }
 334         Notify_DownloadCtrlAddFile( file );
 335         theApp->searchlist->UpdateSearchFileByHash(file->GetFileHash());        // Update file in the search dialog if it's still open
 336         AddLogLineM(true, CFormat(_("Downloading %s")) % file->GetFileName() );
 337 }
 338
 339
 340 bool CDownloadQueue::IsFileExisting( const CMD4Hash& fileid ) const
 341 {
 342         if (CKnownFile* file = theApp->sharedfiles->GetFileByID(fileid)) {
 343                 if (file->IsPartFile()) {
 344                         AddLogLineM(true, CFormat( _("You are already trying to download the file '%s'") ) % file->GetFileName());
 345                 } else {
 346                         // Check if the file exists, since otherwise the user is forced to
 347                         // manually reload the shares to download a file again.
 348                         CPath fullpath = file->GetFilePath().JoinPaths(file->GetFileName());
 349                         if (!fullpath.FileExists()) {
 350                                 // The file is no longer available, unshare it
 351                                 theApp->sharedfiles->RemoveFile(file);
 352
 353                                 return false;
 354                         }
 355
 356                         AddLogLineM(true, CFormat( _("You already have the file '%s'") ) % file->GetFileName());
 357                 }
 358
 359                 return true;
 360         } else if ((file = GetFileByID(fileid))) {
 361                 AddLogLineM(true, CFormat( _("You are already trying to download the file %s") ) % file->GetFileName());
 362                 return true;
 363         }
 364
 365         return false;
 366 }
 367
 368
 369 void CDownloadQueue::Process()
 370 {
 371         // send src requests to local server
 372         ProcessLocalRequests();
 373
 374         {
 375                 wxMutexLocker lock(m_mutex);
 376
 377                 uint32 downspeed = 0;
 378                 if (thePrefs::GetMaxDownload() != UNLIMITED && m_datarate > 1500) {
 379                         downspeed = (((uint32)thePrefs::GetMaxDownload())*1024*100)/(m_datarate+1);
 380                         if (downspeed < 50) {
 381                                 downspeed = 50;
 382                         } else if (downspeed > 200) {
 383                                 downspeed = 200;
 384                         }
 385                 }
 386
 387                 m_datarate = 0;
 388                 m_udcounter++;
 389                 uint32 cur_datarate = 0;
 390                 uint32 cur_udcounter = m_udcounter;
 391
 392                 for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
 393                         CPartFile* file = m_filelist[i];
 394
 395                         CMutexUnlocker unlocker(m_mutex);
 396
 397                         if ( file->GetStatus() == PS_READY || file->GetStatus() == PS_EMPTY ){
 398                                 cur_datarate += file->Process( downspeed, cur_udcounter );
 399                         } else {
 400                                 //This will make sure we don't keep old sources to paused and stoped files..
 401                                 file->StopPausedFile();
 402                         }
 403                 }
 404
 405                 m_datarate += cur_datarate;
 406
 407
 408                 if (m_udcounter == 5) {
 409                         if (theApp->serverconnect->IsUDPSocketAvailable()) {
 410                                 if( (::GetTickCount() - m_lastudpstattime) > UDPSERVERSTATTIME) {
 411                                         m_lastudpstattime = ::GetTickCount();
 412
 413                                         CMutexUnlocker unlocker(m_mutex);
 414                                         theApp->serverlist->ServerStats();
 415                                 }
 416                         }
 417                 }
 418
 419                 if (m_udcounter == 10) {
 420                         m_udcounter = 0;
 421                         if (theApp->serverconnect->IsUDPSocketAvailable()) {
 422                                 if ( (::GetTickCount() - m_lastudpsearchtime) > UDPSERVERREASKTIME) {
 423                                         SendNextUDPPacket();
 424                                 }
 425                         }
 426                 }
 427
 428                 if ( (::GetTickCount() - m_lastsorttime) > 10000 ) {
 429
 430
 431                         DoSortByPriority();
 432                 }
 433                 // Check if any paused files can be resumed
 434
 435                 CheckDiskspace(thePrefs::GetTempDir());
 436         }
 437
 438
 439         // Check for new links once per second.
 440         if ((::GetTickCount() - m_nLastED2KLinkCheck) >= 1000) {
 441                 theApp->AddLinksFromFile();
 442                 m_nLastED2KLinkCheck = ::GetTickCount();
 443         }
 444 }
 445
 446
 447 CPartFile* CDownloadQueue::GetFileByID(const CMD4Hash& filehash) const
 448 {
 449         wxMutexLocker lock( m_mutex );
 450
 451         for ( uint16 i = 0; i < m_filelist.size(); ++i ) {
 452                 if ( filehash == m_filelist[i]->GetFileHash()) {
 453                         return m_filelist[ i ];
 454                 }
 455         }
 456
 457         return NULL;
 458 }
 459
 460
 461 CPartFile* CDownloadQueue::GetFileByIndex(unsigned int index)  const
 462 {
 463         wxMutexLocker lock( m_mutex );
 464
 465         if ( index < m_filelist.size() ) {
 466                 return m_filelist[ index ];
 467         }
 468
 469         wxFAIL;
 470         return NULL;
 471 }
 472
 473
 474 bool CDownloadQueue::IsPartFile(const CKnownFile* file) const
 475 {
 476         wxMutexLocker lock(m_mutex);
 477
 478         for (uint16 i = 0; i < m_filelist.size(); ++i) {
 479                 if (file == m_filelist[i]) {
 480                         return true;
 481                 }
 482         }
 483
 484         return false;
 485 }
 486
 487
 488 void CDownloadQueue::OnConnectionState(bool bConnected)
 489 {
 490         wxMutexLocker lock(m_mutex);
 491
 492         for (uint16 i = 0; i < m_filelist.size(); ++i) {
 493                 if (    m_filelist[i]->GetStatus() == PS_READY ||
 494                         m_filelist[i]->GetStatus() == PS_EMPTY) {
 495                         m_filelist[i]->SetActive(bConnected);
 496                 }
 497         }
 498 }
 499
 500
 501 void CDownloadQueue::CheckAndAddSource(CPartFile* sender, CUpDownClient* source)
 502 {
 503         // if we block loopbacks at this point it should prevent us from connecting to ourself
 504         if ( source->HasValidHash() ) {
 505                 if ( source->GetUserHash() == thePrefs::GetUserHash() ) {
 506                         AddDebugLogLineM( false, logDownloadQueue, wxT("Tried to add source with matching hash to your own.") );
 507                         source->Safe_Delete();
 508                         return;
 509                 }
 510         }
 511
 512         if (sender->IsStopped()) {
 513                 source->Safe_Delete();
 514                 return;
 515         }
 516
 517         // Filter sources which are known to be dead/useless
 518         if ( theApp->clientlist->IsDeadSource( source ) || sender->IsDeadSource(source) ) {
 519                 source->Safe_Delete();
 520                 return;
 521         }
 522
 523         // Filter sources which are incompatible with our encryption setting (one requires it, and the other one doesn't supports it)
 524         if ( (source->RequiresCryptLayer() && (!thePrefs::IsClientCryptLayerSupported() || !source->HasValidHash())) || (thePrefs::IsClientCryptLayerRequired() && (!source->SupportsCryptLayer() || !source->HasValidHash()))) {
 525                 source->Safe_Delete();
 526                 return;
 527         }
 528
 529         // Find all clients with the same hash
 530         if ( source->HasValidHash() ) {
 531                 CClientList::SourceList found = theApp->clientlist->GetClientsByHash( source->GetUserHash() );
 532
 533                 CClientList::SourceList::iterator it = found.begin();
 534                 for ( ; it != found.end(); it++ ) {
 535                         CKnownFile* file = (*it)->GetRequestFile();
 536
 537                         // Only check files on the download-queue
 538                         if ( file ) {
 539                                 // Is the found source queued for something else?
 540                                 if (  file != sender ) {
 541                                         // Try to add a request for the other file
 542                                         if ( (*it)->AddRequestForAnotherFile(sender)) {
 543                                                 // Add it to downloadlistctrl
 544                                                 Notify_DownloadCtrlAddSource(sender, *it, A4AF_SOURCE);
 545                                         }
 546                                 }
 547
 548                                 source->Safe_Delete();
 549                                 return;
 550                         }
 551                 }
 552         }
 553
 554
 555
 556         // Our new source is real new but maybe it is already uploading to us?
 557         // If yes the known client will be attached to the var "source" and the old
 558         // source-client will be deleted. However, if the request file of the known
 559         // source is NULL, then we have to treat it almost like a new source and if
 560         // it isn't NULL and not "sender", then we shouldn't move it, but rather add
 561         // a request for the new file.
 562         ESourceFrom nSourceFrom = source->GetSourceFrom();
 563         if ( theApp->clientlist->AttachToAlreadyKnown(&source, 0) ) {
 564                 // Already queued for another file?
 565                 if ( source->GetRequestFile() ) {
 566                         // If we're already queued for the right file, then there's nothing to do
 567                         if ( sender != source->GetRequestFile() ) {
 568                                 // Add the new file to the request list
 569                                 source->AddRequestForAnotherFile( sender );
 570                         }
 571                 } else {
 572                         // Source was known, but reqfile NULL.
 573                         source->SetRequestFile( sender );
 574                         source->SetSourceFrom(nSourceFrom);
 575                         sender->AddSource( source );
 576                         if ( source->GetFileRating() || !source->GetFileComment().IsEmpty() ) {
 577                                 sender->UpdateFileRatingCommentAvail();
 578                         }
 579
 580                         Notify_DownloadCtrlAddSource(sender, source, UNAVAILABLE_SOURCE);
 581                 }
 582         } else {
 583                 // Unknown client, add it to the clients list
 584                 source->SetRequestFile( sender );
 585
 586                 theApp->clientlist->AddClient(source);
 587
 588                 sender->AddSource( source );
 589                 if ( source->GetFileRating() || !source->GetFileComment().IsEmpty() ) {
 590                         sender->UpdateFileRatingCommentAvail();
 591                 }
 592
 593                 Notify_DownloadCtrlAddSource(sender, source, UNAVAILABLE_SOURCE);
 594         }
 595 }
 596
 597
 598 void CDownloadQueue::CheckAndAddKnownSource(CPartFile* sender,CUpDownClient* source)
 599 {
 600         // Kad reviewed
 601
 602         if (sender->IsStopped()) {
 603                 return;
 604         }
 605
 606         // Filter sources which are known to be dead/useless
 607         if ( sender->IsDeadSource(source) ) {
 608                 return;
 609         }
 610
 611         // "Filter LAN IPs" -- this may be needed here in case we are connected to the internet and are also connected
 612         // to a LAN and some client from within the LAN connected to us. Though this situation may be supported in future
 613         // by adding that client to the source list and filtering that client's LAN IP when sending sources to
 614         // a client within the internet.
 615         //
 616         // "IPfilter" is not needed here, because that "known" client was already IPfiltered when receiving OP_HELLO.
 617         if (!source->HasLowID()) {
 618                 uint32 nClientIP = wxUINT32_SWAP_ALWAYS(source->GetUserIDHybrid());
 619                 if (!IsGoodIP(nClientIP, thePrefs::FilterLanIPs())) { // check for 0-IP, localhost and LAN addresses
 620                         AddDebugLogLineM(false, logIPFilter, wxT("Ignored already known source with IP=%s") + Uint32toStringIP(nClientIP));
 621                         return;
 622                 }
 623         }
 624
 625         // Filter sources which are incompatible with our encryption setting (one requires it, and the other one doesn't supports it)
 626         if ( (source->RequiresCryptLayer() && (!thePrefs::IsClientCryptLayerSupported() || !source->HasValidHash())) || (thePrefs::IsClientCryptLayerRequired() && (!source->SupportsCryptLayer() || !source->HasValidHash())))
 627         {
 628                 source->Safe_Delete();
 629                 return;
 630         }
 631
 632         CPartFile* file = source->GetRequestFile();
 633
 634         // Check if the file is already queued for something else
 635         if ( file ) {
 636                 if ( file != sender ) {
 637                         if ( source->AddRequestForAnotherFile( sender ) ) {
 638                                 Notify_DownloadCtrlAddSource( sender, source, A4AF_SOURCE );
 639                         }
 640                 }
 641         } else {
 642                 source->SetRequestFile( sender );
 643
 644                 if ( source->GetFileRating() || !source->GetFileComment().IsEmpty() ) {
 645                         sender->UpdateFileRatingCommentAvail();
 646                 }
 647
 648                 source->SetSourceFrom(SF_PASSIVE);
 649                 sender->AddSource( source );
 650                 Notify_DownloadCtrlAddSource( sender, source, UNAVAILABLE_SOURCE);
 651         }
 652 }
 653
 654
 655 bool CDownloadQueue::RemoveSource(CUpDownClient* toremove, bool WXUNUSED(updatewindow), bool bDoStatsUpdate)
 656 {
 657         bool removed = false;
 658         toremove->DeleteAllFileRequests();
 659
 660         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
 661                 CPartFile* cur_file = GetFileByIndex( i );
 662
 663                 // Remove from source-list
 664                 if ( cur_file->DelSource( toremove ) ) {
 665                         cur_file->RemoveDownloadingSource(toremove);
 666                         removed = true;
 667                         if ( bDoStatsUpdate ) {
 668                                 cur_file->UpdatePartsInfo();
 669                         }
 670                 }
 671
 672                 // Remove from A4AF-list
 673                 cur_file->RemoveA4AFSource( toremove );
 674         }
 675
 676
 677         if ( !toremove->GetFileComment().IsEmpty() || toremove->GetFileRating()>0) {
 678                 toremove->GetRequestFile()->UpdateFileRatingCommentAvail();
 679         }
 680
 681         toremove->SetRequestFile( NULL );
 682         toremove->SetDownloadState(DS_NONE);
 683
 684         // Remove from downloadlist widget
 685         Notify_DownloadCtrlRemoveSource(toremove, (CPartFile*)NULL);
 686         toremove->ResetFileStatusInfo();
 687
 688         return removed;
 689 }
 690
 691
 692 void CDownloadQueue::RemoveFile(CPartFile* file)
 693 {
 694         RemoveLocalServerRequest( file );
 695
 696         NotifyObservers( EventType( EventType::REMOVED, file ) );
 697
 698         wxMutexLocker lock( m_mutex );
 699
 700         EraseValue( m_filelist, file );
 701 }
 702
 703
 704 CUpDownClient* CDownloadQueue::GetDownloadClientByIP_UDP(uint32 dwIP, uint16 nUDPPort) const
 705 {
 706         wxMutexLocker lock( m_mutex );
 707
 708         for ( unsigned int i = 0; i < m_filelist.size(); i++ ) {
 709                 const CPartFile::SourceSet& set = m_filelist[i]->GetSourceList();
 710
 711                 for ( CPartFile::SourceSet::const_iterator it = set.begin(); it != set.end(); it++ ) {
 712                         if ( (*it)->GetIP() == dwIP && (*it)->GetUDPPort() == nUDPPort ) {
 713                                 return *it;
 714                         }
 715                 }
 716         }
 717         return NULL;
 718 }
 719
 720
 721 /**
 722  * Checks if the specified server is the one we are connected to.
 723  */
 724 bool IsConnectedServer(const CServer* server)
 725 {
 726         if (server && theApp->serverconnect->GetCurrentServer()) {
 727                 wxString srvAddr = theApp->serverconnect->GetCurrentServer()->GetAddress();
 728                 uint16 srvPort = theApp->serverconnect->GetCurrentServer()->GetPort();
 729
 730                 return server->GetAddress() == srvAddr && server->GetPort() == srvPort;
 731         }
 732
 733         return false;
 734 }
 735
 736
 737 bool CDownloadQueue::SendNextUDPPacket()
 738 {
 739         if ( m_filelist.empty() || !theApp->serverconnect->IsUDPSocketAvailable() || !theApp->IsConnectedED2K()) {
 740                 return false;
 741         }
 742
 743         // Start monitoring the server and the files list
 744         if ( !m_queueServers.IsActive() ) {
 745                 AddObserver( &m_queueFiles );
 746
 747                 theApp->serverlist->AddObserver( &m_queueServers );
 748         }
 749
 750
 751         bool packetSent = false;
 752         while ( !packetSent ) {
 753                 // Get max files ids per packet for current server
 754                 int filesAllowed = GetMaxFilesPerUDPServerPacket();
 755
 756                 if (filesAllowed < 1 || !m_udpserver || IsConnectedServer(m_udpserver)) {
 757                         // Select the next server to ask, must not be the connected server
 758                         do {
 759                                 m_udpserver = m_queueServers.GetNext();
 760                         } while (IsConnectedServer(m_udpserver));
 761
 762                         m_cRequestsSentToServer = 0;
 763                         filesAllowed = GetMaxFilesPerUDPServerPacket();
 764                 }
 765
 766
 767                 // Check if we have asked all servers, in which case we are done
 768                 if (m_udpserver == NULL) {
 769                         DoStopUDPRequests();
 770
 771                         return false;
 772                 }
 773
 774                 // Memoryfile containing the hash of every file to request
 775                 // 28bytes allocation because 16b + 4b + 8b is the worse case scenario.
 776                 CMemFile hashlist( 28 );
 777
 778                 CPartFile* file = m_queueFiles.GetNext();
 779
 780                 while ( file && filesAllowed ) {
 781                         uint8 status = file->GetStatus();
 782
 783                         if ( ( status == PS_READY || status == PS_EMPTY ) && file->GetSourceCount() < thePrefs::GetMaxSourcePerFileUDP() ) {
 784                                 if (file->IsLargeFile() && !m_udpserver->SupportsLargeFilesUDP()) {
 785                                         AddDebugLogLineM(false, logDownloadQueue, wxT("UDP Request for sources on a large file ignored: server doesn't support it"));
 786                                 } else {
 787                                         ++m_cRequestsSentToServer;
 788                                         hashlist.WriteHash( file->GetFileHash() );
 789                                         // See the notes on TCP packet
 790                                         if ( m_udpserver->GetUDPFlags() & SRV_UDPFLG_EXT_GETSOURCES2 ) {
 791                                                 if (file->IsLargeFile()) {
 792                                                         wxASSERT(m_udpserver->SupportsLargeFilesUDP());
 793                                                         hashlist.WriteUInt32( 0 );
 794                                                         hashlist.WriteUInt64( file->GetFileSize() );
 795                                                 } else {
 796                                                         hashlist.WriteUInt32( file->GetFileSize() );
 797                                                 }
 798                                         }
 799                                         --filesAllowed;
 800                                 }
 801                         }
 802
 803                         // Avoid skipping a file if we can't send any more currently
 804                         if ( filesAllowed ) {
 805                                 file = m_queueFiles.GetNext();
 806                         }
 807                 }
 808
 809                 // See if we have anything to send
 810                 if ( hashlist.GetLength() ) {
 811                         packetSent = SendGlobGetSourcesUDPPacket(hashlist);
 812                 }
 813
 814                 // Check if we've covered every file
 815                 if ( file == NULL ) {
 816                         // Reset the list of asked files so that the loop will start over
 817                         m_queueFiles.Reset();
 818
 819                         // Unset the server so that the next server will be used
 820                         m_udpserver = NULL;
 821                 }
 822         }
 823
 824         return true;
 825 }
 826
 827
 828 void CDownloadQueue::StopUDPRequests()
 829 {
 830         wxMutexLocker lock( m_mutex );
 831
 832         DoStopUDPRequests();
 833 }
 834
 835
 836 void CDownloadQueue::DoStopUDPRequests()
 837 {
 838         // No need to observe when we wont be using the results
 839         theApp->serverlist->RemoveObserver( &m_queueServers );
 840         RemoveObserver( &m_queueFiles );
 841
 842         m_udpserver = 0;
 843         m_lastudpsearchtime = ::GetTickCount();
 844 }
 845
 846
 847 // Comparison function needed by sort. Returns true if file1 preceeds file2
 848 bool ComparePartFiles(const CPartFile* file1, const CPartFile* file2) {
 849         if (file1->GetDownPriority() != file2->GetDownPriority()) {
 850                 // To place high-priority files before low priority files we have to
 851                 // invert this test, since PR_LOW is lower than PR_HIGH, and since
 852                 // placing a PR_LOW file before a PR_HIGH file would mean that
 853                 // the PR_LOW file gets sources before the PR_HIGH file ...
 854                 return (file1->GetDownPriority() > file2->GetDownPriority());
 855         } else {
 856                 int sourcesA = file1->GetSourceCount();
 857                 int sourcesB = file2->GetSourceCount();
 858
 859                 int notSourcesA = file1->GetNotCurrentSourcesCount();
 860                 int notSourcesB = file2->GetNotCurrentSourcesCount();
 861
 862                 int cmp = CmpAny( sourcesA - notSourcesA, sourcesB - notSourcesB );
 863
 864                 if ( cmp == 0 ) {
 865                         cmp = CmpAny( notSourcesA, notSourcesB );
 866                 }
 867
 868                 return cmp < 0;
 869         }
 870 }
 871
 872
 873 void CDownloadQueue::DoSortByPriority()
 874 {
 875         m_lastsorttime = ::GetTickCount();
 876         sort( m_filelist.begin(), m_filelist.end(), ComparePartFiles );
 877 }
 878
 879
 880 void CDownloadQueue::ResetLocalServerRequests()
 881 {
 882         wxMutexLocker lock( m_mutex );
 883
 884         m_dwNextTCPSrcReq = 0;
 885         m_localServerReqQueue.clear();
 886
 887         for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
 888                 m_filelist[i]->SetLocalSrcRequestQueued(false);
 889         }
 890 }
 891
 892
 893 void CDownloadQueue::RemoveLocalServerRequest( CPartFile* file )
 894 {
 895         wxMutexLocker lock( m_mutex );
 896
 897         EraseValue( m_localServerReqQueue, file );
 898
 899         file->SetLocalSrcRequestQueued(false);
 900 }
 901
 902
 903 void CDownloadQueue::ProcessLocalRequests()
 904 {
 905         wxMutexLocker lock( m_mutex );
 906
 907         bool bServerSupportsLargeFiles = theApp->serverconnect
 908                                                                         && theApp->serverconnect->GetCurrentServer()
 909                                                                         && theApp->serverconnect->GetCurrentServer()->SupportsLargeFilesTCP();
 910
 911         if ( (!m_localServerReqQueue.empty()) && (m_dwNextTCPSrcReq < ::GetTickCount()) ) {
 912                 CMemFile dataTcpFrame(22);
 913                 const int iMaxFilesPerTcpFrame = 15;
 914                 int iFiles = 0;
 915                 while (!m_localServerReqQueue.empty() && iFiles < iMaxFilesPerTcpFrame) {
 916                         // find the file with the longest waitingtime
 917                         uint32 dwBestWaitTime = 0xFFFFFFFF;
 918
 919                         std::list<CPartFile*>::iterator posNextRequest = m_localServerReqQueue.end();
 920                         std::list<CPartFile*>::iterator it = m_localServerReqQueue.begin();
 921                         while( it != m_localServerReqQueue.end() ) {
 922                                 CPartFile* cur_file = (*it);
 923                                 if (cur_file->GetStatus() == PS_READY || cur_file->GetStatus() == PS_EMPTY) {
 924                                         uint8 nPriority = cur_file->GetDownPriority();
 925                                         if (nPriority > PR_HIGH) {
 926                                                 wxFAIL;
 927                                                 nPriority = PR_HIGH;
 928                                         }
 929
 930                                         if (cur_file->GetLastSearchTime() + (PR_HIGH-nPriority) < dwBestWaitTime ){
 931                                                 dwBestWaitTime = cur_file->GetLastSearchTime() + (PR_HIGH - nPriority);
 932                                                 posNextRequest = it;
 933                                         }
 934
 935                                         it++;
 936                                 } else {
 937                                         it = m_localServerReqQueue.erase(it);
 938                                         cur_file->SetLocalSrcRequestQueued(false);
 939                                         AddDebugLogLineM( false, logDownloadQueue,
 940                                                 CFormat(wxT("Local server source request for file '%s' not sent because of status '%s'"))
 941                                                         % cur_file->GetFileName() % cur_file->getPartfileStatus());
 942                                 }
 943                         }
 944
 945                         if (posNextRequest != m_localServerReqQueue.end()) {
 946                                 CPartFile* cur_file = (*posNextRequest);
 947                                 cur_file->SetLocalSrcRequestQueued(false);
 948                                 cur_file->SetLastSearchTime(::GetTickCount());
 949                                 m_localServerReqQueue.erase(posNextRequest);
 950                                 iFiles++;
 951
 952                                 if (!bServerSupportsLargeFiles && cur_file->IsLargeFile()) {
 953                                         AddDebugLogLineM(false, logDownloadQueue, wxT("TCP Request for sources on a large file ignored: server doesn't support it"));
 954                                 } else {
 955                                         AddDebugLogLineM(false, logDownloadQueue,
 956                                                 CFormat(wxT("Creating local sources request packet for '%s'")) % cur_file->GetFileName());
 957                                         // create request packet
 958                                         CMemFile data(16 + (cur_file->IsLargeFile() ? 8 : 4));
 959                                         data.WriteHash(cur_file->GetFileHash());
 960                                         // Kry - lugdunum extended protocol on 17.3 to handle filesize properly.
 961                                         // There is no need to check anything, old server ignore the extra 4 bytes.
 962                                         // As of 17.9, servers accept a 0 32-bits size and then a 64bits size
 963                                         if (cur_file->IsLargeFile()) {
 964                                                 wxASSERT(bServerSupportsLargeFiles);
 965                                                 data.WriteUInt32(0);
 966                                                 data.WriteUInt64(cur_file->GetFileSize());
 967                                         } else {
 968                                                 data.WriteUInt32(cur_file->GetFileSize());
 969                                         }
 970                                         uint8 byOpcode = 0;
 971                                         if (thePrefs::IsClientCryptLayerSupported() && theApp->serverconnect->GetCurrentServer() != NULL && theApp->serverconnect->GetCurrentServer()->SupportsGetSourcesObfuscation()) {
 972                                                 byOpcode = OP_GETSOURCES_OBFU;
 973                                         } else {
 974                                                 byOpcode = OP_GETSOURCES;
 975                                         }
 976                                         CPacket packet(data, OP_EDONKEYPROT, byOpcode);
 977                                         dataTcpFrame.Write(packet.GetPacket(), packet.GetRealPacketSize());
 978                                 }
 979                         }
 980                 }
 981
 982                 int iSize = dataTcpFrame.GetLength();
 983                 if (iSize > 0) {
 984                         // create one 'packet' which contains all buffered OP_GETSOURCES ED2K packets to be sent with one TCP frame
 985                         // server credits: (16+4)*regularfiles + (16+4+8)*largefiles +1
 986                         CScopedPtr<CPacket> packet(new CPacket(new byte[iSize], dataTcpFrame.GetLength(), true, false));
 987                         dataTcpFrame.Seek(0, wxFromStart);
 988                         dataTcpFrame.Read(packet->GetPacket(), iSize);
 989                         uint32 size = packet->GetPacketSize();
 990                         theApp->serverconnect->SendPacket(packet.release(), true);      // Deletes `packet'.
 991                         AddDebugLogLineM(false, logDownloadQueue, wxT("Sent local sources request packet."));
 992                         theStats::AddUpOverheadServer(size);
 993                 }
 994
 995                 // next TCP frame with up to 15 source requests is allowed to be sent in..
 996                 m_dwNextTCPSrcReq = ::GetTickCount() + SEC2MS(iMaxFilesPerTcpFrame*(16+4));
 997         }
 998
 999 }
1000
1001
1002 void CDownloadQueue::SendLocalSrcRequest(CPartFile* sender)
1003 {
1004         wxMutexLocker lock( m_mutex );
1005
1006         m_localServerReqQueue.push_back(sender);
1007 }
1008
1009
1010 void CDownloadQueue::ResetCatParts(uint8 cat)
1011 {
1012         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
1013                 CPartFile* file = GetFileByIndex( i );
1014
1015                 if ( file->GetCategory() == cat ) {
1016                         // Reset the category
1017                         file->SetCategory( 0 );
1018                 } else if ( file->GetCategory() > cat ) {
1019                         // Set to the new position of the original category
1020                         file->SetCategory( file->GetCategory() - 1 );
1021                 }
1022         }
1023 }
1024
1025
1026 void CDownloadQueue::SetCatPrio(uint8 cat, uint8 newprio)
1027 {
1028         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
1029                 CPartFile* file = GetFileByIndex( i );
1030
1031                 if ( !cat || file->GetCategory() == cat ) {
1032                         if ( newprio == PR_AUTO ) {
1033                                 file->SetAutoDownPriority(true);
1034                         } else {
1035                                 file->SetAutoDownPriority(false);
1036                                 file->SetDownPriority(newprio);
1037                         }
1038                 }
1039         }
1040 }
1041
1042
1043 void CDownloadQueue::SetCatStatus(uint8 cat, int newstatus)
1044 {
1045         std::list<CPartFile*> files;
1046
1047         {
1048                 wxMutexLocker lock(m_mutex);
1049
1050                 for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
1051                         if ( m_filelist[i]->CheckShowItemInGivenCat(cat) ) {
1052                                 files.push_back( m_filelist[i] );
1053                         }
1054                 }
1055         }
1056
1057         std::list<CPartFile*>::iterator it = files.begin();
1058
1059         for ( ; it != files.end(); it++ ) {
1060                 switch ( newstatus ) {
1061                         case MP_CANCEL: (*it)->Delete();        break;
1062                         case MP_PAUSE:  (*it)->PauseFile();     break;
1063                         case MP_STOP:   (*it)->StopFile();      break;
1064                         case MP_RESUME: (*it)->ResumeFile();    break;
1065                 }
1066         }
1067 }
1068
1069
1070 uint16 CDownloadQueue::GetDownloadingFileCount() const
1071 {
1072         wxMutexLocker lock( m_mutex );
1073
1074         uint16 count = 0;
1075         for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
1076                 uint8 status = m_filelist[i]->GetStatus();
1077                 if ( status == PS_READY || status == PS_EMPTY ) {
1078                         count++;
1079                 }
1080         }
1081
1082         return count;
1083 }
1084
1085
1086 uint16 CDownloadQueue::GetPausedFileCount() const
1087 {
1088         wxMutexLocker lock( m_mutex );
1089
1090         uint16 count = 0;
1091         for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
1092                 if ( m_filelist[i]->GetStatus() == PS_PAUSED ) {
1093                         count++;
1094                 }
1095         }
1096
1097         return count;
1098 }
1099
1100
1101 void CDownloadQueue::CheckDiskspace( const CPath& path )
1102 {
1103         if ( ::GetTickCount() - m_lastDiskCheck < DISKSPACERECHECKTIME ) {
1104                 return;
1105         }
1106
1107         m_lastDiskCheck = ::GetTickCount();
1108
1109         uint64 min = 0;
1110         // Check if the user has set an explicit limit
1111         if ( thePrefs::IsCheckDiskspaceEnabled() ) {
1112                 min = thePrefs::GetMinFreeDiskSpace();
1113         }
1114
1115         // The very least acceptable diskspace is a single PART
1116         if ( min < PARTSIZE ) {
1117                 min = PARTSIZE;
1118         }
1119
1120         uint64 free = CPath::GetFreeSpaceAt(path);
1121         if (free == static_cast<uint64>(wxInvalidOffset)) {
1122                 return;
1123         } else if (free < min) {
1124                 CUserEvents::ProcessEvent(
1125                         CUserEvents::OutOfDiskSpace,
1126                         wxT("Temporary partition"));
1127         }
1128
1129         for (unsigned int i = 0; i < m_filelist.size(); ++i) {
1130                 CPartFile* file = m_filelist[i];
1131
1132                 switch ( file->GetStatus() ) {
1133                         case PS_ERROR:
1134                         case PS_COMPLETING:
1135                         case PS_COMPLETE:
1136                                 continue;
1137                 }
1138
1139                 if ( free >= min && file->GetInsufficient() ) {
1140                         // We'll try to resume files if there is enough free space
1141                         if ( free - file->GetNeededSpace() > min ) {
1142                                 file->ResumeFile();
1143                         }
1144                 } else if ( free < min && !file->IsPaused() ) {
1145                         // No space left, stop the files.
1146                         file->PauseFile( true );
1147                 }
1148         }
1149 }
1150
1151
1152 int CDownloadQueue::GetMaxFilesPerUDPServerPacket() const
1153 {
1154         if ( m_udpserver ) {
1155                 if ( m_udpserver->GetUDPFlags() & SRV_UDPFLG_EXT_GETSOURCES ) {
1156                         // get max. file ids per packet
1157                         if ( m_cRequestsSentToServer < MAX_REQUESTS_PER_SERVER ) {
1158                                 return std::min(
1159                                         MAX_FILES_PER_UDP_PACKET,
1160                                         MAX_REQUESTS_PER_SERVER - m_cRequestsSentToServer
1161                                 );
1162                         }
1163                 } else if ( m_cRequestsSentToServer < MAX_REQUESTS_PER_SERVER ) {
1164                         return 1;
1165                 }
1166         }
1167
1168         return 0;
1169 }
1170
1171
1172 bool CDownloadQueue::SendGlobGetSourcesUDPPacket(CMemFile& data)
1173 {
1174         if (!m_udpserver) {
1175                 return false;
1176         }
1177
1178         CPacket packet(data, OP_EDONKEYPROT, ((m_udpserver->GetUDPFlags() & SRV_UDPFLG_EXT_GETSOURCES2) ? OP_GLOBGETSOURCES2 : OP_GLOBGETSOURCES));
1179
1180         theStats::AddUpOverheadServer(packet.GetPacketSize());
1181         theApp->serverconnect->SendUDPPacket(&packet,m_udpserver,false);
1182
1183         return true;
1184 }
1185
1186
1187 void CDownloadQueue::AddToResolve(const CMD4Hash& fileid, const wxString& pszHostname, uint16 port, const wxString& hash, uint8 cryptoptions)
1188 {
1189         // double checking
1190         if ( !GetFileByID(fileid) ) {
1191                 return;
1192         }
1193
1194         wxMutexLocker lock( m_mutex );
1195
1196         Hostname_Entry entry = { fileid, pszHostname, port, hash, cryptoptions };
1197         m_toresolve.push_front(entry);
1198
1199         // Check if there are other DNS lookups on queue
1200         if (m_toresolve.size() == 1) {
1201                 // Check if it is a simple dot address
1202                 uint32 ip = StringIPtoUint32(pszHostname);
1203
1204                 if (ip) {
1205                         OnHostnameResolved(ip);
1206                 } else {
1207                         CAsyncDNS* dns = new CAsyncDNS(pszHostname, DNS_SOURCE, theApp);
1208
1209                         if ((dns->Create() != wxTHREAD_NO_ERROR) || (dns->Run() != wxTHREAD_NO_ERROR)) {
1210                                 dns->Delete();
1211                                 m_toresolve.pop_front();
1212                         }
1213                 }
1214         }
1215 }
1216
1217
1218 void CDownloadQueue::OnHostnameResolved(uint32 ip)
1219 {
1220         wxMutexLocker lock( m_mutex );
1221
1222         wxASSERT( m_toresolve.size() );
1223
1224         Hostname_Entry resolved = m_toresolve.front();
1225         m_toresolve.pop_front();
1226
1227         if ( ip ) {
1228                 CPartFile* file = GetFileByID( resolved.fileid );
1229                 if ( file ) {
1230                         CMemFile sources(1+4+2);
1231                         sources.WriteUInt8(1); // No. Sources
1232                         sources.WriteUInt32(ip);
1233                         sources.WriteUInt16(resolved.port);
1234                         sources.WriteUInt8(resolved.cryptoptions);
1235                         if (resolved.cryptoptions & 0x80) {
1236                                 wxASSERT(!resolved.hash.IsEmpty());
1237                                 CMD4Hash sourcehash;
1238                                 sourcehash.Decode(resolved.hash);
1239                                 sources.WriteHash(sourcehash);
1240                         }
1241                         sources.Seek(0,wxFromStart);
1242
1243                         file->AddSources(sources, 0, 0, SF_LINK, true);
1244                 }
1245         }
1246
1247         while (m_toresolve.size()) {
1248                 Hostname_Entry entry = m_toresolve.front();
1249
1250                 // Check if it is a simple dot address
1251                 uint32 tmpIP = StringIPtoUint32(entry.strHostname);
1252
1253                 if (tmpIP) {
1254                         OnHostnameResolved(tmpIP);
1255                 } else {
1256                         CAsyncDNS* dns = new CAsyncDNS(entry.strHostname, DNS_SOURCE, theApp);
1257
1258                         if ((dns->Create() != wxTHREAD_NO_ERROR) || (dns->Run() != wxTHREAD_NO_ERROR)) {
1259                                 dns->Delete();
1260                                 m_toresolve.pop_front();
1261                         } else {
1262                                 break;
1263                         }
1264                 }
1265         }
1266 }
1267
1268
1269 bool CDownloadQueue::AddLink( const wxString& link, uint8 category )
1270 {
1271         wxString uri(link);
1272
1273         if (link.compare(0, 7, wxT("magnet:")) == 0) {
1274                 uri = CMagnetED2KConverter(link);
1275                 if (uri.empty()) {
1276                         AddLogLineM(true, CFormat(_("Cannot convert magnet link to eD2k: %s")) % link);
1277                         return false;
1278                 }
1279         }
1280
1281         if (uri.compare(0, 7, wxT("ed2k://")) == 0) {
1282                 return AddED2KLink(uri, category);
1283         } else {
1284                 AddLogLineM(true, CFormat(_("Unknown protocol of link: %s")) % link);
1285                 return false;
1286         }
1287 }
1288
1289
1290 bool CDownloadQueue::AddED2KLink( const wxString& link, uint8 category )
1291 {
1292         wxASSERT( !link.IsEmpty() );
1293         wxString URI = link;
1294
1295         // Need the links to end with /, otherwise CreateLinkFromUrl crashes us.
1296         if ( URI.Last() != wxT('/') ) {
1297                 URI += wxT("/");
1298         }
1299
1300         try {
1301                 CScopedPtr<CED2KLink> uri(CED2KLink::CreateLinkFromUrl(URI));
1302
1303                 return AddED2KLink( uri.get(), category );
1304         } catch ( const wxString& err ) {
1305                 AddLogLineM( true, CFormat( _("Invalid eD2k link! ERROR: %s")) % err);
1306         }
1307
1308         return false;
1309 }
1310
1311
1312 bool CDownloadQueue::AddED2KLink( const CED2KLink* link, uint8 category )
1313 {
1314         switch ( link->GetKind() ) {
1315                 case CED2KLink::kFile:
1316                         return AddED2KLink( dynamic_cast<const CED2KFileLink*>( link ), category );
1317
1318                 case CED2KLink::kServer:
1319                         return AddED2KLink( dynamic_cast<const CED2KServerLink*>( link ) );
1320
1321                 case CED2KLink::kServerList:
1322                         return AddED2KLink( dynamic_cast<const CED2KServerListLink*>( link ) );
1323
1324                 default:
1325                         return false;
1326         }
1327 }
1328
1329
1330
1331 bool CDownloadQueue::AddED2KLink( const CED2KFileLink* link, uint8 category )
1332 {
1333         CPartFile* file = NULL;
1334         if (IsFileExisting(link->GetHashKey())) {
1335                 // Must be a shared file if we are to add hashes or sources
1336                 if ((file = GetFileByID(link->GetHashKey())) == NULL) {
1337                         return false;
1338                 }
1339         } else {
1340                 if (link->GetSize() > OLD_MAX_FILE_SIZE) {
1341                         if (!PlatformSpecific::CanFSHandleLargeFiles(thePrefs::GetTempDir())) {
1342                                 AddLogLineM(true, _("Filesystem for Temp directory cannot handle large files."));
1343                                 return false;
1344                         } else if (!PlatformSpecific::CanFSHandleLargeFiles(theApp->glob_prefs->GetCatPath(category))) {
1345                                 AddLogLineM(true, _("Filesystem for Incoming directory cannot handle large files."));
1346                                 return false;
1347                         }
1348                 }
1349
1350                 file = new CPartFile(link);
1351
1352                 if (file->GetStatus() == PS_ERROR) {
1353                         delete file;
1354                         return false;
1355                 }
1356
1357                 AddDownload(file, thePrefs::AddNewFilesPaused(), category);
1358         }
1359
1360         if (link->HasValidAICHHash()) {
1361                 CAICHHashSet* hashset = file->GetAICHHashset();
1362
1363                 if (!hashset->HasValidMasterHash() || (hashset->GetMasterHash() != link->GetAICHHash())) {
1364                         hashset->SetMasterHash(link->GetAICHHash(), AICH_VERIFIED);
1365                         hashset->FreeHashSet();
1366                 }
1367         }
1368
1369         const CED2KFileLink::CED2KLinkSourceList& list = link->m_sources;
1370         CED2KFileLink::CED2KLinkSourceList::const_iterator it = list.begin();
1371         for (; it != list.end(); ++it) {
1372                 AddToResolve(link->GetHashKey(), it->addr, it->port, it->hash, it->cryptoptions);
1373         }
1374
1375         return true;
1376 }
1377
1378
1379 bool CDownloadQueue::AddED2KLink( const CED2KServerLink* link )
1380 {
1381         CServer *server = new CServer( link->GetPort(), Uint32toStringIP( link->GetIP() ) );
1382
1383         server->SetListName( Uint32toStringIP( link->GetIP() ) );
1384
1385         theApp->serverlist->AddServer(server);
1386
1387         Notify_ServerAdd(server);
1388
1389         return true;
1390 }
1391
1392
1393 bool CDownloadQueue::AddED2KLink( const CED2KServerListLink* link )
1394 {
1395         theApp->serverlist->UpdateServerMetFromURL( link->GetAddress() );
1396
1397         return true;
1398 }
1399
1400
1401 void CDownloadQueue::ObserverAdded( ObserverType* o )
1402 {
1403         CObservableQueue<CPartFile*>::ObserverAdded( o );
1404
1405         EventType::ValueList list;
1406
1407         {
1408                 wxMutexLocker lock(m_mutex);
1409                 list.reserve( m_filelist.size() );
1410                 list.insert( list.begin(), m_filelist.begin(), m_filelist.end() );
1411         }
1412
1413         NotifyObservers( EventType( EventType::INITIAL, &list ), o );
1414 }
1415
1416 void CDownloadQueue::KademliaSearchFile(uint32_t searchID, const Kademlia::CUInt128* pcontactID, const Kademlia::CUInt128* pbuddyID, uint8_t type, uint32_t ip, uint16_t tcp, uint16_t udp, uint32_t buddyip, uint16_t buddyport, uint8_t byCryptOptions)
1417 {
1418         AddDebugLogLineM(false, logKadSearch, wxString::Format(wxT("Search result sources (type %i)"),type));
1419
1420         //Safety measure to make sure we are looking for these sources
1421         CPartFile* temp = GetFileByKadFileSearchID(searchID);
1422         if( !temp ) {
1423                 AddDebugLogLineM(false, logKadSearch, wxT("This is not the file we're looking for..."));
1424                 return;
1425         }
1426
1427         //Do we need more sources?
1428         if(!(!temp->IsStopped() && thePrefs::GetMaxSourcePerFile() > temp->GetSourceCount())) {
1429                 AddDebugLogLineM(false, logKadSearch, wxT("No more sources needed for this file"));
1430                 return;
1431         }
1432
1433         uint32_t ED2KID = wxUINT32_SWAP_ALWAYS(ip);
1434
1435         if (theApp->ipfilter->IsFiltered(ED2KID)) {
1436                 AddDebugLogLineM(false, logKadSearch, wxT("Source ip got filtered"));
1437                 AddDebugLogLineM(false, logIPFilter, CFormat(wxT("IPfiltered source IP=%s received from Kademlia")) % Uint32toStringIP(ED2KID));
1438                 return;
1439         }
1440
1441         if( (ip == Kademlia::CKademlia::GetIPAddress() || ED2KID == theApp->GetED2KID()) && tcp == thePrefs::GetPort()) {
1442                 AddDebugLogLineM(false, logKadSearch, wxT("Trying to add myself as source, ignore"));
1443                 return;
1444         }
1445
1446         CUpDownClient* ctemp = NULL;
1447         switch (type) {
1448                 case 4:
1449                 case 1: {
1450                         // NonFirewalled users
1451                         if(!tcp) {
1452                                 AddDebugLogLineM(false, logKadSearch, CFormat(wxT("Ignored source (IP=%s) received from Kademlia, no tcp port received")) % Uint32toStringIP(ip));
1453                                 return;
1454                         }
1455                         if (!IsGoodIP(ED2KID,thePrefs::FilterLanIPs())) {
1456                                 AddDebugLogLineM(false, logKadSearch, CFormat(wxT("%s got filtered")) % Uint32toStringIP(ED2KID));
1457                                 AddDebugLogLineM(false, logIPFilter, CFormat(wxT("Ignored source (IP=%s) received from Kademlia, filtered")) % Uint32toStringIP(ED2KID));
1458                                 return;
1459                         }
1460                         ctemp = new CUpDownClient(tcp, ip, 0, 0, temp, false, true);
1461                         ctemp->SetSourceFrom(SF_KADEMLIA);
1462                         // not actually sent or needed for HighID sources
1463                         //ctemp->SetServerIP(serverip);
1464                         //ctemp->SetServerPort(serverport);
1465                         ctemp->SetKadPort(udp);
1466                         byte cID[16];
1467                         pcontactID->ToByteArray(cID);
1468                         ctemp->SetUserHash(CMD4Hash(cID));
1469                         break;
1470                 }
1471                 case 2: {
1472                         // Don't use this type... Some clients will process it wrong..
1473                         break;
1474                 }
1475                 case 5:
1476                 case 3: {
1477                         // This will be a firewalled client connected to Kad only.
1478                         // We set the clientID to 1 as a Kad user only has 1 buddy.
1479                         ctemp = new CUpDownClient(tcp, 1, 0, 0, temp, false, true);
1480                         // The only reason we set the real IP is for when we get a callback
1481                         // from this firewalled source, the compare method will match them.
1482                         ctemp->SetSourceFrom(SF_KADEMLIA);
1483                         ctemp->SetKadPort(udp);
1484                         byte cID[16];
1485                         pcontactID->ToByteArray(cID);
1486                         ctemp->SetUserHash(CMD4Hash(cID));
1487                         pbuddyID->ToByteArray(cID);
1488                         ctemp->SetBuddyID(cID);
1489                         ctemp->SetBuddyIP(buddyip);
1490                         ctemp->SetBuddyPort(buddyport);
1491                         break;
1492                 }
1493                 case 6: {
1494                         // firewalled source which supports direct UDP callback
1495                         // if we are firewalled ourself, the source is useless to us
1496                         if (theApp->IsFirewalled()) {
1497                                 break;
1498                         }
1499
1500                         if ((byCryptOptions & 0x08) == 0){
1501                                 AddDebugLogLineM(false, logKadSearch, CFormat(wxT("Received Kad source type 6 (direct callback) which has the direct callback flag not set (%s)")) % Uint32toStringIP(ED2KID));
1502                                 break;
1503                         }
1504
1505                         ctemp = new CUpDownClient(tcp, 1, 0, 0, temp, false, true);
1506                         ctemp->SetSourceFrom(SF_KADEMLIA);
1507                         ctemp->SetKadPort(udp);
1508                         ctemp->SetIP(ED2KID); // need to set the IP address, which cannot be used for TCP but for UDP
1509                         byte cID[16];
1510                         pcontactID->ToByteArray(cID);
1511                         ctemp->SetUserHash(CMD4Hash(cID));
1512                 }
1513         }
1514
1515         if (ctemp) {
1516                 // add encryption settings
1517                 ctemp->SetConnectOptions(byCryptOptions);
1518
1519                 AddDebugLogLineM(false, logKadSearch, CFormat(wxT("Happily adding a source (%s) type %d")) % Uint32_16toStringIP_Port(ED2KID, ctemp->GetUserPort()) % type);
1520                 CheckAndAddSource(temp, ctemp);
1521         }
1522 }
1523
1524 CPartFile* CDownloadQueue::GetFileByKadFileSearchID(uint32 id) const
1525 {
1526         wxMutexLocker lock( m_mutex );
1527
1528         for ( uint16 i = 0; i < m_filelist.size(); ++i ) {
1529                 if ( id == m_filelist[i]->GetKadFileSearchID()) {
1530                         return m_filelist[ i ];
1531                 }
1532         }
1533
1534         return NULL;
1535 }
1536
1537 bool CDownloadQueue::DoKademliaFileRequest()
1538 {
1539         return ((::GetTickCount() - lastkademliafilerequest) > KADEMLIAASKTIME);
1540 }
1541 // File_checked_for_headers