src/DownloadQueue.cpp

   1 //
   2 // This file is part of the aMule Project.
   3 //
   4 // Copyright (c) 2003-2008 aMule Team ( admin@amule.org / http://www.amule.org )
   5 // Copyright (c) 2002-2008 Merkur ( devs@emule-project.net / http://www.emule-project.net )
   6 //
   7 // Any parts of this program derived from the xMule, lMule or eMule project,
   8 // or contributed by third-party developers are copyrighted by their
   9 // respective authors.
  10 //
  11 // This program is free software; you can redistribute it and/or modify
  12 // it under the terms of the GNU General Public License as published by
  13 // the Free Software Foundation; either version 2 of the License, or
  14 // (at your option) any later version.
  15 //
  16 // This program is distributed in the hope that it will be useful,
  17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 // GNU General Public License for more details.
  20 //
  21 // You should have received a copy of the GNU General Public License
  22 // along with this program; if not, write to the Free Software
  23 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
  24 //
  25
  26 #include "DownloadQueue.h"      // Interface declarations
  27
  28 #include <protocol/Protocols.h>
  29 #include <protocol/kad/Constants.h>
  30 #include <common/Macros.h>
  31 #include <common/MenuIDs.h>
  32 #include <common/Constants.h>
  33
  34 #include <wx/textfile.h>        // Needed for wxTextFile
  35 #include <wx/utils.h>
  36
  37 #include "Server.h"             // Needed for CServer
  38 #include "Packet.h"             // Needed for CPacket
  39 #include "MemFile.h"            // Needed for CMemFile
  40 #include "ClientList.h"         // Needed for CClientList
  41 #include "updownclient.h"       // Needed for CUpDownClient
  42 #include "ServerList.h"         // Needed for CServerList
  43 #include "ServerConnect.h"      // Needed for CServerConnect
  44 #include "ED2KLink.h"           // Needed for CED2KFileLink
  45 #include "SearchList.h"         // Needed for CSearchFile
  46 #include "SharedFileList.h"     // Needed for CSharedFileList
  47 #include "PartFile.h"           // Needed for CPartFile
  48 #include "Preferences.h"        // Needed for thePrefs
  49 #include "amule.h"              // Needed for theApp
  50 #include "AsyncDNS.h"           // Needed for CAsyncDNS
  51 #include "Statistics.h"         // Needed for theStats
  52 #include "Logger.h"
  53 #include <common/Format.h>      // Needed for CFormat
  54 #include "IPFilter.h"
  55 #include <common/FileFunctions.h>       // Needed for CDirIterator
  56 #include "FileLock.h"           // Needed for CFileLock
  57 #include "GuiEvents.h"          // Needed for Notify_*
  58 #include "UserEvents.h"
  59 #include "MagnetURI.h"          // Needed for CMagnetED2KConverter
  60 #include "ScopedPtr.h"          // Needed for CScopedPtr
  61 #include "PlatformSpecific.h"   // Needed for CanFSHandleLargeFiles
  62
  63 #include "kademlia/kademlia/Kademlia.h"
  64
  65 #include <string>                       // Do_not_auto_remove (mingw-gcc-3.4.5)
  66
  67
  68 // Max. file IDs per UDP packet
  69 // ----------------------------
  70 // 576 - 30 bytes of header (28 for UDP, 2 for "E3 9A" edonkey proto) = 546 bytes
  71 // 546 / 16 = 34
  72
  73
  74 #define MAX_FILES_PER_UDP_PACKET        31      // 2+16*31 = 498 ... is still less than 512 bytes!!
  75 #define MAX_REQUESTS_PER_SERVER         35
  76
  77
  78 CDownloadQueue::CDownloadQueue()
  79 // Needs to be recursive that that is can own an observer assigned to itself
  80         : m_mutex( wxMUTEX_RECURSIVE )
  81 {
  82         m_datarate = 0;
  83         m_udpserver = 0;
  84         m_lastsorttime = 0;
  85         m_lastudpsearchtime = 0;
  86         m_lastudpstattime = 0;
  87         m_udcounter = 0;
  88         m_nLastED2KLinkCheck = 0;
  89         m_dwNextTCPSrcReq = 0;
  90         m_cRequestsSentToServer = 0;
  91         m_lastDiskCheck = 0;
  92         SetLastKademliaFileRequest();
  93 }
  94
  95
  96 CDownloadQueue::~CDownloadQueue()
  97 {
  98         if ( !m_filelist.empty() ) {
  99                 for ( unsigned int i = 0; i < m_filelist.size(); i++ ) {
 100                         AddLogLineNS(CFormat(_("Saving PartFile %u of %u")) % (i + 1) % m_filelist.size());
 101                         delete m_filelist[i];
 102                 }
 103                 AddLogLineNS(_("All PartFiles Saved."));
 104         }
 105 }
 106
 107
 108 void CDownloadQueue::LoadMetFiles(const CPath& path)
 109 {
 110         AddLogLineNS(CFormat(_("Loading temp files from %s.")) % path.GetPrintable());
 111
 112         std::vector<CPath> files;
 113
 114         // Locate part-files to be loaded
 115         CDirIterator TempDir(path);
 116         CPath fileName = TempDir.GetFirstFile(CDirIterator::File, wxT("*.part.met"));
 117         while (fileName.IsOk()) {
 118                 files.push_back(path.JoinPaths(fileName));
 119
 120                 fileName = TempDir.GetNextFile();
 121         }
 122
 123         // Loading in order makes it easier to figure which
 124         // file is broken in case of crashes, or the like.
 125         std::sort(files.begin(), files.end());
 126
 127         // Load part-files
 128         for ( size_t i = 0; i < files.size(); i++ ) {
 129                 AddLogLineNS(CFormat(_("Loading PartFile %u of %u")) % (i + 1) % files.size());
 130                 fileName = files[i].GetFullName();
 131                 CPartFile *toadd = new CPartFile();
 132                 bool result = toadd->LoadPartFile(path, fileName) != 0;
 133                 if (!result) {
 134                         // Try from backup
 135                         result = toadd->LoadPartFile(path, fileName, true) != 0;
 136                 }
 137                 if (result && !IsFileExisting(toadd->GetFileHash())) {
 138                         {
 139                                 wxMutexLocker lock(m_mutex);
 140                                 m_filelist.push_back(toadd);
 141                         }
 142                         NotifyObservers(EventType(EventType::INSERTED, toadd));
 143                         Notify_DownloadCtrlAddFile(toadd);
 144                 } else {
 145                         wxString msg;
 146                         if (result) {
 147                                 msg << CFormat(wxT("WARNING: Duplicate partfile with hash '%s' found, skipping: %s"))
 148                                         % toadd->GetFileHash().Encode() % fileName;
 149                         } else {
 150                                 // If result is false, then reading of both the primary and the backup .met failed
 151                                 AddLogLineM(false,
 152                                         _("ERROR: Failed to load backup file. Search http://forum.amule.org for .part.met recovery solutions."));
 153                                 msg << CFormat(wxT("ERROR: Failed to load PartFile '%s'")) % fileName;
 154                         }
 155                         AddLogLineCS(msg);
 156
 157                         // Delete the partfile object in the end.
 158                         delete toadd;
 159                 }
 160         }
 161         AddLogLineNS(_("All PartFiles Loaded."));
 162
 163         if ( GetFileCount() == 0 ) {
 164                 AddLogLineM(false, _("No part files found"));
 165         } else {
 166                 AddLogLineM(false, wxString::Format(wxPLURAL("Found %u part file", "Found %u part files", GetFileCount()), GetFileCount()) );
 167
 168                 DoSortByPriority();
 169                 CheckDiskspace( path );
 170         }
 171 }
 172
 173
 174 uint16 CDownloadQueue::GetFileCount() const
 175 {
 176         wxMutexLocker lock( m_mutex );
 177
 178         return m_filelist.size();
 179 }
 180
 181
 182 CServer* CDownloadQueue::GetUDPServer() const
 183 {
 184         wxMutexLocker lock( m_mutex );
 185
 186         return m_udpserver;
 187 }
 188
 189
 190 void CDownloadQueue::SetUDPServer( CServer* server )
 191 {
 192         wxMutexLocker lock( m_mutex );
 193
 194         m_udpserver = server;
 195 }
 196
 197
 198 void CDownloadQueue::SaveSourceSeeds()
 199 {
 200         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
 201                 GetFileByIndex( i )->SaveSourceSeeds();
 202         }
 203 }
 204
 205
 206 void CDownloadQueue::LoadSourceSeeds()
 207 {
 208         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
 209                 GetFileByIndex( i )->LoadSourceSeeds();
 210         }
 211 }
 212
 213
 214 void CDownloadQueue::AddSearchToDownload(CSearchFile* toadd, uint8 category)
 215 {
 216         if ( IsFileExisting(toadd->GetFileHash()) ) {
 217                 return;
 218         }
 219
 220         if (toadd->GetFileSize() > OLD_MAX_FILE_SIZE) {
 221                 if (!PlatformSpecific::CanFSHandleLargeFiles(thePrefs::GetTempDir())) {
 222                         AddLogLineM(true, _("Filesystem for Temp directory cannot handle large files."));
 223                         return;
 224                 } else if (!PlatformSpecific::CanFSHandleLargeFiles(theApp->glob_prefs->GetCatPath(category))) {
 225                         AddLogLineM(true, _("Filesystem for Incoming directory cannot handle large files."));
 226                         return;
 227                 }
 228         }
 229
 230         CPartFile* newfile = NULL;
 231         try {
 232                 newfile = new CPartFile(toadd);
 233         } catch (const CInvalidPacket& WXUNUSED(e)) {
 234                 AddDebugLogLineM(true, logDownloadQueue, wxT("Search-result contained invalid tags, could not add"));
 235         }
 236
 237         if ( newfile && newfile->GetStatus() != PS_ERROR ) {
 238                 AddDownload( newfile, thePrefs::AddNewFilesPaused(), category );
 239                 // Add any possible sources
 240                 if (toadd->GetClientID() && toadd->GetClientPort()) {
 241                         CMemFile sources(1+4+2);
 242                         sources.WriteUInt8(1);
 243                         sources.WriteUInt32(toadd->GetClientID());
 244                         sources.WriteUInt16(toadd->GetClientPort());
 245                         sources.Reset();
 246                         newfile->AddSources(sources, toadd->GetClientServerIP(), toadd->GetClientServerPort(), SF_SEARCH_RESULT, false);
 247                 }
 248                 for (std::list<CSearchFile::ClientStruct>::const_iterator it = toadd->GetClients().begin(); it != toadd->GetClients().end(); ++it) {
 249                         CMemFile sources(1+4+2);
 250                         sources.WriteUInt8(1);
 251                         sources.WriteUInt32(it->m_ip);
 252                         sources.WriteUInt16(it->m_port);
 253                         sources.Reset();
 254                         newfile->AddSources(sources, it->m_serverIP, it->m_serverPort, SF_SEARCH_RESULT, false);
 255                 }
 256         } else {
 257                 delete newfile;
 258         }
 259 }
 260
 261
 262 struct SFindBestPF
 263 {
 264         void operator()(CPartFile* file) {
 265                 // Check if we should filter out other categories
 266                 if ((m_category != -1) && (file->GetCategory() != m_category)) {
 267                         return;
 268                 } else if (file->GetStatus() != PS_PAUSED) {
 269                         return;
 270                 }
 271
 272                 if (!m_result || (file->GetDownPriority() > m_result->GetDownPriority())) {
 273                         m_result = file;
 274                 }
 275         }
 276
 277         //! The category to look for, or -1 if any category is good
 278         int m_category;
 279         //! If any acceptable files are found, this variable store their pointer
 280         CPartFile* m_result;
 281 };
 282
 283
 284 void CDownloadQueue::StartNextFile(CPartFile* oldfile)
 285 {
 286         if ( thePrefs::StartNextFile() ) {
 287                 SFindBestPF visitor = { -1, NULL };
 288
 289                 {
 290                         wxMutexLocker lock(m_mutex);
 291
 292                         if (thePrefs::StartNextFileSame()) {
 293                                 // Get a download in the same category
 294                                 visitor.m_category = oldfile->GetCategory();
 295
 296                                 visitor = std::for_each(m_filelist.begin(), m_filelist.end(), visitor);
 297                         }
 298
 299                         if (visitor.m_result == NULL) {
 300                                 // Get a download, regardless of category
 301                                 visitor.m_category = -1;
 302
 303                                 visitor = std::for_each(m_filelist.begin(), m_filelist.end(), visitor);
 304                         }
 305                 }
 306
 307                 if (visitor.m_result) {
 308                         visitor.m_result->ResumeFile();
 309                 }
 310         }
 311 }
 312
 313
 314 void CDownloadQueue::AddDownload(CPartFile* file, bool paused, uint8 category)
 315 {
 316         wxCHECK_RET(!IsFileExisting(file->GetFileHash()), wxT("Adding duplicate part-file"));
 317
 318         if (file->GetStatus(true) == PS_ALLOCATING) {
 319                 file->PauseFile();
 320         } else if (paused && GetFileCount()) {
 321                 file->StopFile();
 322         }
 323
 324         {
 325                 wxMutexLocker lock(m_mutex);
 326                 m_filelist.push_back( file );
 327                 DoSortByPriority();
 328         }
 329
 330         NotifyObservers( EventType( EventType::INSERTED, file ) );
 331         if (category < theApp->glob_prefs->GetCatCount()) {
 332                 file->SetCategory(category);
 333         } else {
 334                 AddDebugLogLineM( false, logDownloadQueue, wxT("Tried to add download into invalid category.") );
 335         }
 336         Notify_DownloadCtrlAddFile( file );
 337         AddLogLineM(true, CFormat(_("Downloading %s")) % file->GetFileName() );
 338 }
 339
 340
 341 bool CDownloadQueue::IsFileExisting( const CMD4Hash& fileid ) const
 342 {
 343         if (CKnownFile* file = theApp->sharedfiles->GetFileByID(fileid)) {
 344                 if (file->IsPartFile()) {
 345                         AddLogLineM(true, CFormat( _("You are already trying to download the file '%s'") ) % file->GetFileName());
 346                 } else {
 347                         // Check if the file exists, since otherwise the user is forced to
 348                         // manually reload the shares to download a file again.
 349                         CPath fullpath = file->GetFilePath().JoinPaths(file->GetFileName());
 350                         if (!fullpath.FileExists()) {
 351                                 // The file is no longer available, unshare it
 352                                 theApp->sharedfiles->RemoveFile(file);
 353
 354                                 return false;
 355                         }
 356
 357                         AddLogLineM(true, CFormat( _("You already have the file '%s'") ) % file->GetFileName());
 358                 }
 359
 360                 return true;
 361         } else if ((file = GetFileByID(fileid))) {
 362                 AddLogLineM(true, CFormat( _("You are already trying to download the file %s") ) % file->GetFileName());
 363                 return true;
 364         }
 365
 366         return false;
 367 }
 368
 369
 370 void CDownloadQueue::Process()
 371 {
 372         // send src requests to local server
 373         ProcessLocalRequests();
 374
 375         {
 376                 wxMutexLocker lock(m_mutex);
 377
 378                 uint32 downspeed = 0;
 379                 if (thePrefs::GetMaxDownload() != UNLIMITED && m_datarate > 1500) {
 380                         downspeed = (((uint32)thePrefs::GetMaxDownload())*1024*100)/(m_datarate+1);
 381                         if (downspeed < 50) {
 382                                 downspeed = 50;
 383                         } else if (downspeed > 200) {
 384                                 downspeed = 200;
 385                         }
 386                 }
 387
 388                 m_datarate = 0;
 389                 m_udcounter++;
 390                 uint32 cur_datarate = 0;
 391                 uint32 cur_udcounter = m_udcounter;
 392
 393                 for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
 394                         CPartFile* file = m_filelist[i];
 395
 396                         CMutexUnlocker unlocker(m_mutex);
 397
 398                         if ( file->GetStatus() == PS_READY || file->GetStatus() == PS_EMPTY ){
 399                                 cur_datarate += file->Process( downspeed, cur_udcounter );
 400                         } else {
 401                                 //This will make sure we don't keep old sources to paused and stoped files..
 402                                 file->StopPausedFile();
 403                         }
 404                 }
 405
 406                 m_datarate += cur_datarate;
 407
 408
 409                 if (m_udcounter == 5) {
 410                         if (theApp->serverconnect->IsUDPSocketAvailable()) {
 411                                 if( (::GetTickCount() - m_lastudpstattime) > UDPSERVERSTATTIME) {
 412                                         m_lastudpstattime = ::GetTickCount();
 413
 414                                         CMutexUnlocker unlocker(m_mutex);
 415                                         theApp->serverlist->ServerStats();
 416                                 }
 417                         }
 418                 }
 419
 420                 if (m_udcounter == 10) {
 421                         m_udcounter = 0;
 422                         if (theApp->serverconnect->IsUDPSocketAvailable()) {
 423                                 if ( (::GetTickCount() - m_lastudpsearchtime) > UDPSERVERREASKTIME) {
 424                                         SendNextUDPPacket();
 425                                 }
 426                         }
 427                 }
 428
 429                 if ( (::GetTickCount() - m_lastsorttime) > 10000 ) {
 430
 431
 432                         DoSortByPriority();
 433                 }
 434                 // Check if any paused files can be resumed
 435
 436                 CheckDiskspace(thePrefs::GetTempDir());
 437         }
 438
 439
 440         // Check for new links once per second.
 441         if ((::GetTickCount() - m_nLastED2KLinkCheck) >= 1000) {
 442                 AddLinksFromFile();
 443                 m_nLastED2KLinkCheck = ::GetTickCount();
 444         }
 445 }
 446
 447
 448 CPartFile* CDownloadQueue::GetFileByID(const CMD4Hash& filehash) const
 449 {
 450         wxMutexLocker lock( m_mutex );
 451
 452         for ( uint16 i = 0; i < m_filelist.size(); ++i ) {
 453                 if ( filehash == m_filelist[i]->GetFileHash()) {
 454                         return m_filelist[ i ];
 455                 }
 456         }
 457
 458         return NULL;
 459 }
 460
 461
 462 CPartFile* CDownloadQueue::GetFileByIndex(unsigned int index)  const
 463 {
 464         wxMutexLocker lock( m_mutex );
 465
 466         if ( index < m_filelist.size() ) {
 467                 return m_filelist[ index ];
 468         }
 469
 470         wxFAIL;
 471         return NULL;
 472 }
 473
 474
 475 bool CDownloadQueue::IsPartFile(const CKnownFile* file) const
 476 {
 477         wxMutexLocker lock(m_mutex);
 478
 479         for (uint16 i = 0; i < m_filelist.size(); ++i) {
 480                 if (file == m_filelist[i]) {
 481                         return true;
 482                 }
 483         }
 484
 485         return false;
 486 }
 487
 488
 489 void CDownloadQueue::OnConnectionState(bool bConnected)
 490 {
 491         wxMutexLocker lock(m_mutex);
 492
 493         for (uint16 i = 0; i < m_filelist.size(); ++i) {
 494                 if (    m_filelist[i]->GetStatus() == PS_READY ||
 495                         m_filelist[i]->GetStatus() == PS_EMPTY) {
 496                         m_filelist[i]->SetActive(bConnected);
 497                 }
 498         }
 499 }
 500
 501
 502 void CDownloadQueue::CheckAndAddSource(CPartFile* sender, CUpDownClient* source)
 503 {
 504         // if we block loopbacks at this point it should prevent us from connecting to ourself
 505         if ( source->HasValidHash() ) {
 506                 if ( source->GetUserHash() == thePrefs::GetUserHash() ) {
 507                         AddDebugLogLineM( false, logDownloadQueue, wxT("Tried to add source with matching hash to your own.") );
 508                         source->Safe_Delete();
 509                         return;
 510                 }
 511         }
 512
 513         if (sender->IsStopped()) {
 514                 source->Safe_Delete();
 515                 return;
 516         }
 517
 518         // Filter sources which are known to be dead/useless
 519         if ( theApp->clientlist->IsDeadSource( source ) || sender->IsDeadSource(source) ) {
 520                 source->Safe_Delete();
 521                 return;
 522         }
 523
 524         // Filter sources which are incompatible with our encryption setting (one requires it, and the other one doesn't supports it)
 525         if ( (source->RequiresCryptLayer() && (!thePrefs::IsClientCryptLayerSupported() || !source->HasValidHash())) || (thePrefs::IsClientCryptLayerRequired() && (!source->SupportsCryptLayer() || !source->HasValidHash()))) {
 526                 source->Safe_Delete();
 527                 return;
 528         }
 529
 530         // Find all clients with the same hash
 531         if ( source->HasValidHash() ) {
 532                 CClientList::SourceList found = theApp->clientlist->GetClientsByHash( source->GetUserHash() );
 533
 534                 CClientList::SourceList::iterator it = found.begin();
 535                 for ( ; it != found.end(); it++ ) {
 536                         CKnownFile* file = (*it)->GetRequestFile();
 537
 538                         // Only check files on the download-queue
 539                         if ( file ) {
 540                                 // Is the found source queued for something else?
 541                                 if (  file != sender ) {
 542                                         // Try to add a request for the other file
 543                                         if ( (*it)->AddRequestForAnotherFile(sender)) {
 544                                                 // Add it to downloadlistctrl
 545                                                 Notify_DownloadCtrlAddSource(sender, *it, A4AF_SOURCE);
 546                                         }
 547                                 }
 548
 549                                 source->Safe_Delete();
 550                                 return;
 551                         }
 552                 }
 553         }
 554
 555
 556
 557         // Our new source is real new but maybe it is already uploading to us?
 558         // If yes the known client will be attached to the var "source" and the old
 559         // source-client will be deleted. However, if the request file of the known
 560         // source is NULL, then we have to treat it almost like a new source and if
 561         // it isn't NULL and not "sender", then we shouldn't move it, but rather add
 562         // a request for the new file.
 563         ESourceFrom nSourceFrom = source->GetSourceFrom();
 564         if ( theApp->clientlist->AttachToAlreadyKnown(&source, 0) ) {
 565                 // Already queued for another file?
 566                 if ( source->GetRequestFile() ) {
 567                         // If we're already queued for the right file, then there's nothing to do
 568                         if ( sender != source->GetRequestFile() ) {
 569                                 // Add the new file to the request list
 570                                 source->AddRequestForAnotherFile( sender );
 571                         }
 572                 } else {
 573                         // Source was known, but reqfile NULL.
 574                         source->SetRequestFile( sender );
 575                         if (source->GetSourceFrom() != nSourceFrom) {
 576                                 if (source->GetSourceFrom() != SF_NONE) {
 577                                         theStats::RemoveSourceOrigin(source->GetSourceFrom());
 578                                         theStats::RemoveFoundSource();
 579                                 }
 580                                 source->SetSourceFrom(nSourceFrom);
 581                         }
 582                         sender->AddSource( source );
 583                         if ( source->GetFileRating() || !source->GetFileComment().IsEmpty() ) {
 584                                 sender->UpdateFileRatingCommentAvail();
 585                         }
 586
 587                         Notify_DownloadCtrlAddSource(sender, source, UNAVAILABLE_SOURCE);
 588                 }
 589         } else {
 590                 // Unknown client, add it to the clients list
 591                 source->SetRequestFile( sender );
 592
 593                 theApp->clientlist->AddClient(source);
 594
 595                 sender->AddSource( source );
 596                 if ( source->GetFileRating() || !source->GetFileComment().IsEmpty() ) {
 597                         sender->UpdateFileRatingCommentAvail();
 598                 }
 599
 600                 Notify_DownloadCtrlAddSource(sender, source, UNAVAILABLE_SOURCE);
 601         }
 602 }
 603
 604
 605 void CDownloadQueue::CheckAndAddKnownSource(CPartFile* sender,CUpDownClient* source)
 606 {
 607         // Kad reviewed
 608
 609         if (sender->IsStopped()) {
 610                 return;
 611         }
 612
 613         // Filter sources which are known to be dead/useless
 614         if ( sender->IsDeadSource(source) ) {
 615                 return;
 616         }
 617
 618         // "Filter LAN IPs" -- this may be needed here in case we are connected to the internet and are also connected
 619         // to a LAN and some client from within the LAN connected to us. Though this situation may be supported in future
 620         // by adding that client to the source list and filtering that client's LAN IP when sending sources to
 621         // a client within the internet.
 622         //
 623         // "IPfilter" is not needed here, because that "known" client was already IPfiltered when receiving OP_HELLO.
 624         if (!source->HasLowID()) {
 625                 uint32 nClientIP = wxUINT32_SWAP_ALWAYS(source->GetUserIDHybrid());
 626                 if (!IsGoodIP(nClientIP, thePrefs::FilterLanIPs())) { // check for 0-IP, localhost and LAN addresses
 627                         AddDebugLogLineM(false, logIPFilter, wxT("Ignored already known source with IP=%s") + Uint32toStringIP(nClientIP));
 628                         return;
 629                 }
 630         }
 631
 632         // Filter sources which are incompatible with our encryption setting (one requires it, and the other one doesn't supports it)
 633         if ( (source->RequiresCryptLayer() && (!thePrefs::IsClientCryptLayerSupported() || !source->HasValidHash())) || (thePrefs::IsClientCryptLayerRequired() && (!source->SupportsCryptLayer() || !source->HasValidHash())))
 634         {
 635                 source->Safe_Delete();
 636                 return;
 637         }
 638
 639         CPartFile* file = source->GetRequestFile();
 640
 641         // Check if the file is already queued for something else
 642         if ( file ) {
 643                 if ( file != sender ) {
 644                         if ( source->AddRequestForAnotherFile( sender ) ) {
 645                                 Notify_DownloadCtrlAddSource( sender, source, A4AF_SOURCE );
 646                         }
 647                 }
 648         } else {
 649                 source->SetRequestFile( sender );
 650
 651                 if ( source->GetFileRating() || !source->GetFileComment().IsEmpty() ) {
 652                         sender->UpdateFileRatingCommentAvail();
 653                 }
 654
 655                 source->SetSourceFrom(SF_PASSIVE);
 656                 sender->AddSource( source );
 657                 Notify_DownloadCtrlAddSource( sender, source, UNAVAILABLE_SOURCE);
 658         }
 659 }
 660
 661
 662 bool CDownloadQueue::RemoveSource(CUpDownClient* toremove, bool WXUNUSED(updatewindow), bool bDoStatsUpdate)
 663 {
 664         bool removed = false;
 665         toremove->DeleteAllFileRequests();
 666
 667         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
 668                 CPartFile* cur_file = GetFileByIndex( i );
 669
 670                 // Remove from source-list
 671                 if ( cur_file->DelSource( toremove ) ) {
 672                         cur_file->RemoveDownloadingSource(toremove);
 673                         removed = true;
 674                         if ( bDoStatsUpdate ) {
 675                                 cur_file->UpdatePartsInfo();
 676                         }
 677                 }
 678
 679                 // Remove from A4AF-list
 680                 cur_file->RemoveA4AFSource( toremove );
 681         }
 682
 683
 684         if ( !toremove->GetFileComment().IsEmpty() || toremove->GetFileRating()>0) {
 685                 toremove->GetRequestFile()->UpdateFileRatingCommentAvail();
 686         }
 687
 688         toremove->SetRequestFile( NULL );
 689         toremove->SetDownloadState(DS_NONE);
 690
 691         // Remove from downloadlist widget
 692         Notify_DownloadCtrlRemoveSource(toremove, (CPartFile*)NULL);
 693         toremove->ResetFileStatusInfo();
 694
 695         return removed;
 696 }
 697
 698
 699 void CDownloadQueue::RemoveFile(CPartFile* file)
 700 {
 701         RemoveLocalServerRequest( file );
 702
 703         NotifyObservers( EventType( EventType::REMOVED, file ) );
 704
 705         wxMutexLocker lock( m_mutex );
 706
 707         EraseValue( m_filelist, file );
 708 }
 709
 710
 711 CUpDownClient* CDownloadQueue::GetDownloadClientByIP_UDP(uint32 dwIP, uint16 nUDPPort) const
 712 {
 713         wxMutexLocker lock( m_mutex );
 714
 715         for ( unsigned int i = 0; i < m_filelist.size(); i++ ) {
 716                 const CPartFile::SourceSet& set = m_filelist[i]->GetSourceList();
 717
 718                 for ( CPartFile::SourceSet::const_iterator it = set.begin(); it != set.end(); it++ ) {
 719                         if ( (*it)->GetIP() == dwIP && (*it)->GetUDPPort() == nUDPPort ) {
 720                                 return *it;
 721                         }
 722                 }
 723         }
 724         return NULL;
 725 }
 726
 727
 728 /**
 729  * Checks if the specified server is the one we are connected to.
 730  */
 731 bool IsConnectedServer(const CServer* server)
 732 {
 733         if (server && theApp->serverconnect->GetCurrentServer()) {
 734                 wxString srvAddr = theApp->serverconnect->GetCurrentServer()->GetAddress();
 735                 uint16 srvPort = theApp->serverconnect->GetCurrentServer()->GetPort();
 736
 737                 return server->GetAddress() == srvAddr && server->GetPort() == srvPort;
 738         }
 739
 740         return false;
 741 }
 742
 743
 744 bool CDownloadQueue::SendNextUDPPacket()
 745 {
 746         if ( m_filelist.empty() || !theApp->serverconnect->IsUDPSocketAvailable() || !theApp->IsConnectedED2K()) {
 747                 return false;
 748         }
 749
 750         // Start monitoring the server and the files list
 751         if ( !m_queueServers.IsActive() ) {
 752                 AddObserver( &m_queueFiles );
 753
 754                 theApp->serverlist->AddObserver( &m_queueServers );
 755         }
 756
 757
 758         bool packetSent = false;
 759         while ( !packetSent ) {
 760                 // Get max files ids per packet for current server
 761                 int filesAllowed = GetMaxFilesPerUDPServerPacket();
 762
 763                 if (filesAllowed < 1 || !m_udpserver || IsConnectedServer(m_udpserver)) {
 764                         // Select the next server to ask, must not be the connected server
 765                         do {
 766                                 m_udpserver = m_queueServers.GetNext();
 767                         } while (IsConnectedServer(m_udpserver));
 768
 769                         m_cRequestsSentToServer = 0;
 770                         filesAllowed = GetMaxFilesPerUDPServerPacket();
 771                 }
 772
 773
 774                 // Check if we have asked all servers, in which case we are done
 775                 if (m_udpserver == NULL) {
 776                         DoStopUDPRequests();
 777
 778                         return false;
 779                 }
 780
 781                 // Memoryfile containing the hash of every file to request
 782                 // 28bytes allocation because 16b + 4b + 8b is the worse case scenario.
 783                 CMemFile hashlist( 28 );
 784
 785                 CPartFile* file = m_queueFiles.GetNext();
 786
 787                 while ( file && filesAllowed ) {
 788                         uint8 status = file->GetStatus();
 789
 790                         if ( ( status == PS_READY || status == PS_EMPTY ) && file->GetSourceCount() < thePrefs::GetMaxSourcePerFileUDP() ) {
 791                                 if (file->IsLargeFile() && !m_udpserver->SupportsLargeFilesUDP()) {
 792                                         AddDebugLogLineM(false, logDownloadQueue, wxT("UDP Request for sources on a large file ignored: server doesn't support it"));
 793                                 } else {
 794                                         ++m_cRequestsSentToServer;
 795                                         hashlist.WriteHash( file->GetFileHash() );
 796                                         // See the notes on TCP packet
 797                                         if ( m_udpserver->GetUDPFlags() & SRV_UDPFLG_EXT_GETSOURCES2 ) {
 798                                                 if (file->IsLargeFile()) {
 799                                                         wxASSERT(m_udpserver->SupportsLargeFilesUDP());
 800                                                         hashlist.WriteUInt32( 0 );
 801                                                         hashlist.WriteUInt64( file->GetFileSize() );
 802                                                 } else {
 803                                                         hashlist.WriteUInt32( file->GetFileSize() );
 804                                                 }
 805                                         }
 806                                         --filesAllowed;
 807                                 }
 808                         }
 809
 810                         // Avoid skipping a file if we can't send any more currently
 811                         if ( filesAllowed ) {
 812                                 file = m_queueFiles.GetNext();
 813                         }
 814                 }
 815
 816                 // See if we have anything to send
 817                 if ( hashlist.GetLength() ) {
 818                         packetSent = SendGlobGetSourcesUDPPacket(hashlist);
 819                 }
 820
 821                 // Check if we've covered every file
 822                 if ( file == NULL ) {
 823                         // Reset the list of asked files so that the loop will start over
 824                         m_queueFiles.Reset();
 825
 826                         // Unset the server so that the next server will be used
 827                         m_udpserver = NULL;
 828                 }
 829         }
 830
 831         return true;
 832 }
 833
 834
 835 void CDownloadQueue::StopUDPRequests()
 836 {
 837         wxMutexLocker lock( m_mutex );
 838
 839         DoStopUDPRequests();
 840 }
 841
 842
 843 void CDownloadQueue::DoStopUDPRequests()
 844 {
 845         // No need to observe when we wont be using the results
 846         theApp->serverlist->RemoveObserver( &m_queueServers );
 847         RemoveObserver( &m_queueFiles );
 848
 849         m_udpserver = 0;
 850         m_lastudpsearchtime = ::GetTickCount();
 851 }
 852
 853
 854 // Comparison function needed by sort. Returns true if file1 preceeds file2
 855 bool ComparePartFiles(const CPartFile* file1, const CPartFile* file2) {
 856         if (file1->GetDownPriority() != file2->GetDownPriority()) {
 857                 // To place high-priority files before low priority files we have to
 858                 // invert this test, since PR_LOW is lower than PR_HIGH, and since
 859                 // placing a PR_LOW file before a PR_HIGH file would mean that
 860                 // the PR_LOW file gets sources before the PR_HIGH file ...
 861                 return (file1->GetDownPriority() > file2->GetDownPriority());
 862         } else {
 863                 int sourcesA = file1->GetSourceCount();
 864                 int sourcesB = file2->GetSourceCount();
 865
 866                 int notSourcesA = file1->GetNotCurrentSourcesCount();
 867                 int notSourcesB = file2->GetNotCurrentSourcesCount();
 868
 869                 int cmp = CmpAny( sourcesA - notSourcesA, sourcesB - notSourcesB );
 870
 871                 if ( cmp == 0 ) {
 872                         cmp = CmpAny( notSourcesA, notSourcesB );
 873                 }
 874
 875                 return cmp < 0;
 876         }
 877 }
 878
 879
 880 void CDownloadQueue::DoSortByPriority()
 881 {
 882         m_lastsorttime = ::GetTickCount();
 883         sort( m_filelist.begin(), m_filelist.end(), ComparePartFiles );
 884 }
 885
 886
 887 void CDownloadQueue::ResetLocalServerRequests()
 888 {
 889         wxMutexLocker lock( m_mutex );
 890
 891         m_dwNextTCPSrcReq = 0;
 892         m_localServerReqQueue.clear();
 893
 894         for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
 895                 m_filelist[i]->SetLocalSrcRequestQueued(false);
 896         }
 897 }
 898
 899
 900 void CDownloadQueue::RemoveLocalServerRequest( CPartFile* file )
 901 {
 902         wxMutexLocker lock( m_mutex );
 903
 904         EraseValue( m_localServerReqQueue, file );
 905
 906         file->SetLocalSrcRequestQueued(false);
 907 }
 908
 909
 910 void CDownloadQueue::ProcessLocalRequests()
 911 {
 912         wxMutexLocker lock( m_mutex );
 913
 914         bool bServerSupportsLargeFiles = theApp->serverconnect
 915                                                                         && theApp->serverconnect->GetCurrentServer()
 916                                                                         && theApp->serverconnect->GetCurrentServer()->SupportsLargeFilesTCP();
 917
 918         if ( (!m_localServerReqQueue.empty()) && (m_dwNextTCPSrcReq < ::GetTickCount()) ) {
 919                 CMemFile dataTcpFrame(22);
 920                 const int iMaxFilesPerTcpFrame = 15;
 921                 int iFiles = 0;
 922                 while (!m_localServerReqQueue.empty() && iFiles < iMaxFilesPerTcpFrame) {
 923                         // find the file with the longest waitingtime
 924                         uint32 dwBestWaitTime = 0xFFFFFFFF;
 925
 926                         std::list<CPartFile*>::iterator posNextRequest = m_localServerReqQueue.end();
 927                         std::list<CPartFile*>::iterator it = m_localServerReqQueue.begin();
 928                         while( it != m_localServerReqQueue.end() ) {
 929                                 CPartFile* cur_file = (*it);
 930                                 if (cur_file->GetStatus() == PS_READY || cur_file->GetStatus() == PS_EMPTY) {
 931                                         uint8 nPriority = cur_file->GetDownPriority();
 932                                         if (nPriority > PR_HIGH) {
 933                                                 wxFAIL;
 934                                                 nPriority = PR_HIGH;
 935                                         }
 936
 937                                         if (cur_file->GetLastSearchTime() + (PR_HIGH-nPriority) < dwBestWaitTime ){
 938                                                 dwBestWaitTime = cur_file->GetLastSearchTime() + (PR_HIGH - nPriority);
 939                                                 posNextRequest = it;
 940                                         }
 941
 942                                         it++;
 943                                 } else {
 944                                         it = m_localServerReqQueue.erase(it);
 945                                         cur_file->SetLocalSrcRequestQueued(false);
 946                                         AddDebugLogLineM( false, logDownloadQueue,
 947                                                 CFormat(wxT("Local server source request for file '%s' not sent because of status '%s'"))
 948                                                         % cur_file->GetFileName() % cur_file->getPartfileStatus());
 949                                 }
 950                         }
 951
 952                         if (posNextRequest != m_localServerReqQueue.end()) {
 953                                 CPartFile* cur_file = (*posNextRequest);
 954                                 cur_file->SetLocalSrcRequestQueued(false);
 955                                 cur_file->SetLastSearchTime(::GetTickCount());
 956                                 m_localServerReqQueue.erase(posNextRequest);
 957                                 iFiles++;
 958
 959                                 if (!bServerSupportsLargeFiles && cur_file->IsLargeFile()) {
 960                                         AddDebugLogLineM(false, logDownloadQueue, wxT("TCP Request for sources on a large file ignored: server doesn't support it"));
 961                                 } else {
 962                                         AddDebugLogLineM(false, logDownloadQueue,
 963                                                 CFormat(wxT("Creating local sources request packet for '%s'")) % cur_file->GetFileName());
 964                                         // create request packet
 965                                         CMemFile data(16 + (cur_file->IsLargeFile() ? 8 : 4));
 966                                         data.WriteHash(cur_file->GetFileHash());
 967                                         // Kry - lugdunum extended protocol on 17.3 to handle filesize properly.
 968                                         // There is no need to check anything, old server ignore the extra 4 bytes.
 969                                         // As of 17.9, servers accept a 0 32-bits size and then a 64bits size
 970                                         if (cur_file->IsLargeFile()) {
 971                                                 wxASSERT(bServerSupportsLargeFiles);
 972                                                 data.WriteUInt32(0);
 973                                                 data.WriteUInt64(cur_file->GetFileSize());
 974                                         } else {
 975                                                 data.WriteUInt32(cur_file->GetFileSize());
 976                                         }
 977                                         uint8 byOpcode = 0;
 978                                         if (thePrefs::IsClientCryptLayerSupported() && theApp->serverconnect->GetCurrentServer() != NULL && theApp->serverconnect->GetCurrentServer()->SupportsGetSourcesObfuscation()) {
 979                                                 byOpcode = OP_GETSOURCES_OBFU;
 980                                         } else {
 981                                                 byOpcode = OP_GETSOURCES;
 982                                         }
 983                                         CPacket packet(data, OP_EDONKEYPROT, byOpcode);
 984                                         dataTcpFrame.Write(packet.GetPacket(), packet.GetRealPacketSize());
 985                                 }
 986                         }
 987                 }
 988
 989                 int iSize = dataTcpFrame.GetLength();
 990                 if (iSize > 0) {
 991                         // create one 'packet' which contains all buffered OP_GETSOURCES ED2K packets to be sent with one TCP frame
 992                         // server credits: (16+4)*regularfiles + (16+4+8)*largefiles +1
 993                         CScopedPtr<CPacket> packet(new CPacket(new byte[iSize], dataTcpFrame.GetLength(), true, false));
 994                         dataTcpFrame.Seek(0, wxFromStart);
 995                         dataTcpFrame.Read(packet->GetPacket(), iSize);
 996                         uint32 size = packet->GetPacketSize();
 997                         theApp->serverconnect->SendPacket(packet.release(), true);      // Deletes `packet'.
 998                         AddDebugLogLineM(false, logDownloadQueue, wxT("Sent local sources request packet."));
 999                         theStats::AddUpOverheadServer(size);
1000                 }
1001
1002                 // next TCP frame with up to 15 source requests is allowed to be sent in..
1003                 m_dwNextTCPSrcReq = ::GetTickCount() + SEC2MS(iMaxFilesPerTcpFrame*(16+4));
1004         }
1005
1006 }
1007
1008
1009 void CDownloadQueue::SendLocalSrcRequest(CPartFile* sender)
1010 {
1011         wxMutexLocker lock( m_mutex );
1012
1013         m_localServerReqQueue.push_back(sender);
1014 }
1015
1016
1017 void CDownloadQueue::AddLinksFromFile()
1018 {
1019         const wxString fullPath = theApp->ConfigDir + wxT("ED2KLinks");
1020         if (!wxFile::Exists(fullPath)) {
1021                 return;
1022         }
1023
1024         // Attempt to lock the ED2KLinks file.
1025         CFileLock lock((const char*)unicode2char(fullPath));
1026
1027         wxTextFile file(fullPath);
1028         if ( file.Open() ) {
1029                 for ( unsigned int i = 0; i < file.GetLineCount(); i++ ) {
1030                         wxString line = file.GetLine( i ).Strip( wxString::both );
1031
1032                         if ( !line.IsEmpty() ) {
1033                                 // Special case! used by a secondary running mule to raise this one.
1034                                 if ( line == wxT("RAISE_DIALOG")  ) {
1035                                         Notify_ShowGUI();
1036                                         continue;
1037                                 }
1038
1039                                 unsigned long category = 0;
1040                                 if (line.AfterLast(wxT(':')).ToULong(&category) == true) {
1041                                         line = line.BeforeLast(wxT(':'));
1042                                         if (category >= theApp->glob_prefs->GetCatCount()) {
1043                                                 category = 0;
1044                                         }
1045                                 } else { // If ToULong returns false the category still can have been changed!
1046                                                  // This is fixed in wx 2.9
1047                                         category = 0;
1048                                 }
1049                                 AddLink(line, category);
1050                         }
1051                 }
1052
1053                 file.Close();
1054         } else {
1055                 AddLogLineNS(_("Failed to open ED2KLinks file."));
1056         }
1057
1058         // Delete the file.
1059         wxRemoveFile(theApp->ConfigDir +  wxT("ED2KLinks"));
1060 }
1061
1062
1063 void CDownloadQueue::ResetCatParts(uint8 cat)
1064 {
1065         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
1066                 CPartFile* file = GetFileByIndex( i );
1067
1068                 if ( file->GetCategory() == cat ) {
1069                         // Reset the category
1070                         file->SetCategory( 0 );
1071                 } else if ( file->GetCategory() > cat ) {
1072                         // Set to the new position of the original category
1073                         file->SetCategory( file->GetCategory() - 1 );
1074                 }
1075         }
1076 }
1077
1078
1079 void CDownloadQueue::SetCatPrio(uint8 cat, uint8 newprio)
1080 {
1081         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
1082                 CPartFile* file = GetFileByIndex( i );
1083
1084                 if ( !cat || file->GetCategory() == cat ) {
1085                         if ( newprio == PR_AUTO ) {
1086                                 file->SetAutoDownPriority(true);
1087                         } else {
1088                                 file->SetAutoDownPriority(false);
1089                                 file->SetDownPriority(newprio);
1090                         }
1091                 }
1092         }
1093 }
1094
1095
1096 void CDownloadQueue::SetCatStatus(uint8 cat, int newstatus)
1097 {
1098         std::list<CPartFile*> files;
1099
1100         {
1101                 wxMutexLocker lock(m_mutex);
1102
1103                 for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
1104                         if ( m_filelist[i]->CheckShowItemInGivenCat(cat) ) {
1105                                 files.push_back( m_filelist[i] );
1106                         }
1107                 }
1108         }
1109
1110         std::list<CPartFile*>::iterator it = files.begin();
1111
1112         for ( ; it != files.end(); it++ ) {
1113                 switch ( newstatus ) {
1114                         case MP_CANCEL: (*it)->Delete();        break;
1115                         case MP_PAUSE:  (*it)->PauseFile();     break;
1116                         case MP_STOP:   (*it)->StopFile();      break;
1117                         case MP_RESUME: (*it)->ResumeFile();    break;
1118                 }
1119         }
1120 }
1121
1122
1123 uint16 CDownloadQueue::GetDownloadingFileCount() const
1124 {
1125         wxMutexLocker lock( m_mutex );
1126
1127         uint16 count = 0;
1128         for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
1129                 uint8 status = m_filelist[i]->GetStatus();
1130                 if ( status == PS_READY || status == PS_EMPTY ) {
1131                         count++;
1132                 }
1133         }
1134
1135         return count;
1136 }
1137
1138
1139 uint16 CDownloadQueue::GetPausedFileCount() const
1140 {
1141         wxMutexLocker lock( m_mutex );
1142
1143         uint16 count = 0;
1144         for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
1145                 if ( m_filelist[i]->GetStatus() == PS_PAUSED ) {
1146                         count++;
1147                 }
1148         }
1149
1150         return count;
1151 }
1152
1153
1154 void CDownloadQueue::CheckDiskspace( const CPath& path )
1155 {
1156         if ( ::GetTickCount() - m_lastDiskCheck < DISKSPACERECHECKTIME ) {
1157                 return;
1158         }
1159
1160         m_lastDiskCheck = ::GetTickCount();
1161
1162         uint64 min = 0;
1163         // Check if the user has set an explicit limit
1164         if ( thePrefs::IsCheckDiskspaceEnabled() ) {
1165                 min = thePrefs::GetMinFreeDiskSpace();
1166         }
1167
1168         // The very least acceptable diskspace is a single PART
1169         if ( min < PARTSIZE ) {
1170                 min = PARTSIZE;
1171         }
1172
1173         uint64 free = CPath::GetFreeSpaceAt(path);
1174         if (free == static_cast<uint64>(wxInvalidOffset)) {
1175                 return;
1176         } else if (free < min) {
1177                 CUserEvents::ProcessEvent(
1178                         CUserEvents::OutOfDiskSpace,
1179                         wxT("Temporary partition"));
1180         }
1181
1182         for (unsigned int i = 0; i < m_filelist.size(); ++i) {
1183                 CPartFile* file = m_filelist[i];
1184
1185                 switch ( file->GetStatus() ) {
1186                         case PS_ERROR:
1187                         case PS_COMPLETING:
1188                         case PS_COMPLETE:
1189                                 continue;
1190                 }
1191
1192                 if ( free >= min && file->GetInsufficient() ) {
1193                         // We'll try to resume files if there is enough free space
1194                         if ( free - file->GetNeededSpace() > min ) {
1195                                 file->ResumeFile();
1196                         }
1197                 } else if ( free < min && !file->IsPaused() ) {
1198                         // No space left, stop the files.
1199                         file->PauseFile( true );
1200                 }
1201         }
1202 }
1203
1204
1205 int CDownloadQueue::GetMaxFilesPerUDPServerPacket() const
1206 {
1207         if ( m_udpserver ) {
1208                 if ( m_udpserver->GetUDPFlags() & SRV_UDPFLG_EXT_GETSOURCES ) {
1209                         // get max. file ids per packet
1210                         if ( m_cRequestsSentToServer < MAX_REQUESTS_PER_SERVER ) {
1211                                 return std::min(
1212                                         MAX_FILES_PER_UDP_PACKET,
1213                                         MAX_REQUESTS_PER_SERVER - m_cRequestsSentToServer
1214                                 );
1215                         }
1216                 } else if ( m_cRequestsSentToServer < MAX_REQUESTS_PER_SERVER ) {
1217                         return 1;
1218                 }
1219         }
1220
1221         return 0;
1222 }
1223
1224
1225 bool CDownloadQueue::SendGlobGetSourcesUDPPacket(CMemFile& data)
1226 {
1227         if (!m_udpserver) {
1228                 return false;
1229         }
1230
1231         CPacket packet(data, OP_EDONKEYPROT, ((m_udpserver->GetUDPFlags() & SRV_UDPFLG_EXT_GETSOURCES2) ? OP_GLOBGETSOURCES2 : OP_GLOBGETSOURCES));
1232
1233         theStats::AddUpOverheadServer(packet.GetPacketSize());
1234         theApp->serverconnect->SendUDPPacket(&packet,m_udpserver,false);
1235
1236         return true;
1237 }
1238
1239
1240 void CDownloadQueue::AddToResolve(const CMD4Hash& fileid, const wxString& pszHostname, uint16 port, const wxString& hash, uint8 cryptoptions)
1241 {
1242         // double checking
1243         if ( !GetFileByID(fileid) ) {
1244                 return;
1245         }
1246
1247         wxMutexLocker lock( m_mutex );
1248
1249         Hostname_Entry entry = { fileid, pszHostname, port, hash, cryptoptions };
1250         m_toresolve.push_front(entry);
1251
1252         // Check if there are other DNS lookups on queue
1253         if (m_toresolve.size() == 1) {
1254                 // Check if it is a simple dot address
1255                 uint32 ip = StringIPtoUint32(pszHostname);
1256
1257                 if (ip) {
1258                         OnHostnameResolved(ip);
1259                 } else {
1260                         CAsyncDNS* dns = new CAsyncDNS(pszHostname, DNS_SOURCE, theApp);
1261
1262                         if ((dns->Create() != wxTHREAD_NO_ERROR) || (dns->Run() != wxTHREAD_NO_ERROR)) {
1263                                 dns->Delete();
1264                                 m_toresolve.pop_front();
1265                         }
1266                 }
1267         }
1268 }
1269
1270
1271 void CDownloadQueue::OnHostnameResolved(uint32 ip)
1272 {
1273         wxMutexLocker lock( m_mutex );
1274
1275         wxASSERT( m_toresolve.size() );
1276
1277         Hostname_Entry resolved = m_toresolve.front();
1278         m_toresolve.pop_front();
1279
1280         if ( ip ) {
1281                 CPartFile* file = GetFileByID( resolved.fileid );
1282                 if ( file ) {
1283                         CMemFile sources(1+4+2);
1284                         sources.WriteUInt8(1); // No. Sources
1285                         sources.WriteUInt32(ip);
1286                         sources.WriteUInt16(resolved.port);
1287                         sources.WriteUInt8(resolved.cryptoptions);
1288                         if (resolved.cryptoptions & 0x80) {
1289                                 wxASSERT(!resolved.hash.IsEmpty());
1290                                 CMD4Hash sourcehash;
1291                                 sourcehash.Decode(resolved.hash);
1292                                 sources.WriteHash(sourcehash);
1293                         }
1294                         sources.Seek(0,wxFromStart);
1295
1296                         file->AddSources(sources, 0, 0, SF_LINK, true);
1297                 }
1298         }
1299
1300         while (m_toresolve.size()) {
1301                 Hostname_Entry entry = m_toresolve.front();
1302
1303                 // Check if it is a simple dot address
1304                 uint32 tmpIP = StringIPtoUint32(entry.strHostname);
1305
1306                 if (tmpIP) {
1307                         OnHostnameResolved(tmpIP);
1308                 } else {
1309                         CAsyncDNS* dns = new CAsyncDNS(entry.strHostname, DNS_SOURCE, theApp);
1310
1311                         if ((dns->Create() != wxTHREAD_NO_ERROR) || (dns->Run() != wxTHREAD_NO_ERROR)) {
1312                                 dns->Delete();
1313                                 m_toresolve.pop_front();
1314                         } else {
1315                                 break;
1316                         }
1317                 }
1318         }
1319 }
1320
1321
1322 bool CDownloadQueue::AddLink( const wxString& link, uint8 category )
1323 {
1324         wxString uri(link);
1325
1326         if (link.compare(0, 7, wxT("magnet:")) == 0) {
1327                 uri = CMagnetED2KConverter(link);
1328                 if (uri.empty()) {
1329                         AddLogLineM(true, CFormat(_("Cannot convert magnet link to eD2k: %s")) % link);
1330                         return false;
1331                 }
1332         }
1333
1334         if (uri.compare(0, 7, wxT("ed2k://")) == 0) {
1335                 return AddED2KLink(uri, category);
1336         } else {
1337                 AddLogLineM(true, CFormat(_("Unknown protocol of link: %s")) % link);
1338                 return false;
1339         }
1340 }
1341
1342
1343 bool CDownloadQueue::AddED2KLink( const wxString& link, uint8 category )
1344 {
1345         wxASSERT( !link.IsEmpty() );
1346         wxString URI = link;
1347
1348         // Need the links to end with /, otherwise CreateLinkFromUrl crashes us.
1349         if ( URI.Last() != wxT('/') ) {
1350                 URI += wxT("/");
1351         }
1352
1353         try {
1354                 CScopedPtr<CED2KLink> uri(CED2KLink::CreateLinkFromUrl(URI));
1355
1356                 return AddED2KLink( uri.get(), category );
1357         } catch ( const wxString& err ) {
1358                 AddLogLineM( true, CFormat( _("Invalid eD2k link! ERROR: %s")) % err);
1359         }
1360
1361         return false;
1362 }
1363
1364
1365 bool CDownloadQueue::AddED2KLink( const CED2KLink* link, uint8 category )
1366 {
1367         switch ( link->GetKind() ) {
1368                 case CED2KLink::kFile:
1369                         return AddED2KLink( dynamic_cast<const CED2KFileLink*>( link ), category );
1370
1371                 case CED2KLink::kServer:
1372                         return AddED2KLink( dynamic_cast<const CED2KServerLink*>( link ) );
1373
1374                 case CED2KLink::kServerList:
1375                         return AddED2KLink( dynamic_cast<const CED2KServerListLink*>( link ) );
1376
1377                 default:
1378                         return false;
1379         }
1380 }
1381
1382
1383
1384 bool CDownloadQueue::AddED2KLink( const CED2KFileLink* link, uint8 category )
1385 {
1386         CPartFile* file = NULL;
1387         if (IsFileExisting(link->GetHashKey())) {
1388                 // Must be a shared file if we are to add hashes or sources
1389                 if ((file = GetFileByID(link->GetHashKey())) == NULL) {
1390                         return false;
1391                 }
1392         } else {
1393                 if (link->GetSize() > OLD_MAX_FILE_SIZE) {
1394                         if (!PlatformSpecific::CanFSHandleLargeFiles(thePrefs::GetTempDir())) {
1395                                 AddLogLineM(true, _("Filesystem for Temp directory cannot handle large files."));
1396                                 return false;
1397                         } else if (!PlatformSpecific::CanFSHandleLargeFiles(theApp->glob_prefs->GetCatPath(category))) {
1398                                 AddLogLineM(true, _("Filesystem for Incoming directory cannot handle large files."));
1399                                 return false;
1400                         }
1401                 }
1402
1403                 file = new CPartFile(link);
1404
1405                 if (file->GetStatus() == PS_ERROR) {
1406                         delete file;
1407                         return false;
1408                 }
1409
1410                 AddDownload(file, thePrefs::AddNewFilesPaused(), category);
1411         }
1412
1413         if (link->HasValidAICHHash()) {
1414                 CAICHHashSet* hashset = file->GetAICHHashset();
1415
1416                 if (!hashset->HasValidMasterHash() || (hashset->GetMasterHash() != link->GetAICHHash())) {
1417                         hashset->SetMasterHash(link->GetAICHHash(), AICH_VERIFIED);
1418                         hashset->FreeHashSet();
1419                 }
1420         }
1421
1422         const CED2KFileLink::CED2KLinkSourceList& list = link->m_sources;
1423         CED2KFileLink::CED2KLinkSourceList::const_iterator it = list.begin();
1424         for (; it != list.end(); ++it) {
1425                 AddToResolve(link->GetHashKey(), it->addr, it->port, it->hash, it->cryptoptions);
1426         }
1427
1428         return true;
1429 }
1430
1431
1432 bool CDownloadQueue::AddED2KLink( const CED2KServerLink* link )
1433 {
1434         CServer *server = new CServer( link->GetPort(), Uint32toStringIP( link->GetIP() ) );
1435
1436         server->SetListName( Uint32toStringIP( link->GetIP() ) );
1437
1438         theApp->serverlist->AddServer(server);
1439
1440         Notify_ServerAdd(server);
1441
1442         return true;
1443 }
1444
1445
1446 bool CDownloadQueue::AddED2KLink( const CED2KServerListLink* link )
1447 {
1448         theApp->serverlist->UpdateServerMetFromURL( link->GetAddress() );
1449
1450         return true;
1451 }
1452
1453
1454 void CDownloadQueue::ObserverAdded( ObserverType* o )
1455 {
1456         CObservableQueue<CPartFile*>::ObserverAdded( o );
1457
1458         EventType::ValueList list;
1459
1460         {
1461                 wxMutexLocker lock(m_mutex);
1462                 list.reserve( m_filelist.size() );
1463                 list.insert( list.begin(), m_filelist.begin(), m_filelist.end() );
1464         }
1465
1466         NotifyObservers( EventType( EventType::INITIAL, &list ), o );
1467 }
1468
1469 void CDownloadQueue::KademliaSearchFile(uint32_t searchID, const Kademlia::CUInt128* pcontactID, const Kademlia::CUInt128* pbuddyID, uint8_t type, uint32_t ip, uint16_t tcp, uint16_t udp, uint32_t buddyip, uint16_t buddyport, uint8_t byCryptOptions)
1470 {
1471         AddDebugLogLineM(false, logKadSearch, wxString::Format(wxT("Search result sources (type %i)"),type));
1472
1473         //Safety measure to make sure we are looking for these sources
1474         CPartFile* temp = GetFileByKadFileSearchID(searchID);
1475         if( !temp ) {
1476                 AddDebugLogLineM(false, logKadSearch, wxT("This is not the file we're looking for..."));
1477                 return;
1478         }
1479
1480         //Do we need more sources?
1481         if(!(!temp->IsStopped() && thePrefs::GetMaxSourcePerFile() > temp->GetSourceCount())) {
1482                 AddDebugLogLineM(false, logKadSearch, wxT("No more sources needed for this file"));
1483                 return;
1484         }
1485
1486         uint32_t ED2KID = wxUINT32_SWAP_ALWAYS(ip);
1487
1488         if (theApp->ipfilter->IsFiltered(ED2KID)) {
1489                 AddDebugLogLineM(false, logKadSearch, wxT("Source ip got filtered"));
1490                 AddDebugLogLineM(false, logIPFilter, CFormat(wxT("IPfiltered source IP=%s received from Kademlia")) % Uint32toStringIP(ED2KID));
1491                 return;
1492         }
1493
1494         if( (ip == Kademlia::CKademlia::GetIPAddress() || ED2KID == theApp->GetED2KID()) && tcp == thePrefs::GetPort()) {
1495                 AddDebugLogLineM(false, logKadSearch, wxT("Trying to add myself as source, ignore"));
1496                 return;
1497         }
1498
1499         CUpDownClient* ctemp = NULL;
1500         switch (type) {
1501                 case 4:
1502                 case 1: {
1503                         // NonFirewalled users
1504                         if(!tcp) {
1505                                 AddDebugLogLineM(false, logKadSearch, CFormat(wxT("Ignored source (IP=%s) received from Kademlia, no tcp port received")) % Uint32toStringIP(ip));
1506                                 return;
1507                         }
1508                         if (!IsGoodIP(ED2KID,thePrefs::FilterLanIPs())) {
1509                                 AddDebugLogLineM(false, logKadSearch, CFormat(wxT("%s got filtered")) % Uint32toStringIP(ED2KID));
1510                                 AddDebugLogLineM(false, logIPFilter, CFormat(wxT("Ignored source (IP=%s) received from Kademlia, filtered")) % Uint32toStringIP(ED2KID));
1511                                 return;
1512                         }
1513                         ctemp = new CUpDownClient(tcp, ip, 0, 0, temp, false, true);
1514                         ctemp->SetSourceFrom(SF_KADEMLIA);
1515                         // not actually sent or needed for HighID sources
1516                         //ctemp->SetServerIP(serverip);
1517                         //ctemp->SetServerPort(serverport);
1518                         ctemp->SetKadPort(udp);
1519                         byte cID[16];
1520                         pcontactID->ToByteArray(cID);
1521                         ctemp->SetUserHash(CMD4Hash(cID));
1522                         break;
1523                 }
1524                 case 2: {
1525                         // Don't use this type... Some clients will process it wrong..
1526                         break;
1527                 }
1528                 case 5:
1529                 case 3: {
1530                         // This will be a firewalled client connected to Kad only.
1531                         // We set the clientID to 1 as a Kad user only has 1 buddy.
1532                         ctemp = new CUpDownClient(tcp, 1, 0, 0, temp, false, true);
1533                         // The only reason we set the real IP is for when we get a callback
1534                         // from this firewalled source, the compare method will match them.
1535                         ctemp->SetSourceFrom(SF_KADEMLIA);
1536                         ctemp->SetKadPort(udp);
1537                         byte cID[16];
1538                         pcontactID->ToByteArray(cID);
1539                         ctemp->SetUserHash(CMD4Hash(cID));
1540                         pbuddyID->ToByteArray(cID);
1541                         ctemp->SetBuddyID(cID);
1542                         ctemp->SetBuddyIP(buddyip);
1543                         ctemp->SetBuddyPort(buddyport);
1544                         break;
1545                 }
1546                 case 6: {
1547                         // firewalled source which supports direct UDP callback
1548                         // if we are firewalled ourself, the source is useless to us
1549                         if (theApp->IsFirewalled()) {
1550                                 break;
1551                         }
1552
1553                         if ((byCryptOptions & 0x08) == 0){
1554                                 AddDebugLogLineM(false, logKadSearch, CFormat(wxT("Received Kad source type 6 (direct callback) which has the direct callback flag not set (%s)")) % Uint32toStringIP(ED2KID));
1555                                 break;
1556                         }
1557
1558                         ctemp = new CUpDownClient(tcp, 1, 0, 0, temp, false, true);
1559                         ctemp->SetSourceFrom(SF_KADEMLIA);
1560                         ctemp->SetKadPort(udp);
1561                         ctemp->SetIP(ED2KID); // need to set the IP address, which cannot be used for TCP but for UDP
1562                         byte cID[16];
1563                         pcontactID->ToByteArray(cID);
1564                         ctemp->SetUserHash(CMD4Hash(cID));
1565                 }
1566         }
1567
1568         if (ctemp) {
1569                 // add encryption settings
1570                 ctemp->SetConnectOptions(byCryptOptions);
1571
1572                 AddDebugLogLineM(false, logKadSearch, CFormat(wxT("Happily adding a source (%s) type %d")) % Uint32_16toStringIP_Port(ED2KID, ctemp->GetUserPort()) % type);
1573                 CheckAndAddSource(temp, ctemp);
1574         }
1575 }
1576
1577 CPartFile* CDownloadQueue::GetFileByKadFileSearchID(uint32 id) const
1578 {
1579         wxMutexLocker lock( m_mutex );
1580
1581         for ( uint16 i = 0; i < m_filelist.size(); ++i ) {
1582                 if ( id == m_filelist[i]->GetKadFileSearchID()) {
1583                         return m_filelist[ i ];
1584                 }
1585         }
1586
1587         return NULL;
1588 }
1589
1590 bool CDownloadQueue::DoKademliaFileRequest()
1591 {
1592         return ((::GetTickCount() - lastkademliafilerequest) > KADEMLIAASKTIME);
1593 }
1594 // File_checked_for_headers