src/DownloadQueue.cpp

   1 //
   2 // This file is part of the aMule Project.
   3 //
   4 // Copyright (c) 2003-2011 aMule Team ( admin@amule.org / http://www.amule.org )
   5 // Copyright (c) 2002-2011 Merkur ( devs@emule-project.net / http://www.emule-project.net )
   6 //
   7 // Any parts of this program derived from the xMule, lMule or eMule project,
   8 // or contributed by third-party developers are copyrighted by their
   9 // respective authors.
  10 //
  11 // This program is free software; you can redistribute it and/or modify
  12 // it under the terms of the GNU General Public License as published by
  13 // the Free Software Foundation; either version 2 of the License, or
  14 // (at your option) any later version.
  15 //
  16 // This program is distributed in the hope that it will be useful,
  17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 // GNU General Public License for more details.
  20 //
  21 // You should have received a copy of the GNU General Public License
  22 // along with this program; if not, write to the Free Software
  23 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
  24 //
  25
  26 #include "DownloadQueue.h"      // Interface declarations
  27
  28 #include <protocol/Protocols.h>
  29 #include <protocol/kad/Constants.h>
  30 #include <common/Macros.h>
  31 #include <common/MenuIDs.h>
  32 #include <common/Constants.h>
  33
  34 #include <wx/utils.h>
  35
  36 #include "Server.h"             // Needed for CServer
  37 #include "Packet.h"             // Needed for CPacket
  38 #include "MemFile.h"            // Needed for CMemFile
  39 #include "ClientList.h"         // Needed for CClientList
  40 #include "updownclient.h"       // Needed for CUpDownClient
  41 #include "ServerList.h"         // Needed for CServerList
  42 #include "ServerConnect.h"      // Needed for CServerConnect
  43 #include "ED2KLink.h"           // Needed for CED2KFileLink
  44 #include "SearchList.h"         // Needed for CSearchFile
  45 #include "SharedFileList.h"     // Needed for CSharedFileList
  46 #include "PartFile.h"           // Needed for CPartFile
  47 #include "Preferences.h"        // Needed for thePrefs
  48 #include "amule.h"              // Needed for theApp
  49 #include "AsyncDNS.h"           // Needed for CAsyncDNS
  50 #include "Statistics.h"         // Needed for theStats
  51 #include "Logger.h"
  52 #include <common/Format.h>      // Needed for CFormat
  53 #include "IPFilter.h"
  54 #include <common/FileFunctions.h>       // Needed for CDirIterator
  55 #include "GuiEvents.h"          // Needed for Notify_*
  56 #include "UserEvents.h"
  57 #include "MagnetURI.h"          // Needed for CMagnetED2KConverter
  58 #include "ScopedPtr.h"          // Needed for CScopedPtr
  59 #include "PlatformSpecific.h"   // Needed for CanFSHandleLargeFiles
  60
  61 #include "kademlia/kademlia/Kademlia.h"
  62
  63 #include <string>                       // Do_not_auto_remove (mingw-gcc-3.4.5)
  64
  65
  66 // Max. file IDs per UDP packet
  67 // ----------------------------
  68 // 576 - 30 bytes of header (28 for UDP, 2 for "E3 9A" edonkey proto) = 546 bytes
  69 // 546 / 16 = 34
  70
  71
  72 #define MAX_FILES_PER_UDP_PACKET        31      // 2+16*31 = 498 ... is still less than 512 bytes!!
  73 #define MAX_REQUESTS_PER_SERVER         35
  74
  75
  76 CDownloadQueue::CDownloadQueue()
  77 // Needs to be recursive that that is can own an observer assigned to itself
  78         : m_mutex( wxMUTEX_RECURSIVE )
  79 {
  80         m_datarate = 0;
  81         m_udpserver = 0;
  82         m_lastsorttime = 0;
  83         m_lastudpsearchtime = 0;
  84         m_lastudpstattime = 0;
  85         m_udcounter = 0;
  86         m_nLastED2KLinkCheck = 0;
  87         m_dwNextTCPSrcReq = 0;
  88         m_cRequestsSentToServer = 0;
  89         m_lastDiskCheck = 0;
  90
  91         // Static thresholds until dynamic kicks in.
  92         m_rareFileThreshold = RARE_FILE;
  93         m_commonFileThreshold = 100;
  94
  95         SetLastKademliaFileRequest();
  96 }
  97
  98
  99 CDownloadQueue::~CDownloadQueue()
 100 {
 101         if ( !m_filelist.empty() ) {
 102                 for ( unsigned int i = 0; i < m_filelist.size(); i++ ) {
 103                         AddLogLineNS(CFormat(_("Saving PartFile %u of %u")) % (i + 1) % m_filelist.size());
 104                         delete m_filelist[i];
 105                 }
 106                 AddLogLineNS(_("All PartFiles Saved."));
 107         }
 108 }
 109
 110
 111 void CDownloadQueue::LoadMetFiles(const CPath& path)
 112 {
 113         AddLogLineNS(CFormat(_("Loading temp files from %s.")) % path.GetPrintable());
 114
 115         std::vector<CPath> files;
 116
 117         // Locate part-files to be loaded
 118         CDirIterator TempDir(path);
 119         CPath fileName = TempDir.GetFirstFile(CDirIterator::File, wxT("*.part.met"));
 120         while (fileName.IsOk()) {
 121                 files.push_back(path.JoinPaths(fileName));
 122
 123                 fileName = TempDir.GetNextFile();
 124         }
 125
 126         // Loading in order makes it easier to figure which
 127         // file is broken in case of crashes, or the like.
 128         std::sort(files.begin(), files.end());
 129
 130         // Load part-files
 131         for ( size_t i = 0; i < files.size(); i++ ) {
 132                 AddLogLineNS(CFormat(_("Loading PartFile %u of %u")) % (i + 1) % files.size());
 133                 fileName = files[i].GetFullName();
 134                 CPartFile *toadd = new CPartFile();
 135                 bool result = toadd->LoadPartFile(path, fileName) != 0;
 136                 if (!result) {
 137                         // Try from backup
 138                         result = toadd->LoadPartFile(path, fileName, true) != 0;
 139                 }
 140                 if (result && !IsFileExisting(toadd->GetFileHash())) {
 141                         {
 142                                 wxMutexLocker lock(m_mutex);
 143                                 m_filelist.push_back(toadd);
 144                         }
 145                         NotifyObservers(EventType(EventType::INSERTED, toadd));
 146                         Notify_DownloadCtrlAddFile(toadd);
 147                 } else {
 148                         wxString msg;
 149                         if (result) {
 150                                 msg << CFormat(wxT("WARNING: Duplicate partfile with hash '%s' found, skipping: %s"))
 151                                         % toadd->GetFileHash().Encode() % fileName;
 152                         } else {
 153                                 // If result is false, then reading of both the primary and the backup .met failed
 154                                 AddLogLineN(_("ERROR: Failed to load backup file. Search http://forum.amule.org for .part.met recovery solutions."));
 155                                 msg << CFormat(wxT("ERROR: Failed to load PartFile '%s'")) % fileName;
 156                         }
 157                         AddLogLineCS(msg);
 158
 159                         // Delete the partfile object in the end.
 160                         delete toadd;
 161                 }
 162         }
 163         AddLogLineNS(_("All PartFiles Loaded."));
 164
 165         if ( GetFileCount() == 0 ) {
 166                 AddLogLineN(_("No part files found"));
 167         } else {
 168                 AddLogLineN(CFormat(wxPLURAL("Found %u part file", "Found %u part files", GetFileCount())) % GetFileCount());
 169
 170                 DoSortByPriority();
 171                 CheckDiskspace( path );
 172                 Notify_ShowUpdateCatTabTitles();
 173         }
 174 }
 175
 176
 177 uint16 CDownloadQueue::GetFileCount() const
 178 {
 179         wxMutexLocker lock( m_mutex );
 180
 181         return m_filelist.size();
 182 }
 183
 184
 185 void CDownloadQueue::CopyFileList(std::vector<CPartFile*>& out_list, bool includeCompleted) const
 186 {
 187         wxMutexLocker lock(m_mutex);
 188         uint32 reserve = m_filelist.size();
 189         if (includeCompleted) {
 190                 reserve += m_completedDownloads.size();
 191         }
 192         out_list.reserve(reserve);
 193         for (FileQueue::const_iterator it = m_filelist.begin(); it != m_filelist.end(); ++it) {
 194                 out_list.push_back(*it);
 195         }
 196         if (includeCompleted) {
 197                 for (FileList::const_iterator it = m_completedDownloads.begin(); it != m_completedDownloads.end(); ++it) {
 198                         out_list.push_back(*it);
 199                 }
 200         }
 201 }
 202
 203
 204 CServer* CDownloadQueue::GetUDPServer() const
 205 {
 206         wxMutexLocker lock( m_mutex );
 207
 208         return m_udpserver;
 209 }
 210
 211
 212 void CDownloadQueue::SetUDPServer( CServer* server )
 213 {
 214         wxMutexLocker lock( m_mutex );
 215
 216         m_udpserver = server;
 217 }
 218
 219
 220 void CDownloadQueue::SaveSourceSeeds()
 221 {
 222         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
 223                 GetFileByIndex( i )->SaveSourceSeeds();
 224         }
 225 }
 226
 227
 228 void CDownloadQueue::LoadSourceSeeds()
 229 {
 230         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
 231                 GetFileByIndex( i )->LoadSourceSeeds();
 232         }
 233 }
 234
 235
 236 void CDownloadQueue::AddSearchToDownload(CSearchFile* toadd, uint8 category)
 237 {
 238         if ( IsFileExisting(toadd->GetFileHash()) ) {
 239                 return;
 240         }
 241
 242         if (toadd->GetFileSize() > OLD_MAX_FILE_SIZE) {
 243                 if (!PlatformSpecific::CanFSHandleLargeFiles(thePrefs::GetTempDir())) {
 244                         AddLogLineC(_("Filesystem for Temp directory cannot handle large files."));
 245                         return;
 246                 } else if (!PlatformSpecific::CanFSHandleLargeFiles(theApp->glob_prefs->GetCatPath(category))) {
 247                         AddLogLineC(_("Filesystem for Incoming directory cannot handle large files."));
 248                         return;
 249                 }
 250         }
 251
 252         CPartFile* newfile = NULL;
 253         try {
 254                 newfile = new CPartFile(toadd);
 255         } catch (const CInvalidPacket& WXUNUSED(e)) {
 256                 AddDebugLogLineC(logDownloadQueue, wxT("Search-result contained invalid tags, could not add"));
 257         }
 258
 259         if ( newfile && newfile->GetStatus() != PS_ERROR ) {
 260                 AddDownload( newfile, thePrefs::AddNewFilesPaused(), category );
 261                 // Add any possible sources
 262                 if (toadd->GetClientID() && toadd->GetClientPort()) {
 263                         CMemFile sources(1+4+2);
 264                         sources.WriteUInt8(1);
 265                         sources.WriteUInt32(toadd->GetClientID());
 266                         sources.WriteUInt16(toadd->GetClientPort());
 267                         sources.Reset();
 268                         newfile->AddSources(sources, toadd->GetClientServerIP(), toadd->GetClientServerPort(), SF_SEARCH_RESULT, false);
 269                 }
 270                 for (std::list<CSearchFile::ClientStruct>::const_iterator it = toadd->GetClients().begin(); it != toadd->GetClients().end(); ++it) {
 271                         CMemFile sources(1+4+2);
 272                         sources.WriteUInt8(1);
 273                         sources.WriteUInt32(it->m_ip);
 274                         sources.WriteUInt16(it->m_port);
 275                         sources.Reset();
 276                         newfile->AddSources(sources, it->m_serverIP, it->m_serverPort, SF_SEARCH_RESULT, false);
 277                 }
 278         } else {
 279                 delete newfile;
 280         }
 281 }
 282
 283
 284 struct SFindBestPF
 285 {
 286         void operator()(CPartFile* file) {
 287                 // Check if we should filter out other categories
 288                 int alphaorder = 0;
 289
 290                 if ((m_category != -1) && (file->GetCategory() != m_category)) {
 291                         return;
 292                 } else if (file->GetStatus() != PS_PAUSED) {
 293                         return;
 294                 } else if (m_alpha && m_result && ((alphaorder = file->GetFileName().GetPrintable().CmpNoCase(m_result->GetFileName().GetPrintable())) > 0)) {
 295                         return;
 296                 }
 297
 298                 if (!m_result) {
 299                         m_result = file;
 300                 } else {
 301                         if (m_alpha && (alphaorder < 0)) {
 302                                 m_result = file;
 303                         } else if (file->GetDownPriority() > m_result->GetDownPriority()) {
 304                                 // Either not alpha ordered, or they have the same alpha ordering (could happen if they have same name)
 305                                 m_result = file;
 306                         } else {
 307                                 // Lower priority file
 308                         }
 309                 }
 310         }
 311
 312         //! The category to look for, or -1 if any category is good
 313         int m_category;
 314         //! If any acceptable files are found, this variable store their pointer
 315         CPartFile* m_result;
 316         //! If we should order alphabetically
 317         bool m_alpha;
 318 };
 319
 320
 321 void CDownloadQueue::StartNextFile(CPartFile* oldfile)
 322 {
 323         if ( thePrefs::StartNextFile() ) {
 324                 SFindBestPF visitor = { -1, NULL, thePrefs::StartNextFileAlpha() };
 325
 326                 {
 327                         wxMutexLocker lock(m_mutex);
 328
 329                         if (thePrefs::StartNextFileSame()) {
 330                                 // Get a download in the same category
 331                                 visitor.m_category = oldfile->GetCategory();
 332
 333                                 visitor = std::for_each(m_filelist.begin(), m_filelist.end(), visitor);
 334                         }
 335
 336                         if (visitor.m_result == NULL) {
 337                                 // Get a download, regardless of category
 338                                 visitor.m_category = -1;
 339
 340                                 visitor = std::for_each(m_filelist.begin(), m_filelist.end(), visitor);
 341                         }
 342
 343                         // Alpha doesn't need special cases
 344                 }
 345
 346                 if (visitor.m_result) {
 347                         visitor.m_result->ResumeFile();
 348                 }
 349         }
 350 }
 351
 352
 353 void CDownloadQueue::AddDownload(CPartFile* file, bool paused, uint8 category)
 354 {
 355         wxCHECK_RET(!IsFileExisting(file->GetFileHash()), wxT("Adding duplicate part-file"));
 356
 357         if (file->GetStatus(true) == PS_ALLOCATING) {
 358                 file->PauseFile();
 359         } else if (paused && GetFileCount()) {
 360                 file->StopFile();
 361         }
 362
 363         {
 364                 wxMutexLocker lock(m_mutex);
 365                 m_filelist.push_back( file );
 366                 DoSortByPriority();
 367         }
 368
 369         NotifyObservers( EventType( EventType::INSERTED, file ) );
 370         if (category < theApp->glob_prefs->GetCatCount()) {
 371                 file->SetCategory(category);
 372         } else {
 373                 AddDebugLogLineN( logDownloadQueue, wxT("Tried to add download into invalid category.") );
 374         }
 375         Notify_DownloadCtrlAddFile( file );
 376         theApp->searchlist->UpdateSearchFileByHash(file->GetFileHash());        // Update file in the search dialog if it's still open
 377         AddLogLineC(CFormat(_("Downloading %s")) % file->GetFileName() );
 378 }
 379
 380
 381 bool CDownloadQueue::IsFileExisting( const CMD4Hash& fileid ) const
 382 {
 383         if (CKnownFile* file = theApp->sharedfiles->GetFileByID(fileid)) {
 384                 if (file->IsPartFile()) {
 385                         AddLogLineC(CFormat( _("You are already trying to download the file '%s'") ) % file->GetFileName());
 386                 } else {
 387                         // Check if the file exists, since otherwise the user is forced to
 388                         // manually reload the shares to download a file again.
 389                         CPath fullpath = file->GetFilePath().JoinPaths(file->GetFileName());
 390                         if (!fullpath.FileExists()) {
 391                                 // The file is no longer available, unshare it
 392                                 theApp->sharedfiles->RemoveFile(file);
 393
 394                                 return false;
 395                         }
 396
 397                         AddLogLineC(CFormat( _("You already have the file '%s'") ) % file->GetFileName());
 398                 }
 399
 400                 return true;
 401         } else if ((file = GetFileByID(fileid))) {
 402                 AddLogLineC(CFormat( _("You are already trying to download the file %s") ) % file->GetFileName());
 403                 return true;
 404         }
 405
 406         return false;
 407 }
 408
 409 #define RARITY_FACTOR 4 // < 25%
 410 #define NORMALITY_FACTOR 2 // <50%
 411 // x > NORMALITY_FACTOR -> High availablity.
 412
 413 void CDownloadQueue::Process()
 414 {
 415         // send src requests to local server
 416         ProcessLocalRequests();
 417
 418         {
 419                 wxMutexLocker lock(m_mutex);
 420
 421                 uint32 downspeed = 0;
 422                 if (thePrefs::GetMaxDownload() != UNLIMITED && m_datarate > 1500) {
 423                         downspeed = (((uint32)thePrefs::GetMaxDownload())*1024*100)/(m_datarate+1);
 424                         if (downspeed < 50) {
 425                                 downspeed = 50;
 426                         } else if (downspeed > 200) {
 427                                 downspeed = 200;
 428                         }
 429                 }
 430
 431                 m_datarate = 0;
 432                 m_udcounter++;
 433                 uint32 cur_datarate = 0;
 434                 uint32 cur_udcounter = m_udcounter;
 435
 436                 std::list<int> m_sourcecountlist;
 437
 438                 bool mustPreventSleep = false;
 439
 440                 for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
 441                         CPartFile* file = m_filelist[i];
 442
 443                         CMutexUnlocker unlocker(m_mutex);
 444
 445                         uint8 status = file->GetStatus();
 446                         mustPreventSleep |= !(status == PS_ERROR || status == PS_INSUFFICIENT || status == PS_PAUSED || status == PS_COMPLETE);
 447
 448                         if (status == PS_READY || status == PS_EMPTY ){
 449                                 cur_datarate += file->Process( downspeed, cur_udcounter );
 450                         } else {
 451                                 //This will make sure we don't keep old sources to paused and stoped files..
 452                                 file->StopPausedFile();
 453                         }
 454
 455                         if (!file->IsPaused() && !file->IsStopped()) {
 456                                 m_sourcecountlist.push_back(file->GetSourceCount());
 457                         }
 458                 }
 459
 460                 if (thePrefs::GetPreventSleepWhileDownloading()) {
 461                         if ((mustPreventSleep == false) && (theStats::GetSessionSentBytes() < theStats::GetSessionReceivedBytes())) {
 462                                 // I can see right through your clever plan.
 463                                 mustPreventSleep = true;
 464                         }
 465
 466                         if (mustPreventSleep) {
 467                                 PlatformSpecific::PreventSleepMode();
 468                         } else {
 469                                 PlatformSpecific::AllowSleepMode();
 470                         }
 471                 } else {
 472                         // Just in case the value changes while we're preventing. Calls to this function are totally inexpensive anwyay
 473                         PlatformSpecific::AllowSleepMode();
 474                 }
 475
 476
 477                 // Set the source rarity thresholds
 478                 int nSourceGroups = m_sourcecountlist.size();
 479                 if (nSourceGroups) {
 480                         m_sourcecountlist.sort();
 481                         if (nSourceGroups == 1) {
 482                                 // High anyway.
 483                                 m_rareFileThreshold = m_sourcecountlist.front() + 1;
 484                                 m_commonFileThreshold = m_rareFileThreshold + 1;
 485                         } else if (nSourceGroups == 2) {
 486                                 // One high, one low (unless they're both 0, then both high)
 487                                 m_rareFileThreshold = (m_sourcecountlist.back() > 0) ? (m_sourcecountlist.back() - 1) : 1;
 488                                 m_commonFileThreshold = m_rareFileThreshold + 1;
 489                         } else {
 490                                 // More than two, time to do some math.
 491
 492                                 // Lower 25% with the current #define values.
 493                                 int rarecutpoint = (nSourceGroups / RARITY_FACTOR);
 494                                 for (int i = 0; i < rarecutpoint; ++ i) {
 495                                         m_sourcecountlist.pop_front();
 496                                 }
 497                                 m_rareFileThreshold = (m_sourcecountlist.front() > 0) ? (m_sourcecountlist.front() - 1) : 1;
 498
 499                                 // 50% of the non-rare ones, with the curent #define values.
 500                                 int commoncutpoint = (nSourceGroups - rarecutpoint) / NORMALITY_FACTOR;
 501                                 for (int i = 0; i < commoncutpoint; ++ i) {
 502                                         m_sourcecountlist.pop_front();
 503                                 }
 504                                 m_commonFileThreshold = (m_sourcecountlist.front() > 0) ? (m_sourcecountlist.front() - 1) : 1;
 505                         }
 506                 } else {
 507                         m_rareFileThreshold = RARE_FILE;
 508                         m_commonFileThreshold = 100;
 509                 }
 510
 511                 m_datarate += cur_datarate;
 512
 513                 if (m_udcounter == 5) {
 514                         if (theApp->serverconnect->IsUDPSocketAvailable()) {
 515                                 if( (::GetTickCount() - m_lastudpstattime) > UDPSERVERSTATTIME) {
 516                                         m_lastudpstattime = ::GetTickCount();
 517
 518                                         CMutexUnlocker unlocker(m_mutex);
 519                                         theApp->serverlist->ServerStats();
 520                                 }
 521                         }
 522                 }
 523
 524                 if (m_udcounter == 10) {
 525                         m_udcounter = 0;
 526                         if (theApp->serverconnect->IsUDPSocketAvailable()) {
 527                                 if ( (::GetTickCount() - m_lastudpsearchtime) > UDPSERVERREASKTIME) {
 528                                         SendNextUDPPacket();
 529                                 }
 530                         }
 531                 }
 532
 533                 if ( (::GetTickCount() - m_lastsorttime) > 10000 ) {
 534                         DoSortByPriority();
 535                 }
 536                 // Check if any paused files can be resumed
 537
 538                 CheckDiskspace(thePrefs::GetTempDir());
 539         }
 540
 541
 542         // Check for new links once per second.
 543         if ((::GetTickCount() - m_nLastED2KLinkCheck) >= 1000) {
 544                 theApp->AddLinksFromFile();
 545                 m_nLastED2KLinkCheck = ::GetTickCount();
 546         }
 547 }
 548
 549
 550 CPartFile* CDownloadQueue::GetFileByID(const CMD4Hash& filehash) const
 551 {
 552         wxMutexLocker lock( m_mutex );
 553
 554         for ( uint16 i = 0; i < m_filelist.size(); ++i ) {
 555                 if ( filehash == m_filelist[i]->GetFileHash()) {
 556                         return m_filelist[ i ];
 557                 }
 558         }
 559         // Check completed too so we can execute remote commands (like change cat) on them
 560         for (FileList::const_iterator it = m_completedDownloads.begin(); it != m_completedDownloads.end(); ++it) {
 561                 if ( filehash == (*it)->GetFileHash()) {
 562                         return *it;
 563                 }
 564         }
 565
 566         return NULL;
 567 }
 568
 569
 570 CPartFile* CDownloadQueue::GetFileByIndex(unsigned int index)  const
 571 {
 572         wxMutexLocker lock( m_mutex );
 573
 574         if ( index < m_filelist.size() ) {
 575                 return m_filelist[ index ];
 576         }
 577
 578         wxFAIL;
 579         return NULL;
 580 }
 581
 582
 583 bool CDownloadQueue::IsPartFile(const CKnownFile* file) const
 584 {
 585         wxMutexLocker lock(m_mutex);
 586
 587         for (uint16 i = 0; i < m_filelist.size(); ++i) {
 588                 if (file == m_filelist[i]) {
 589                         return true;
 590                 }
 591         }
 592
 593         return false;
 594 }
 595
 596
 597 void CDownloadQueue::OnConnectionState(bool bConnected)
 598 {
 599         wxMutexLocker lock(m_mutex);
 600
 601         for (uint16 i = 0; i < m_filelist.size(); ++i) {
 602                 if (    m_filelist[i]->GetStatus() == PS_READY ||
 603                         m_filelist[i]->GetStatus() == PS_EMPTY) {
 604                         m_filelist[i]->SetActive(bConnected);
 605                 }
 606         }
 607 }
 608
 609
 610 void CDownloadQueue::CheckAndAddSource(CPartFile* sender, CUpDownClient* source)
 611 {
 612         // if we block loopbacks at this point it should prevent us from connecting to ourself
 613         if ( source->HasValidHash() ) {
 614                 if ( source->GetUserHash() == thePrefs::GetUserHash() ) {
 615                         AddDebugLogLineN( logDownloadQueue, wxT("Tried to add source with matching hash to your own.") );
 616                         source->Safe_Delete();
 617                         return;
 618                 }
 619         }
 620
 621         if (sender->IsStopped()) {
 622                 source->Safe_Delete();
 623                 return;
 624         }
 625
 626         // Filter sources which are known to be dead/useless
 627         if ( theApp->clientlist->IsDeadSource( source ) || sender->IsDeadSource(source) ) {
 628                 source->Safe_Delete();
 629                 return;
 630         }
 631
 632         // Filter sources which are incompatible with our encryption setting (one requires it, and the other one doesn't supports it)
 633         if ( (source->RequiresCryptLayer() && (!thePrefs::IsClientCryptLayerSupported() || !source->HasValidHash())) || (thePrefs::IsClientCryptLayerRequired() && (!source->SupportsCryptLayer() || !source->HasValidHash()))) {
 634                 source->Safe_Delete();
 635                 return;
 636         }
 637
 638         // Find all clients with the same hash
 639         if ( source->HasValidHash() ) {
 640                 CClientList::SourceList found = theApp->clientlist->GetClientsByHash( source->GetUserHash() );
 641
 642                 CClientList::SourceList::iterator it = found.begin();
 643                 for ( ; it != found.end(); ++it ) {
 644                         CKnownFile* file = it->GetRequestFile();
 645
 646                         // Only check files on the download-queue
 647                         if ( file ) {
 648                                 // Is the found source queued for something else?
 649                                 if (  file != sender ) {
 650                                         // Try to add a request for the other file
 651                                         if ( it->GetClient()->AddRequestForAnotherFile(sender)) {
 652                                                 // Add it to downloadlistctrl
 653                                                 Notify_SourceCtrlAddSource(sender, *it, A4AF_SOURCE);
 654                                         }
 655                                 }
 656
 657                                 source->Safe_Delete();
 658                                 return;
 659                         }
 660                 }
 661         }
 662
 663
 664
 665         // Our new source is real new but maybe it is already uploading to us?
 666         // If yes the known client will be attached to the var "source" and the old
 667         // source-client will be deleted. However, if the request file of the known
 668         // source is NULL, then we have to treat it almost like a new source and if
 669         // it isn't NULL and not "sender", then we shouldn't move it, but rather add
 670         // a request for the new file.
 671         ESourceFrom nSourceFrom = source->GetSourceFrom();
 672         if ( theApp->clientlist->AttachToAlreadyKnown(&source, 0) ) {
 673                 // Already queued for another file?
 674                 if ( source->GetRequestFile() ) {
 675                         // If we're already queued for the right file, then there's nothing to do
 676                         if ( sender != source->GetRequestFile() ) {
 677                                 // Add the new file to the request list
 678                                 source->AddRequestForAnotherFile( sender );
 679                         }
 680                 } else {
 681                         // Source was known, but reqfile NULL.
 682                         source->SetRequestFile( sender );
 683                         source->SetSourceFrom(nSourceFrom);
 684                         sender->AddSource( source );
 685                         if ( source->GetFileRating() || !source->GetFileComment().IsEmpty() ) {
 686                                 sender->UpdateFileRatingCommentAvail();
 687                         }
 688
 689                         Notify_SourceCtrlAddSource(sender, CCLIENTREF(source, wxT("CDownloadQueue::CheckAndAddSource Notify_SourceCtrlAddSource 1")), UNAVAILABLE_SOURCE);
 690                 }
 691         } else {
 692                 // Unknown client, add it to the clients list
 693                 source->SetRequestFile( sender );
 694
 695                 theApp->clientlist->AddClient(source);
 696
 697                 sender->AddSource( source );
 698                 if ( source->GetFileRating() || !source->GetFileComment().IsEmpty() ) {
 699                         sender->UpdateFileRatingCommentAvail();
 700                 }
 701
 702                 Notify_SourceCtrlAddSource(sender, CCLIENTREF(source, wxT("CDownloadQueue::CheckAndAddSource Notify_SourceCtrlAddSource 2")), UNAVAILABLE_SOURCE);
 703         }
 704 }
 705
 706
 707 void CDownloadQueue::CheckAndAddKnownSource(CPartFile* sender,CUpDownClient* source)
 708 {
 709         // Kad reviewed
 710
 711         if (sender->IsStopped()) {
 712                 return;
 713         }
 714
 715         // Filter sources which are known to be dead/useless
 716         if ( sender->IsDeadSource(source) ) {
 717                 return;
 718         }
 719
 720         // "Filter LAN IPs" -- this may be needed here in case we are connected to the internet and are also connected
 721         // to a LAN and some client from within the LAN connected to us. Though this situation may be supported in future
 722         // by adding that client to the source list and filtering that client's LAN IP when sending sources to
 723         // a client within the internet.
 724         //
 725         // "IPfilter" is not needed here, because that "known" client was already IPfiltered when receiving OP_HELLO.
 726         if (!source->HasLowID()) {
 727                 uint32 nClientIP = wxUINT32_SWAP_ALWAYS(source->GetUserIDHybrid());
 728                 if (!IsGoodIP(nClientIP, thePrefs::FilterLanIPs())) { // check for 0-IP, localhost and LAN addresses
 729                         AddDebugLogLineN(logIPFilter, wxT("Ignored already known source with IP=%s") + Uint32toStringIP(nClientIP));
 730                         return;
 731                 }
 732         }
 733
 734         // Filter sources which are incompatible with our encryption setting (one requires it, and the other one doesn't supports it)
 735         if ( (source->RequiresCryptLayer() && (!thePrefs::IsClientCryptLayerSupported() || !source->HasValidHash()))
 736                 || (thePrefs::IsClientCryptLayerRequired() && (!source->SupportsCryptLayer() || !source->HasValidHash()))) {
 737                 return;
 738         }
 739
 740         CPartFile* file = source->GetRequestFile();
 741
 742         // Check if the file is already queued for something else
 743         if ( file ) {
 744                 if ( file != sender ) {
 745                         if ( source->AddRequestForAnotherFile( sender ) ) {
 746                                 Notify_SourceCtrlAddSource( sender, CCLIENTREF(source, wxT("CDownloadQueue::CheckAndAddKnownSource Notify_SourceCtrlAddSource 1")), A4AF_SOURCE );
 747                         }
 748                 }
 749         } else {
 750                 source->SetRequestFile( sender );
 751
 752                 if ( source->GetFileRating() || !source->GetFileComment().IsEmpty() ) {
 753                         sender->UpdateFileRatingCommentAvail();
 754                 }
 755
 756                 source->SetSourceFrom(SF_PASSIVE);
 757                 sender->AddSource( source );
 758                 Notify_SourceCtrlAddSource( sender, CCLIENTREF(source, wxT("CDownloadQueue::CheckAndAddKnownSource Notify_SourceCtrlAddSource 2")), UNAVAILABLE_SOURCE);
 759         }
 760 }
 761
 762
 763 bool CDownloadQueue::RemoveSource(CUpDownClient* toremove, bool WXUNUSED(updatewindow), bool bDoStatsUpdate)
 764 {
 765         bool removed = false;
 766         toremove->DeleteAllFileRequests();
 767
 768         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
 769                 CPartFile* cur_file = GetFileByIndex( i );
 770
 771                 // Remove from source-list
 772                 if ( cur_file->DelSource( toremove ) ) {
 773
 774                         // Remove from sourcelist widget
 775                         Notify_SourceCtrlRemoveSource(toremove->ECID(), cur_file);
 776
 777                         cur_file->RemoveDownloadingSource(toremove);
 778                         removed = true;
 779                         if ( bDoStatsUpdate ) {
 780                                 cur_file->UpdatePartsInfo();
 781                         }
 782                 }
 783
 784                 // Remove from A4AF-list
 785                 cur_file->RemoveA4AFSource( toremove );
 786         }
 787
 788
 789         if ( !toremove->GetFileComment().IsEmpty() || toremove->GetFileRating()>0) {
 790                 toremove->GetRequestFile()->UpdateFileRatingCommentAvail();
 791         }
 792
 793         toremove->SetRequestFile( NULL );
 794         toremove->SetDownloadState(DS_NONE);
 795         toremove->ResetFileStatusInfo();
 796
 797         return removed;
 798 }
 799
 800
 801 void CDownloadQueue::RemoveFile(CPartFile* file, bool keepAsCompleted)
 802 {
 803         RemoveLocalServerRequest( file );
 804
 805         NotifyObservers( EventType( EventType::REMOVED, file ) );
 806
 807         wxMutexLocker lock( m_mutex );
 808
 809         EraseValue( m_filelist, file );
 810
 811         if (keepAsCompleted) {
 812                 m_completedDownloads.push_back(file);
 813         }
 814 }
 815
 816
 817 void CDownloadQueue::ClearCompleted(const ListOfUInts32 & ecids)
 818 {
 819         for (ListOfUInts32::const_iterator it1 = ecids.begin(); it1 != ecids.end(); ++it1) {
 820                 uint32 ecid = *it1;
 821                 for (FileList::iterator it = m_completedDownloads.begin(); it != m_completedDownloads.end(); ++it) {
 822                         CPartFile * file = *it;
 823                         if (file->ECID() == ecid) {
 824                                 m_completedDownloads.erase(it);
 825                                 // get a new EC ID so it is resent and cleared in remote gui
 826                                 file->RenewECID();
 827                                 Notify_DownloadCtrlRemoveFile(file);
 828                                 break;
 829                         }
 830                 }
 831         }
 832 }
 833
 834
 835 CUpDownClient* CDownloadQueue::GetDownloadClientByIP_UDP(uint32 dwIP, uint16 nUDPPort) const
 836 {
 837         wxMutexLocker lock( m_mutex );
 838
 839         for ( unsigned int i = 0; i < m_filelist.size(); i++ ) {
 840                 const CKnownFile::SourceSet& set = m_filelist[i]->GetSourceList();
 841
 842                 for ( CKnownFile::SourceSet::const_iterator it = set.begin(); it != set.end(); ++it ) {
 843                         if ( it->GetIP() == dwIP && it->GetUDPPort() == nUDPPort ) {
 844                                 return it->GetClient();
 845                         }
 846                 }
 847         }
 848         return NULL;
 849 }
 850
 851
 852 /**
 853  * Checks if the specified server is the one we are connected to.
 854  */
 855 static bool IsConnectedServer(const CServer* server)
 856 {
 857         if (server && theApp->serverconnect->GetCurrentServer()) {
 858                 wxString srvAddr = theApp->serverconnect->GetCurrentServer()->GetAddress();
 859                 uint16 srvPort = theApp->serverconnect->GetCurrentServer()->GetPort();
 860
 861                 return server->GetAddress() == srvAddr && server->GetPort() == srvPort;
 862         }
 863
 864         return false;
 865 }
 866
 867
 868 bool CDownloadQueue::SendNextUDPPacket()
 869 {
 870         if ( m_filelist.empty() || !theApp->serverconnect->IsUDPSocketAvailable() || !theApp->IsConnectedED2K()) {
 871                 return false;
 872         }
 873
 874         // Start monitoring the server and the files list
 875         if ( !m_queueServers.IsActive() ) {
 876                 AddObserver( &m_queueFiles );
 877
 878                 theApp->serverlist->AddObserver( &m_queueServers );
 879         }
 880
 881
 882         bool packetSent = false;
 883         while ( !packetSent ) {
 884                 // Get max files ids per packet for current server
 885                 int filesAllowed = GetMaxFilesPerUDPServerPacket();
 886
 887                 if (filesAllowed < 1 || !m_udpserver || IsConnectedServer(m_udpserver)) {
 888                         // Select the next server to ask, must not be the connected server
 889                         do {
 890                                 m_udpserver = m_queueServers.GetNext();
 891                         } while (IsConnectedServer(m_udpserver));
 892
 893                         m_cRequestsSentToServer = 0;
 894                         filesAllowed = GetMaxFilesPerUDPServerPacket();
 895                 }
 896
 897
 898                 // Check if we have asked all servers, in which case we are done
 899                 if (m_udpserver == NULL) {
 900                         DoStopUDPRequests();
 901
 902                         return false;
 903                 }
 904
 905                 // Memoryfile containing the hash of every file to request
 906                 // 28bytes allocation because 16b + 4b + 8b is the worse case scenario.
 907                 CMemFile hashlist( 28 );
 908
 909                 CPartFile* file = m_queueFiles.GetNext();
 910
 911                 while ( file && filesAllowed ) {
 912                         uint8 status = file->GetStatus();
 913
 914                         if ( ( status == PS_READY || status == PS_EMPTY ) && file->GetSourceCount() < thePrefs::GetMaxSourcePerFileUDP() ) {
 915                                 if (file->IsLargeFile() && !m_udpserver->SupportsLargeFilesUDP()) {
 916                                         AddDebugLogLineN(logDownloadQueue, wxT("UDP Request for sources on a large file ignored: server doesn't support it"));
 917                                 } else {
 918                                         ++m_cRequestsSentToServer;
 919                                         hashlist.WriteHash( file->GetFileHash() );
 920                                         // See the notes on TCP packet
 921                                         if ( m_udpserver->GetUDPFlags() & SRV_UDPFLG_EXT_GETSOURCES2 ) {
 922                                                 if (file->IsLargeFile()) {
 923                                                         wxASSERT(m_udpserver->SupportsLargeFilesUDP());
 924                                                         hashlist.WriteUInt32( 0 );
 925                                                         hashlist.WriteUInt64( file->GetFileSize() );
 926                                                 } else {
 927                                                         hashlist.WriteUInt32( file->GetFileSize() );
 928                                                 }
 929                                         }
 930                                         --filesAllowed;
 931                                 }
 932                         }
 933
 934                         // Avoid skipping a file if we can't send any more currently
 935                         if ( filesAllowed ) {
 936                                 file = m_queueFiles.GetNext();
 937                         }
 938                 }
 939
 940                 // See if we have anything to send
 941                 if ( hashlist.GetLength() ) {
 942                         packetSent = SendGlobGetSourcesUDPPacket(hashlist);
 943                 }
 944
 945                 // Check if we've covered every file
 946                 if ( file == NULL ) {
 947                         // Reset the list of asked files so that the loop will start over
 948                         m_queueFiles.Reset();
 949
 950                         // Unset the server so that the next server will be used
 951                         m_udpserver = NULL;
 952                 }
 953         }
 954
 955         return true;
 956 }
 957
 958
 959 void CDownloadQueue::StopUDPRequests()
 960 {
 961         wxMutexLocker lock( m_mutex );
 962
 963         DoStopUDPRequests();
 964 }
 965
 966
 967 void CDownloadQueue::DoStopUDPRequests()
 968 {
 969         // No need to observe when we wont be using the results
 970         theApp->serverlist->RemoveObserver( &m_queueServers );
 971         RemoveObserver( &m_queueFiles );
 972
 973         m_udpserver = 0;
 974         m_lastudpsearchtime = ::GetTickCount();
 975 }
 976
 977
 978 // Comparison function needed by sort. Returns true if file1 preceeds file2
 979 static bool ComparePartFiles(const CPartFile* file1, const CPartFile* file2) {
 980         if (file1->GetDownPriority() != file2->GetDownPriority()) {
 981                 // To place high-priority files before low priority files we have to
 982                 // invert this test, since PR_LOW is lower than PR_HIGH, and since
 983                 // placing a PR_LOW file before a PR_HIGH file would mean that
 984                 // the PR_LOW file gets sources before the PR_HIGH file ...
 985                 return (file1->GetDownPriority() > file2->GetDownPriority());
 986         } else {
 987                 int sourcesA = file1->GetSourceCount();
 988                 int sourcesB = file2->GetSourceCount();
 989
 990                 int notSourcesA = file1->GetNotCurrentSourcesCount();
 991                 int notSourcesB = file2->GetNotCurrentSourcesCount();
 992
 993                 int cmp = CmpAny( sourcesA - notSourcesA, sourcesB - notSourcesB );
 994
 995                 if ( cmp == 0 ) {
 996                         cmp = CmpAny( notSourcesA, notSourcesB );
 997                 }
 998
 999                 return cmp < 0;
1000         }
1001 }
1002
1003
1004 void CDownloadQueue::DoSortByPriority()
1005 {
1006         m_lastsorttime = ::GetTickCount();
1007         sort( m_filelist.begin(), m_filelist.end(), ComparePartFiles );
1008 }
1009
1010
1011 void CDownloadQueue::ResetLocalServerRequests()
1012 {
1013         wxMutexLocker lock( m_mutex );
1014
1015         m_dwNextTCPSrcReq = 0;
1016         m_localServerReqQueue.clear();
1017
1018         for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
1019                 m_filelist[i]->SetLocalSrcRequestQueued(false);
1020         }
1021 }
1022
1023
1024 void CDownloadQueue::RemoveLocalServerRequest( CPartFile* file )
1025 {
1026         wxMutexLocker lock( m_mutex );
1027
1028         EraseValue( m_localServerReqQueue, file );
1029
1030         file->SetLocalSrcRequestQueued(false);
1031 }
1032
1033
1034 void CDownloadQueue::ProcessLocalRequests()
1035 {
1036         wxMutexLocker lock( m_mutex );
1037
1038         bool bServerSupportsLargeFiles = theApp->serverconnect
1039                                                                         && theApp->serverconnect->GetCurrentServer()
1040                                                                         && theApp->serverconnect->GetCurrentServer()->SupportsLargeFilesTCP();
1041
1042         if ( (!m_localServerReqQueue.empty()) && (m_dwNextTCPSrcReq < ::GetTickCount()) ) {
1043                 CMemFile dataTcpFrame(22);
1044                 const int iMaxFilesPerTcpFrame = 15;
1045                 int iFiles = 0;
1046                 while (!m_localServerReqQueue.empty() && iFiles < iMaxFilesPerTcpFrame) {
1047                         // find the file with the longest waitingtime
1048                         uint32 dwBestWaitTime = 0xFFFFFFFF;
1049
1050                         std::list<CPartFile*>::iterator posNextRequest = m_localServerReqQueue.end();
1051                         std::list<CPartFile*>::iterator it = m_localServerReqQueue.begin();
1052                         while( it != m_localServerReqQueue.end() ) {
1053                                 CPartFile* cur_file = (*it);
1054                                 if (cur_file->GetStatus() == PS_READY || cur_file->GetStatus() == PS_EMPTY) {
1055                                         uint8 nPriority = cur_file->GetDownPriority();
1056                                         if (nPriority > PR_HIGH) {
1057                                                 wxFAIL;
1058                                                 nPriority = PR_HIGH;
1059                                         }
1060
1061                                         if (cur_file->GetLastSearchTime() + (PR_HIGH-nPriority) < dwBestWaitTime ){
1062                                                 dwBestWaitTime = cur_file->GetLastSearchTime() + (PR_HIGH - nPriority);
1063                                                 posNextRequest = it;
1064                                         }
1065
1066                                         ++it;
1067                                 } else {
1068                                         it = m_localServerReqQueue.erase(it);
1069                                         cur_file->SetLocalSrcRequestQueued(false);
1070                                         AddDebugLogLineN(logDownloadQueue,
1071                                                 CFormat(wxT("Local server source request for file '%s' not sent because of status '%s'"))
1072                                                         % cur_file->GetFileName() % cur_file->getPartfileStatus());
1073                                 }
1074                         }
1075
1076                         if (posNextRequest != m_localServerReqQueue.end()) {
1077                                 CPartFile* cur_file = (*posNextRequest);
1078                                 cur_file->SetLocalSrcRequestQueued(false);
1079                                 cur_file->SetLastSearchTime(::GetTickCount());
1080                                 m_localServerReqQueue.erase(posNextRequest);
1081                                 iFiles++;
1082
1083                                 if (!bServerSupportsLargeFiles && cur_file->IsLargeFile()) {
1084                                         AddDebugLogLineN(logDownloadQueue, wxT("TCP Request for sources on a large file ignored: server doesn't support it"));
1085                                 } else {
1086                                         AddDebugLogLineN(logDownloadQueue,
1087                                                 CFormat(wxT("Creating local sources request packet for '%s'")) % cur_file->GetFileName());
1088                                         // create request packet
1089                                         CMemFile data(16 + (cur_file->IsLargeFile() ? 8 : 4));
1090                                         data.WriteHash(cur_file->GetFileHash());
1091                                         // Kry - lugdunum extended protocol on 17.3 to handle filesize properly.
1092                                         // There is no need to check anything, old server ignore the extra 4 bytes.
1093                                         // As of 17.9, servers accept a 0 32-bits size and then a 64bits size
1094                                         if (cur_file->IsLargeFile()) {
1095                                                 wxASSERT(bServerSupportsLargeFiles);
1096                                                 data.WriteUInt32(0);
1097                                                 data.WriteUInt64(cur_file->GetFileSize());
1098                                         } else {
1099                                                 data.WriteUInt32(cur_file->GetFileSize());
1100                                         }
1101                                         uint8 byOpcode = 0;
1102                                         if (thePrefs::IsClientCryptLayerSupported() && theApp->serverconnect->GetCurrentServer() != NULL && theApp->serverconnect->GetCurrentServer()->SupportsGetSourcesObfuscation()) {
1103                                                 byOpcode = OP_GETSOURCES_OBFU;
1104                                         } else {
1105                                                 byOpcode = OP_GETSOURCES;
1106                                         }
1107                                         CPacket packet(data, OP_EDONKEYPROT, byOpcode);
1108                                         dataTcpFrame.Write(packet.GetPacket(), packet.GetRealPacketSize());
1109                                 }
1110                         }
1111                 }
1112
1113                 int iSize = dataTcpFrame.GetLength();
1114                 if (iSize > 0) {
1115                         // create one 'packet' which contains all buffered OP_GETSOURCES ED2K packets to be sent with one TCP frame
1116                         // server credits: (16+4)*regularfiles + (16+4+8)*largefiles +1
1117                         CScopedPtr<CPacket> packet(new CPacket(new uint8_t[iSize], dataTcpFrame.GetLength(), true, false));
1118                         dataTcpFrame.Seek(0, wxFromStart);
1119                         dataTcpFrame.Read(packet->GetPacket(), iSize);
1120                         uint32 size = packet->GetPacketSize();
1121                         theApp->serverconnect->SendPacket(packet.release(), true);      // Deletes `packet'.
1122                         AddDebugLogLineN(logDownloadQueue, wxT("Sent local sources request packet."));
1123                         theStats::AddUpOverheadServer(size);
1124                 }
1125
1126                 // next TCP frame with up to 15 source requests is allowed to be sent in..
1127                 m_dwNextTCPSrcReq = ::GetTickCount() + SEC2MS(iMaxFilesPerTcpFrame*(16+4));
1128         }
1129
1130 }
1131
1132
1133 void CDownloadQueue::SendLocalSrcRequest(CPartFile* sender)
1134 {
1135         wxMutexLocker lock( m_mutex );
1136
1137         m_localServerReqQueue.push_back(sender);
1138 }
1139
1140
1141 void CDownloadQueue::ResetCatParts(uint8 cat)
1142 {
1143         for (FileQueue::iterator it = m_filelist.begin(); it != m_filelist.end(); ++it) {
1144                 CPartFile* file = *it;
1145                 file->RemoveCategory(cat);
1146         }
1147         for (FileList::iterator it = m_completedDownloads.begin(); it != m_completedDownloads.end(); ++it) {
1148                 CPartFile* file = *it;
1149                 file->RemoveCategory(cat);
1150         }
1151 }
1152
1153
1154 void CDownloadQueue::SetCatPrio(uint8 cat, uint8 newprio)
1155 {
1156         for ( uint16 i = 0; i < GetFileCount(); i++ ) {
1157                 CPartFile* file = GetFileByIndex( i );
1158
1159                 if ( !cat || file->GetCategory() == cat ) {
1160                         if ( newprio == PR_AUTO ) {
1161                                 file->SetAutoDownPriority(true);
1162                         } else {
1163                                 file->SetAutoDownPriority(false);
1164                                 file->SetDownPriority(newprio);
1165                         }
1166                 }
1167         }
1168 }
1169
1170
1171 void CDownloadQueue::SetCatStatus(uint8 cat, int newstatus)
1172 {
1173         std::list<CPartFile*> files;
1174
1175         {
1176                 wxMutexLocker lock(m_mutex);
1177
1178                 for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
1179                         if ( m_filelist[i]->CheckShowItemInGivenCat(cat) ) {
1180                                 files.push_back( m_filelist[i] );
1181                         }
1182                 }
1183         }
1184
1185         std::list<CPartFile*>::iterator it = files.begin();
1186
1187         for ( ; it != files.end(); ++it ) {
1188                 switch ( newstatus ) {
1189                         case MP_CANCEL: (*it)->Delete();        break;
1190                         case MP_PAUSE:  (*it)->PauseFile();     break;
1191                         case MP_STOP:   (*it)->StopFile();      break;
1192                         case MP_RESUME: (*it)->ResumeFile();    break;
1193                 }
1194         }
1195 }
1196
1197
1198 uint16 CDownloadQueue::GetDownloadingFileCount() const
1199 {
1200         wxMutexLocker lock( m_mutex );
1201
1202         uint16 count = 0;
1203         for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
1204                 uint8 status = m_filelist[i]->GetStatus();
1205                 if ( status == PS_READY || status == PS_EMPTY ) {
1206                         count++;
1207                 }
1208         }
1209
1210         return count;
1211 }
1212
1213
1214 uint16 CDownloadQueue::GetPausedFileCount() const
1215 {
1216         wxMutexLocker lock( m_mutex );
1217
1218         uint16 count = 0;
1219         for ( uint16 i = 0; i < m_filelist.size(); i++ ) {
1220                 if ( m_filelist[i]->GetStatus() == PS_PAUSED ) {
1221                         count++;
1222                 }
1223         }
1224
1225         return count;
1226 }
1227
1228
1229 void CDownloadQueue::CheckDiskspace( const CPath& path )
1230 {
1231         if ( ::GetTickCount() - m_lastDiskCheck < DISKSPACERECHECKTIME ) {
1232                 return;
1233         }
1234
1235         m_lastDiskCheck = ::GetTickCount();
1236
1237         uint64 min = 0;
1238         // Check if the user has set an explicit limit
1239         if ( thePrefs::IsCheckDiskspaceEnabled() ) {
1240                 min = thePrefs::GetMinFreeDiskSpace();
1241         }
1242
1243         // The very least acceptable diskspace is a single PART
1244         if ( min < PARTSIZE ) {
1245                 min = PARTSIZE;
1246         }
1247
1248         uint64 free = CPath::GetFreeSpaceAt(path);
1249         if (free == static_cast<uint64>(wxInvalidOffset)) {
1250                 return;
1251         } else if (free < min) {
1252                 CUserEvents::ProcessEvent(
1253                         CUserEvents::OutOfDiskSpace,
1254                         wxT("Temporary partition"));
1255         }
1256
1257         for (unsigned int i = 0; i < m_filelist.size(); ++i) {
1258                 CPartFile* file = m_filelist[i];
1259
1260                 switch ( file->GetStatus() ) {
1261                         case PS_ERROR:
1262                         case PS_COMPLETING:
1263                         case PS_COMPLETE:
1264                                 continue;
1265                 }
1266
1267                 if ( free >= min && file->GetInsufficient() ) {
1268                         // We'll try to resume files if there is enough free space
1269                         if ( free - file->GetNeededSpace() > min ) {
1270                                 file->ResumeFile();
1271                         }
1272                 } else if ( free < min && !file->IsPaused() ) {
1273                         // No space left, stop the files.
1274                         file->PauseFile( true );
1275                 }
1276         }
1277 }
1278
1279
1280 int CDownloadQueue::GetMaxFilesPerUDPServerPacket() const
1281 {
1282         if ( m_udpserver ) {
1283                 if ( m_udpserver->GetUDPFlags() & SRV_UDPFLG_EXT_GETSOURCES ) {
1284                         // get max. file ids per packet
1285                         if ( m_cRequestsSentToServer < MAX_REQUESTS_PER_SERVER ) {
1286                                 return std::min(
1287                                         MAX_FILES_PER_UDP_PACKET,
1288                                         MAX_REQUESTS_PER_SERVER - m_cRequestsSentToServer
1289                                 );
1290                         }
1291                 } else if ( m_cRequestsSentToServer < MAX_REQUESTS_PER_SERVER ) {
1292                         return 1;
1293                 }
1294         }
1295
1296         return 0;
1297 }
1298
1299
1300 bool CDownloadQueue::SendGlobGetSourcesUDPPacket(CMemFile& data)
1301 {
1302         if (!m_udpserver) {
1303                 return false;
1304         }
1305
1306         CPacket packet(data, OP_EDONKEYPROT, ((m_udpserver->GetUDPFlags() & SRV_UDPFLG_EXT_GETSOURCES2) ? OP_GLOBGETSOURCES2 : OP_GLOBGETSOURCES));
1307
1308         theStats::AddUpOverheadServer(packet.GetPacketSize());
1309         theApp->serverconnect->SendUDPPacket(&packet,m_udpserver,false);
1310
1311         return true;
1312 }
1313
1314
1315 void CDownloadQueue::AddToResolve(const CMD4Hash& fileid, const wxString& pszHostname, uint16 port, const wxString& hash, uint8 cryptoptions)
1316 {
1317         // double checking
1318         if ( !GetFileByID(fileid) ) {
1319                 return;
1320         }
1321
1322         wxMutexLocker lock( m_mutex );
1323
1324         Hostname_Entry entry = { fileid, pszHostname, port, hash, cryptoptions };
1325         m_toresolve.push_front(entry);
1326
1327         // Check if there are other DNS lookups on queue
1328         if (m_toresolve.size() == 1) {
1329                 // Check if it is a simple dot address
1330                 uint32 ip = StringIPtoUint32(pszHostname);
1331
1332                 if (ip) {
1333                         OnHostnameResolved(ip);
1334                 } else {
1335                         CAsyncDNS* dns = new CAsyncDNS(pszHostname, DNS_SOURCE, theApp);
1336
1337                         if ((dns->Create() != wxTHREAD_NO_ERROR) || (dns->Run() != wxTHREAD_NO_ERROR)) {
1338                                 dns->Delete();
1339                                 m_toresolve.pop_front();
1340                         }
1341                 }
1342         }
1343 }
1344
1345
1346 void CDownloadQueue::OnHostnameResolved(uint32 ip)
1347 {
1348         wxMutexLocker lock( m_mutex );
1349
1350         wxASSERT( m_toresolve.size() );
1351
1352         Hostname_Entry resolved = m_toresolve.front();
1353         m_toresolve.pop_front();
1354
1355         if ( ip ) {
1356                 CPartFile* file = GetFileByID( resolved.fileid );
1357                 if ( file ) {
1358                         CMemFile sources(1+4+2);
1359                         sources.WriteUInt8(1); // No. Sources
1360                         sources.WriteUInt32(ip);
1361                         sources.WriteUInt16(resolved.port);
1362                         sources.WriteUInt8(resolved.cryptoptions);
1363                         if (resolved.cryptoptions & 0x80) {
1364                                 wxASSERT(!resolved.hash.IsEmpty());
1365                                 CMD4Hash sourcehash;
1366                                 sourcehash.Decode(resolved.hash);
1367                                 sources.WriteHash(sourcehash);
1368                         }
1369                         sources.Seek(0,wxFromStart);
1370
1371                         file->AddSources(sources, 0, 0, SF_LINK, true);
1372                 }
1373         }
1374
1375         while (!m_toresolve.empty()) {
1376                 Hostname_Entry entry = m_toresolve.front();
1377
1378                 // Check if it is a simple dot address
1379                 uint32 tmpIP = StringIPtoUint32(entry.strHostname);
1380
1381                 if (tmpIP) {
1382                         OnHostnameResolved(tmpIP);
1383                 } else {
1384                         CAsyncDNS* dns = new CAsyncDNS(entry.strHostname, DNS_SOURCE, theApp);
1385
1386                         if ((dns->Create() != wxTHREAD_NO_ERROR) || (dns->Run() != wxTHREAD_NO_ERROR)) {
1387                                 dns->Delete();
1388                                 m_toresolve.pop_front();
1389                         } else {
1390                                 break;
1391                         }
1392                 }
1393         }
1394 }
1395
1396
1397 bool CDownloadQueue::AddLink( const wxString& link, uint8 category )
1398 {
1399         wxString uri(link);
1400
1401         if (link.compare(0, 7, wxT("magnet:")) == 0) {
1402                 uri = CMagnetED2KConverter(link);
1403                 if (uri.empty()) {
1404                         AddLogLineC(CFormat(_("Cannot convert magnet link to eD2k: %s")) % link);
1405                         return false;
1406                 }
1407         }
1408
1409         if (uri.compare(0, 7, wxT("ed2k://")) == 0) {
1410                 return AddED2KLink(uri, category);
1411         } else {
1412                 AddLogLineC(CFormat(_("Unknown protocol of link: %s")) % link);
1413                 return false;
1414         }
1415 }
1416
1417
1418 bool CDownloadQueue::AddED2KLink( const wxString& link, uint8 category )
1419 {
1420         wxASSERT( !link.IsEmpty() );
1421         wxString URI = link;
1422
1423         // Need the links to end with /, otherwise CreateLinkFromUrl crashes us.
1424         if ( URI.Last() != wxT('/') ) {
1425                 URI += wxT("/");
1426         }
1427
1428         try {
1429                 CScopedPtr<CED2KLink> uri(CED2KLink::CreateLinkFromUrl(URI));
1430
1431                 return AddED2KLink( uri.get(), category );
1432         } catch ( const wxString& err ) {
1433                 AddLogLineC(CFormat( _("Invalid eD2k link! ERROR: %s")) % err);
1434         }
1435
1436         return false;
1437 }
1438
1439
1440 bool CDownloadQueue::AddED2KLink( const CED2KLink* link, uint8 category )
1441 {
1442         switch ( link->GetKind() ) {
1443                 case CED2KLink::kFile:
1444                         return AddED2KLink( dynamic_cast<const CED2KFileLink*>( link ), category );
1445
1446                 case CED2KLink::kServer:
1447                         return AddED2KLink( dynamic_cast<const CED2KServerLink*>( link ) );
1448
1449                 case CED2KLink::kServerList:
1450                         return AddED2KLink( dynamic_cast<const CED2KServerListLink*>( link ) );
1451
1452                 default:
1453                         return false;
1454         }
1455 }
1456
1457
1458
1459 bool CDownloadQueue::AddED2KLink( const CED2KFileLink* link, uint8 category )
1460 {
1461         CPartFile* file = NULL;
1462         if (IsFileExisting(link->GetHashKey())) {
1463                 // Must be a shared file if we are to add hashes or sources
1464                 if ((file = GetFileByID(link->GetHashKey())) == NULL) {
1465                         return false;
1466                 }
1467         } else {
1468                 if (link->GetSize() > OLD_MAX_FILE_SIZE) {
1469                         if (!PlatformSpecific::CanFSHandleLargeFiles(thePrefs::GetTempDir())) {
1470                                 AddLogLineC(_("Filesystem for Temp directory cannot handle large files."));
1471                                 return false;
1472                         } else if (!PlatformSpecific::CanFSHandleLargeFiles(theApp->glob_prefs->GetCatPath(category))) {
1473                                 AddLogLineC(_("Filesystem for Incoming directory cannot handle large files."));
1474                                 return false;
1475                         }
1476                 }
1477
1478                 file = new CPartFile(link);
1479
1480                 if (file->GetStatus() == PS_ERROR) {
1481                         delete file;
1482                         return false;
1483                 }
1484
1485                 AddDownload(file, thePrefs::AddNewFilesPaused(), category);
1486         }
1487
1488         if (link->HasValidAICHHash()) {
1489                 CAICHHashSet* hashset = file->GetAICHHashset();
1490
1491                 if (!hashset->HasValidMasterHash() || (hashset->GetMasterHash() != link->GetAICHHash())) {
1492                         hashset->SetMasterHash(link->GetAICHHash(), AICH_VERIFIED);
1493                         hashset->FreeHashSet();
1494                 }
1495         }
1496
1497         const CED2KFileLink::CED2KLinkSourceList& list = link->m_sources;
1498         CED2KFileLink::CED2KLinkSourceList::const_iterator it = list.begin();
1499         for (; it != list.end(); ++it) {
1500                 AddToResolve(link->GetHashKey(), it->addr, it->port, it->hash, it->cryptoptions);
1501         }
1502
1503         return true;
1504 }
1505
1506
1507 bool CDownloadQueue::AddED2KLink( const CED2KServerLink* link )
1508 {
1509         CServer *server = new CServer( link->GetPort(), Uint32toStringIP( link->GetIP() ) );
1510
1511         server->SetListName( Uint32toStringIP( link->GetIP() ) );
1512
1513         theApp->serverlist->AddServer(server);
1514
1515         Notify_ServerAdd(server);
1516
1517         return true;
1518 }
1519
1520
1521 bool CDownloadQueue::AddED2KLink( const CED2KServerListLink* link )
1522 {
1523         theApp->serverlist->UpdateServerMetFromURL( link->GetAddress() );
1524
1525         return true;
1526 }
1527
1528
1529 void CDownloadQueue::ObserverAdded( ObserverType* o )
1530 {
1531         CObservableQueue<CPartFile*>::ObserverAdded( o );
1532
1533         EventType::ValueList list;
1534
1535         {
1536                 wxMutexLocker lock(m_mutex);
1537                 list.reserve( m_filelist.size() );
1538                 list.insert( list.begin(), m_filelist.begin(), m_filelist.end() );
1539         }
1540
1541         NotifyObservers( EventType( EventType::INITIAL, &list ), o );
1542 }
1543
1544 void CDownloadQueue::KademliaSearchFile(uint32_t searchID, const Kademlia::CUInt128* pcontactID, const Kademlia::CUInt128* pbuddyID, uint8_t type, uint32_t ip, uint16_t tcp, uint16_t udp, uint32_t buddyip, uint16_t buddyport, uint8_t byCryptOptions)
1545 {
1546         AddDebugLogLineN(logKadSearch, CFormat(wxT("Search result sources (type %i)")) % type);
1547
1548         //Safety measure to make sure we are looking for these sources
1549         CPartFile* temp = GetFileByKadFileSearchID(searchID);
1550         if( !temp ) {
1551                 AddDebugLogLineN(logKadSearch, wxT("This is not the file we're looking for..."));
1552                 return;
1553         }
1554
1555         //Do we need more sources?
1556         if(!(!temp->IsStopped() && thePrefs::GetMaxSourcePerFile() > temp->GetSourceCount())) {
1557                 AddDebugLogLineN(logKadSearch, wxT("No more sources needed for this file"));
1558                 return;
1559         }
1560
1561         uint32_t ED2KID = wxUINT32_SWAP_ALWAYS(ip);
1562
1563         if (theApp->ipfilter->IsFiltered(ED2KID)) {
1564                 AddDebugLogLineN(logKadSearch, wxT("Source ip got filtered"));
1565                 AddDebugLogLineN(logIPFilter, CFormat(wxT("IPfiltered source IP=%s received from Kademlia")) % Uint32toStringIP(ED2KID));
1566                 return;
1567         }
1568
1569         if( (ip == Kademlia::CKademlia::GetIPAddress() || ED2KID == theApp->GetED2KID()) && tcp == thePrefs::GetPort()) {
1570                 AddDebugLogLineN(logKadSearch, wxT("Trying to add myself as source, ignore"));
1571                 return;
1572         }
1573
1574         CUpDownClient* ctemp = NULL;
1575         switch (type) {
1576                 case 4:
1577                 case 1: {
1578                         // NonFirewalled users
1579                         if(!tcp) {
1580                                 AddDebugLogLineN(logKadSearch, CFormat(wxT("Ignored source (IP=%s) received from Kademlia, no tcp port received")) % Uint32toStringIP(ip));
1581                                 return;
1582                         }
1583                         if (!IsGoodIP(ED2KID,thePrefs::FilterLanIPs())) {
1584                                 AddDebugLogLineN(logKadSearch, CFormat(wxT("%s got filtered")) % Uint32toStringIP(ED2KID));
1585                                 AddDebugLogLineN(logIPFilter, CFormat(wxT("Ignored source (IP=%s) received from Kademlia, filtered")) % Uint32toStringIP(ED2KID));
1586                                 return;
1587                         }
1588                         ctemp = new CUpDownClient(tcp, ip, 0, 0, temp, false, true);
1589                         ctemp->SetSourceFrom(SF_KADEMLIA);
1590                         // not actually sent or needed for HighID sources
1591                         //ctemp->SetServerIP(serverip);
1592                         //ctemp->SetServerPort(serverport);
1593                         ctemp->SetKadPort(udp);
1594                         uint8_t cID[16];
1595                         pcontactID->ToByteArray(cID);
1596                         ctemp->SetUserHash(CMD4Hash(cID));
1597                         break;
1598                 }
1599                 case 2: {
1600                         // Don't use this type... Some clients will process it wrong..
1601                         break;
1602                 }
1603                 case 5:
1604                 case 3: {
1605                         // This will be a firewalled client connected to Kad only.
1606                         // We set the clientID to 1 as a Kad user only has 1 buddy.
1607                         ctemp = new CUpDownClient(tcp, 1, 0, 0, temp, false, true);
1608                         // The only reason we set the real IP is for when we get a callback
1609                         // from this firewalled source, the compare method will match them.
1610                         ctemp->SetSourceFrom(SF_KADEMLIA);
1611                         ctemp->SetKadPort(udp);
1612                         uint8_t cID[16];
1613                         pcontactID->ToByteArray(cID);
1614                         ctemp->SetUserHash(CMD4Hash(cID));
1615                         pbuddyID->ToByteArray(cID);
1616                         ctemp->SetBuddyID(cID);
1617                         ctemp->SetBuddyIP(buddyip);
1618                         ctemp->SetBuddyPort(buddyport);
1619                         break;
1620                 }
1621                 case 6: {
1622                         // firewalled source which supports direct UDP callback
1623                         // if we are firewalled ourself, the source is useless to us
1624                         if (theApp->IsFirewalled()) {
1625                                 break;
1626                         }
1627
1628                         if ((byCryptOptions & 0x08) == 0){
1629                                 AddDebugLogLineN(logKadSearch, CFormat(wxT("Received Kad source type 6 (direct callback) which has the direct callback flag not set (%s)")) % Uint32toStringIP(ED2KID));
1630                                 break;
1631                         }
1632
1633                         ctemp = new CUpDownClient(tcp, 1, 0, 0, temp, false, true);
1634                         ctemp->SetSourceFrom(SF_KADEMLIA);
1635                         ctemp->SetKadPort(udp);
1636                         ctemp->SetIP(ED2KID); // need to set the IP address, which cannot be used for TCP but for UDP
1637                         uint8_t cID[16];
1638                         pcontactID->ToByteArray(cID);
1639                         ctemp->SetUserHash(CMD4Hash(cID));
1640                 }
1641         }
1642
1643         if (ctemp) {
1644                 // add encryption settings
1645                 ctemp->SetConnectOptions(byCryptOptions);
1646
1647                 AddDebugLogLineN(logKadSearch, CFormat(wxT("Happily adding a source (%s) type %d")) % Uint32_16toStringIP_Port(ED2KID, ctemp->GetUserPort()) % type);
1648                 CheckAndAddSource(temp, ctemp);
1649         }
1650 }
1651
1652 CPartFile* CDownloadQueue::GetFileByKadFileSearchID(uint32 id) const
1653 {
1654         wxMutexLocker lock( m_mutex );
1655
1656         for ( uint16 i = 0; i < m_filelist.size(); ++i ) {
1657                 if ( id == m_filelist[i]->GetKadFileSearchID()) {
1658                         return m_filelist[ i ];
1659                 }
1660         }
1661
1662         return NULL;
1663 }
1664
1665 bool CDownloadQueue::DoKademliaFileRequest()
1666 {
1667         return ((::GetTickCount() - lastkademliafilerequest) > KADEMLIAASKTIME);
1668 }
1669 // File_checked_for_headers