src/SharedFileList.cpp

   1 //
   2 // This file is part of the aMule Project.
   3 //
   4 // Copyright (c) 2003-2008 aMule Team ( admin@amule.org / http://www.amule.org )
   5 // Copyright (c) 2002 Merkur ( devs@emule-project.net / http://www.emule-project.net )
   6 //
   7 // Any parts of this program derived from the xMule, lMule or eMule project,
   8 // or contributed by third-party developers are copyrighted by their
   9 // respective authors.
  10 //
  11 // This program is free software; you can redistribute it and/or modify
  12 // it under the terms of the GNU General Public License as published by
  13 // the Free Software Foundation; either version 2 of the License, or
  14 // (at your option) any later version.
  15 //
  16 // This program is distributed in the hope that it will be useful,
  17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 // GNU General Public License for more details.
  20 //
  21 // You should have received a copy of the GNU General Public License
  22 // along with this program; if not, write to the Free Software
  23 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
  24 //
  25
  26 #include "SharedFileList.h"     // Interface declarations  // Do_not_auto_remove
  27
  28 #include <protocol/Protocols.h>
  29 #include <protocol/ed2k/ClientSoftware.h>
  30 #include <protocol/kad/Constants.h>
  31 #include <tags/FileTags.h>
  32
  33 #include <wx/utils.h>
  34
  35 #include "Packet.h"             // Needed for CPacket
  36 #include "MemFile.h"            // Needed for CMemFile
  37 #include "ServerConnect.h"      // Needed for CServerConnect
  38 #include "KnownFileList.h"      // Needed for CKnownFileList
  39 #include "ThreadTasks.h"        // Needed for CThreadScheduler and CHasherTask
  40 #include "Preferences.h"        // Needed for thePrefs
  41 #include "DownloadQueue.h"      // Needed for CDownloadQueue
  42 #include "amule.h"              // Needed for theApp
  43 #include "PartFile.h"           // Needed for PartFile
  44 #include "Server.h"             // Needed for CServer
  45 #include "updownclient.h"
  46 #include "Statistics.h"         // Needed for theStats
  47 #include "Logger.h"
  48 #include <common/Format.h>
  49 #include <common/FileFunctions.h>
  50 #include "GuiEvents.h"          // Needed for Notify_*
  51 #include "SHAHashSet.h"         // Needed for CAICHHash
  52
  53
  54 #include "kademlia/kademlia/Kademlia.h"
  55 #include "kademlia/kademlia/Search.h"
  56 #include "ClientList.h"
  57
  58 typedef std::deque<CKnownFile*> KnownFileArray;
  59
  60 ///////////////////////////////////////////////////////////////////////////////
  61 // CPublishKeyword
  62
  63 class CPublishKeyword
  64 {
  65 public:
  66         CPublishKeyword(const wxString& rstrKeyword)
  67         {
  68                 m_strKeyword = rstrKeyword;
  69                 // min. keyword char is allowed to be < 3 in some cases (see also 'CSearchManager::getWords')
  70                 //ASSERT( rstrKeyword.GetLength() >= 3 );
  71                 wxASSERT( !rstrKeyword.IsEmpty() );
  72                 KadGetKeywordHash(rstrKeyword, &m_nKadID);
  73                 SetNextPublishTime(0);
  74                 SetPublishedCount(0);
  75         }
  76
  77         const Kademlia::CUInt128& GetKadID() const { return m_nKadID; }
  78         const wxString& GetKeyword() const { return m_strKeyword; }
  79         int GetRefCount() const { return m_aFiles.size(); }
  80         const KnownFileArray& GetReferences() const { return m_aFiles; }
  81
  82         uint32 GetNextPublishTime() const { return m_tNextPublishTime; }
  83         void SetNextPublishTime(uint32 tNextPublishTime) { m_tNextPublishTime = tNextPublishTime; }
  84
  85         uint32 GetPublishedCount() const { return m_uPublishedCount; }
  86         void SetPublishedCount(uint32 uPublishedCount) { m_uPublishedCount = uPublishedCount; }
  87         void IncPublishedCount() { m_uPublishedCount++; }
  88
  89         bool AddRef(CKnownFile* pFile) {
  90                 if (std::find(m_aFiles.begin(), m_aFiles.end(), pFile) != m_aFiles.end()) {
  91                         wxASSERT(0);
  92                         return false;
  93                 }
  94                 m_aFiles.push_back(pFile);
  95                 return true;
  96         }
  97
  98         int RemoveRef(CKnownFile* pFile) {
  99                 KnownFileArray::iterator it = std::find(m_aFiles.begin(), m_aFiles.end(), pFile);
 100                 if (it != m_aFiles.end()) {
 101                         m_aFiles.erase(it);
 102                 }
 103                 return m_aFiles.size();
 104         }
 105
 106         void RemoveAllReferences() {
 107                 m_aFiles.clear();
 108         }
 109
 110         void RotateReferences(unsigned iRotateSize) {
 111                 wxCHECK_RET(m_aFiles.size(), wxT("RotateReferences: Rotating empty array"));
 112
 113                 unsigned shift = (iRotateSize % m_aFiles.size());
 114                 std::rotate(m_aFiles.begin(), m_aFiles.begin() + shift, m_aFiles.end());
 115         }
 116
 117 protected:
 118         wxString m_strKeyword;
 119         Kademlia::CUInt128 m_nKadID;
 120         uint32 m_tNextPublishTime;
 121         uint32 m_uPublishedCount;
 122         KnownFileArray m_aFiles;
 123 };
 124
 125
 126 ///////////////////////////////////////////////////////////////////////////////
 127 // CPublishKeywordList
 128
 129 class CPublishKeywordList
 130 {
 131 public:
 132         CPublishKeywordList();
 133         ~CPublishKeywordList();
 134
 135         void AddKeywords(CKnownFile* pFile);
 136         void RemoveKeywords(CKnownFile* pFile);
 137         void RemoveAllKeywords();
 138
 139         void RemoveAllKeywordReferences();
 140         void PurgeUnreferencedKeywords();
 141
 142         int GetCount() const { return m_lstKeywords.size(); }
 143
 144         CPublishKeyword* GetNextKeyword();
 145         void ResetNextKeyword();
 146
 147         uint32 GetNextPublishTime() const { return m_tNextPublishKeywordTime; }
 148         void SetNextPublishTime(uint32 tNextPublishKeywordTime) { m_tNextPublishKeywordTime = tNextPublishKeywordTime; }
 149
 150 protected:
 151         // can't use a CMap - too many disadvantages in processing the 'list'
 152         //CTypedPtrMap<CMapStringToPtr, CString, CPublishKeyword*> m_lstKeywords;
 153         typedef std::list<CPublishKeyword*> CKeyWordList;
 154         CKeyWordList m_lstKeywords;
 155         CKeyWordList::iterator m_posNextKeyword;
 156         uint32 m_tNextPublishKeywordTime;
 157
 158         CPublishKeyword* FindKeyword(const wxString& rstrKeyword, CKeyWordList::iterator* ppos = NULL);
 159 };
 160
 161 CPublishKeywordList::CPublishKeywordList()
 162 {
 163         ResetNextKeyword();
 164         SetNextPublishTime(0);
 165 }
 166
 167 CPublishKeywordList::~CPublishKeywordList()
 168 {
 169         RemoveAllKeywords();
 170 }
 171
 172 CPublishKeyword* CPublishKeywordList::GetNextKeyword()
 173 {
 174         if (m_posNextKeyword == m_lstKeywords.end()) {
 175                 m_posNextKeyword = m_lstKeywords.begin();
 176                 if (m_posNextKeyword == m_lstKeywords.end()) {
 177                         return NULL;
 178                 }
 179         }
 180         return *m_posNextKeyword++;
 181 }
 182
 183 void CPublishKeywordList::ResetNextKeyword()
 184 {
 185         m_posNextKeyword = m_lstKeywords.begin();
 186 }
 187
 188 CPublishKeyword* CPublishKeywordList::FindKeyword(const wxString& rstrKeyword, CKeyWordList::iterator* ppos)
 189 {
 190         CKeyWordList::iterator it = m_lstKeywords.begin();
 191         for (; it != m_lstKeywords.end(); ++it) {
 192                 CPublishKeyword* pPubKw = *it;
 193                 if (pPubKw->GetKeyword() == rstrKeyword) {
 194                         if (ppos) {
 195                                 (*ppos) = it;
 196                         }
 197
 198                         return pPubKw;
 199                 }
 200         }
 201
 202         return NULL;
 203 }
 204
 205 void CPublishKeywordList::AddKeywords(CKnownFile* pFile)
 206 {
 207         const Kademlia::WordList& wordlist = pFile->GetKadKeywords();
 208
 209         Kademlia::WordList::const_iterator it;
 210         for (it = wordlist.begin(); it != wordlist.end(); ++it) {
 211                 const wxString& strKeyword = *it;
 212                 CPublishKeyword* pPubKw = FindKeyword(strKeyword);
 213                 if (pPubKw == NULL) {
 214                         pPubKw = new CPublishKeyword(strKeyword);
 215                         m_lstKeywords.push_back(pPubKw);
 216                         SetNextPublishTime(0);
 217                 }
 218                 pPubKw->AddRef(pFile);
 219         }
 220 }
 221
 222 void CPublishKeywordList::RemoveKeywords(CKnownFile* pFile)
 223 {
 224         const Kademlia::WordList& wordlist = pFile->GetKadKeywords();
 225         Kademlia::WordList::const_iterator it;
 226         for (it = wordlist.begin(); it != wordlist.end(); ++it) {
 227                 const wxString& strKeyword = *it;
 228                 CKeyWordList::iterator pos;
 229                 CPublishKeyword* pPubKw = FindKeyword(strKeyword, &pos);
 230                 if (pPubKw != NULL) {
 231                         if (pPubKw->RemoveRef(pFile) == 0) {
 232                                 if (pos == m_posNextKeyword) {
 233                                         ++m_posNextKeyword;
 234                                 }
 235                                 m_lstKeywords.erase(pos);
 236                                 delete pPubKw;
 237                                 SetNextPublishTime(0);
 238                         }
 239                 }
 240         }
 241 }
 242
 243
 244 void CPublishKeywordList::RemoveAllKeywords()
 245 {
 246         DeleteContents(m_lstKeywords);
 247         ResetNextKeyword();
 248         SetNextPublishTime(0);
 249 }
 250
 251
 252 void CPublishKeywordList::RemoveAllKeywordReferences()
 253 {
 254         CKeyWordList::iterator it = m_lstKeywords.begin();
 255         for (; it != m_lstKeywords.end(); ++it) {
 256                 (*it)->RemoveAllReferences();
 257         }
 258 }
 259
 260
 261 void CPublishKeywordList::PurgeUnreferencedKeywords()
 262 {
 263         CKeyWordList::iterator it = m_lstKeywords.begin();
 264         while (it != m_lstKeywords.end()) {
 265                 CPublishKeyword* pPubKw = *it;
 266                 if (pPubKw->GetRefCount() == 0) {
 267                         if (it == m_posNextKeyword) {
 268                                 ++m_posNextKeyword;
 269                         }
 270                         m_lstKeywords.erase(it++);
 271                         delete pPubKw;
 272                         SetNextPublishTime(0);
 273                 } else {
 274                         ++it;
 275                 }
 276         }
 277 }
 278
 279
 280 CSharedFileList::CSharedFileList(CKnownFileList* in_filelist){
 281         filelist = in_filelist;
 282         reloading = false;
 283         m_lastPublishED2K = 0;
 284         m_lastPublishED2KFlag = true;
 285         /* Kad Stuff */
 286         m_keywords = new CPublishKeywordList;
 287         m_currFileSrc = 0;
 288         m_currFileNotes = 0;
 289         m_lastPublishKadSrc = 0;
 290         m_lastPublishKadNotes = 0;
 291         m_currFileKey = 0;
 292 }
 293
 294
 295 CSharedFileList::~CSharedFileList()
 296 {
 297         delete m_keywords;
 298 }
 299
 300
 301 void CSharedFileList::FindSharedFiles()
 302 {
 303         /* Abort loading if we are shutting down. */
 304         if(theApp->IsOnShutDown()) {
 305                 return;
 306         }
 307
 308         // Clear statistics.
 309         theStats::ClearSharedFilesInfo();
 310
 311         // Reload shareddir.dat
 312         theApp->glob_prefs->ReloadSharedFolders();
 313
 314         {
 315                 wxMutexLocker lock(list_mut);
 316                 m_Files_map.clear();
 317         }
 318
 319         // All part files are automatically shared.
 320         for ( uint32 i = 0; i < theApp->downloadqueue->GetFileCount(); ++i ) {
 321                 CPartFile* file = theApp->downloadqueue->GetFileByIndex( i );
 322
 323                 if ( file->GetStatus(true) == PS_READY ) {
 324                         printf("Adding file %s to shares\n",
 325                                 (const char *)unicode2char(file->GetFullName().GetPrintable()));
 326                         AddFile(file);
 327                 }
 328         }
 329
 330         // Create a list of all shared paths and weed out duplicates.
 331         std::list<CPath> sharedPaths;
 332
 333         // Global incoming dir and all category incoming directories are automatically shared.
 334         sharedPaths.push_back(thePrefs::GetIncomingDir());
 335         for (unsigned int i = 1;i < theApp->glob_prefs->GetCatCount(); ++i) {
 336                 sharedPaths.push_back(theApp->glob_prefs->GetCatPath(i));
 337         }
 338
 339         const thePrefs::PathList& shared = theApp->glob_prefs->shareddir_list;
 340         sharedPaths.insert(sharedPaths.end(), shared.begin(), shared.end());
 341
 342         sharedPaths.sort();
 343         sharedPaths.unique();
 344
 345         unsigned addedFiles = 0;
 346         std::list<CPath>::iterator it = sharedPaths.begin();
 347         for (; it != sharedPaths.end(); ++it) {
 348                 addedFiles += AddFilesFromDirectory(*it);
 349         }
 350
 351         if (addedFiles == 0) {
 352                 AddLogLineM(false, wxString::Format(wxPLURAL("Found %i known shared file", "Found %i known shared files", GetCount()), GetCount()));
 353
 354                 // Make sure the AICH-hashes are up to date.
 355                 CThreadScheduler::AddTask(new CAICHSyncTask());
 356         } else {
 357                 // New files, AICH thread will be run at the end of the hashing thread.
 358                 AddLogLineM(false, wxString::Format(wxPLURAL("Found %i known shared file, %i unknown", "Found %i known shared files, %i unknown", GetCount()), GetCount(), addedFiles));
 359         }
 360 }
 361
 362
 363 // Checks if the dir a is the same as b. If they are, then logs the message and returns true.
 364 bool CheckDirectory(const wxString& a, const CPath& b)
 365 {
 366         if (CPath(a).IsSameDir(b)) {
 367                 AddLogLineM(true, CFormat( _("ERROR! Attempted to share %s") ) % a);
 368
 369                 return true;
 370         }
 371
 372         return false;
 373 }
 374
 375
 376 unsigned CSharedFileList::AddFilesFromDirectory(const CPath& directory)
 377 {
 378         // Do not allow these folders to be shared:
 379         //  - The .aMule folder
 380         //  - The Temp folder
 381         //  - The users home-dir
 382         if (CheckDirectory(wxGetHomeDir(), directory)) {
 383                 return 0;
 384         } else if (CheckDirectory(theApp->ConfigDir, directory)) {
 385                 return 0;
 386         } else if (CheckDirectory(thePrefs::GetTempDir().GetRaw(), directory)) {
 387                 return 0;
 388         }
 389
 390         if (!directory.DirExists()) {
 391                 printf("Shared directory not found, skipping: %s\n",
 392                         (const char *)unicode2char(directory.GetPrintable()));
 393
 394                 return 0;
 395         }
 396
 397         CDirIterator::FileType searchFor = CDirIterator::FileNoHidden;
 398         if (thePrefs::ShareHiddenFiles()) {
 399                  searchFor = CDirIterator::File;
 400         }
 401
 402         unsigned knownFiles = 0;
 403         unsigned addedFiles = 0;
 404
 405         CDirIterator SharedDir(directory);
 406
 407         CPath fname = SharedDir.GetFirstFile(searchFor);
 408         while (fname.IsOk()) {
 409                 CPath fullPath = directory.JoinPaths(fname);
 410
 411                 if (!fullPath.FileExists()) {
 412                         AddDebugLogLineM(false, logKnownFiles,
 413                                 CFormat(wxT("Shared file does not exist (possibly a broken link): %s")) % fullPath);
 414
 415                         fname = SharedDir.GetNextFile();
 416                         continue;
 417                 }
 418
 419                 AddDebugLogLineM(false, logKnownFiles,
 420                         CFormat(wxT("Found shared file: %s")) % fullPath);
 421
 422                 time_t fdate = CPath::GetModificationTime(fullPath);
 423                 sint64 fsize = fullPath.GetFileSize();
 424
 425                 // This will also catch files with too strict permissions.
 426                 if ((fdate == (time_t)-1) || (fsize == wxInvalidOffset)) {
 427                         AddDebugLogLineM(false, logKnownFiles,
 428                                 CFormat(wxT("Failed to retrive modification time or size for '%s', skipping.")) % fullPath);
 429
 430                         fname = SharedDir.GetNextFile();
 431                         continue;
 432                 }
 433
 434
 435                 CKnownFile* toadd = filelist->FindKnownFile(fname, fdate, fsize);
 436                 if (toadd) {
 437                         knownFiles++;
 438                         if (AddFile(toadd)) {
 439                                 AddDebugLogLineM(false, logKnownFiles,
 440                                         CFormat(wxT("Added known file '%s' to shares"))
 441                                                 % fname);
 442
 443                                 toadd->SetFilePath(directory);
 444                         } else {
 445                                 AddDebugLogLineM(false, logKnownFiles,
 446                                         CFormat(wxT("File already shared, skipping: %s"))
 447                                                 % fname);
 448                         }
 449                 } else {
 450                         //not in knownfilelist - start adding thread to hash file
 451                         AddDebugLogLineM(false, logKnownFiles,
 452                                 CFormat(wxT("Hashing new unknown shared file '%s'")) % fname);
 453
 454                         if (CThreadScheduler::AddTask(new CHashingTask(directory, fname))) {
 455                                 addedFiles++;
 456                         }
 457                 }
 458
 459                 fname = SharedDir.GetNextFile();
 460         }
 461
 462         if ((addedFiles == 0) && (knownFiles == 0)) {
 463                 printf("No shareable files found in directory: %s\n",
 464                         (const char *)unicode2char(directory.GetPrintable()));
 465         }
 466
 467         return addedFiles;
 468 }
 469
 470
 471 bool CSharedFileList::AddFile(CKnownFile* pFile)
 472 {
 473         wxASSERT(pFile->GetHashCount() == pFile->GetED2KPartHashCount());
 474
 475         wxMutexLocker lock(list_mut);
 476
 477         CKnownFileMap::value_type entry(pFile->GetFileHash(), pFile);
 478         if (m_Files_map.insert(entry).second) {
 479                 /* Keywords to publish on Kad */
 480                 m_keywords->AddKeywords(pFile);
 481                 theStats::AddSharedFile(pFile->GetFileSize());
 482                 return true;
 483         }
 484         return false;
 485 }
 486
 487
 488 void CSharedFileList::SafeAddKFile(CKnownFile* toadd, bool bOnlyAdd)
 489 {
 490         // TODO: Check if the file is already known - only with another date
 491
 492         if (AddFile(toadd)) {
 493                 Notify_SharedFilesShowFile(toadd);
 494         }
 495
 496         if (!bOnlyAdd && theApp->IsConnectedED2K()) {
 497                 // Publishing of files is not anymore handled here.
 498                 // Instead, the timer does it by itself.
 499                 m_lastPublishED2KFlag = true;
 500         }
 501 }
 502
 503
 504 // removes first occurrence of 'toremove' in 'list'
 505 void CSharedFileList::RemoveFile(CKnownFile* toremove){
 506         Notify_SharedFilesRemoveFile(toremove);
 507         wxMutexLocker lock(list_mut);
 508         if (m_Files_map.erase(toremove->GetFileHash()) > 0) {
 509                 theStats::RemoveSharedFile(toremove->GetFileSize());
 510         }
 511         /* This file keywords must not be published to kad anymore */
 512         m_keywords->RemoveKeywords(toremove);
 513 }
 514
 515
 516 void CSharedFileList::Reload()
 517 {
 518         // Madcat - Disable reloading if reloading already in progress.
 519         // Kry - Fixed to let non-english language users use the 'Reload' button :P
 520         // deltaHF - removed the old ugly button and changed the code to use the new small one
 521         // Kry - bah, let's use a var.
 522         if (!reloading) {
 523                 reloading = true;
 524                 Notify_SharedFilesRemoveAllItems();
 525
 526                 /* All Kad keywords must be removed */
 527                 m_keywords->RemoveAllKeywordReferences();
 528
 529                 FindSharedFiles();
 530
 531                 /* And now the unreferenced keywords must be removed also */
 532                 m_keywords->PurgeUnreferencedKeywords();
 533
 534                 Notify_SharedFilesShowFileList();
 535
 536                 reloading = false;
 537         }
 538 }
 539
 540
 541 const CKnownFile *CSharedFileList::GetFileByIndex(unsigned int index) const
 542 {
 543         wxMutexLocker lock(list_mut);
 544         if ( index >= m_Files_map.size() ) {
 545                 return NULL;
 546         }
 547         CKnownFileMap::const_iterator pos = m_Files_map.begin();
 548         std::advance(pos, index);
 549         return pos->second;
 550 }
 551
 552
 553 CKnownFile*     CSharedFileList::GetFileByID(const CMD4Hash& filehash)
 554 {
 555         wxMutexLocker lock(list_mut);
 556         CKnownFileMap::iterator it = m_Files_map.find(filehash);
 557
 558         if ( it != m_Files_map.end() ) {
 559                 return it->second;
 560         } else {
 561                 return NULL;
 562         }
 563 }
 564
 565 short CSharedFileList::GetFilePriorityByID(const CMD4Hash& filehash)
 566 {
 567         CKnownFile* tocheck = GetFileByID(filehash);
 568         if (tocheck)
 569                 return tocheck->GetUpPriority();
 570         else
 571                 return -10;     // file doesn't exist
 572 }
 573
 574
 575 void CSharedFileList::CopyFileList(std::vector<CKnownFile*>& out_list)
 576 {
 577         wxMutexLocker lock(list_mut);
 578
 579         out_list.reserve(m_Files_map.size());
 580         for (
 581                 CKnownFileMap::iterator it = m_Files_map.begin();
 582                 it != m_Files_map.end();
 583                 ++it
 584                 ) {
 585                 out_list.push_back(it->second);
 586         }
 587 }
 588
 589
 590
 591 void CSharedFileList::UpdateItem(CKnownFile* toupdate)
 592 {
 593         Notify_SharedFilesUpdateItem(toupdate);
 594 }
 595
 596
 597 void CSharedFileList::GetSharedFilesByDirectory(const wxString& directory,
 598                                 CKnownFilePtrList& list)
 599 {
 600         wxMutexLocker lock(list_mut);
 601
 602         const CPath dir = CPath(directory);
 603         for (CKnownFileMap::iterator pos = m_Files_map.begin();
 604              pos != m_Files_map.end(); ++pos ) {
 605                 CKnownFile *cur_file = pos->second;
 606
 607                 if (dir.IsSameDir(cur_file->GetFilePath())) {
 608                         continue;
 609                 }
 610
 611                 list.push_back(cur_file);
 612         }
 613 }
 614
 615 /* ---------------- Network ----------------- */
 616
 617 void CSharedFileList::ClearED2KPublishInfo(){
 618         CKnownFile* cur_file;
 619         m_lastPublishED2KFlag = true;
 620         wxMutexLocker lock(list_mut);
 621         for (CKnownFileMap::iterator pos = m_Files_map.begin(); pos != m_Files_map.end(); ++pos ) {
 622                 cur_file = pos->second;
 623                 cur_file->SetPublishedED2K(false);
 624         }
 625 }
 626
 627 void CSharedFileList::ClearKadSourcePublishInfo()
 628 {
 629         wxMutexLocker lock(list_mut);
 630         CKnownFile* cur_file;
 631         for (CKnownFileMap::iterator pos = m_Files_map.begin(); pos != m_Files_map.end(); ++pos ) {
 632                 cur_file = pos->second;
 633                 cur_file->SetLastPublishTimeKadSrc(0,0);
 634         }
 635 }
 636
 637 void CSharedFileList::RepublishFile(CKnownFile* pFile)
 638 {
 639         CServer* server = theApp->serverconnect->GetCurrentServer();
 640         if (server && (server->GetTCPFlags() & SRV_TCPFLG_COMPRESSION)) {
 641                 m_lastPublishED2KFlag = true;
 642                 pFile->SetPublishedED2K(false); // FIXME: this creates a wrong 'No' for the ed2k shared info in the listview until the file is shared again.
 643         }
 644 }
 645
 646 uint8 GetRealPrio(uint8 in)
 647 {
 648         switch(in) {
 649                 case 4 : return 0;
 650                 case 0 : return 1;
 651                 case 1 : return 2;
 652                 case 2 : return 3;
 653                 case 3 : return 4;
 654         }
 655         return 0;
 656 }
 657
 658 bool SortFunc( const CKnownFile* fileA, const CKnownFile* fileB )
 659 {
 660     return GetRealPrio(fileA->GetUpPriority()) < GetRealPrio(fileB->GetUpPriority());
 661 }
 662
 663 void CSharedFileList::SendListToServer(){
 664         std::vector<CKnownFile*> SortedList;
 665
 666         {
 667                 wxMutexLocker lock(list_mut);
 668
 669                 if (m_Files_map.empty() || !theApp->IsConnectedED2K() ) {
 670                         return;
 671                 }
 672
 673                 // Getting a sorted list of the non-published files.
 674                 SortedList.reserve( m_Files_map.size() );
 675
 676                 CKnownFileMap::iterator it = m_Files_map.begin();
 677                 for ( ; it != m_Files_map.end(); ++it ) {
 678                         if (!it->second->GetPublishedED2K()) {
 679                                 SortedList.push_back( it->second );
 680                         }
 681                 }
 682         }
 683
 684         std::sort( SortedList.begin(), SortedList.end(), SortFunc );
 685
 686         // Limits for the server.
 687
 688         CServer* server = theApp->serverconnect->GetCurrentServer();
 689
 690         uint32 limit = server ? server->GetSoftFiles() : 0;
 691         if( limit == 0 || limit > 200 ) {
 692                 limit = 200;
 693         }
 694
 695         if( (uint32)SortedList.size() < limit ) {
 696                 limit = SortedList.size();
 697                 if (limit == 0) {
 698                         m_lastPublishED2KFlag = false;
 699                         return;
 700                 }
 701         }
 702
 703         CMemFile files;
 704
 705         // Files sent.
 706         files.WriteUInt32(limit);
 707
 708         uint16 count = 0;
 709         // Add to packet
 710         std::vector<CKnownFile*>::iterator sorted_it = SortedList.begin();
 711         for ( ; (sorted_it != SortedList.end()) && (count < limit); ++sorted_it ) {
 712                 CKnownFile* file = *sorted_it;
 713                 if (!file->IsLargeFile() || (server && server->SupportsLargeFilesTCP())) {
 714                         CreateOfferedFilePacket(file, &files, server, NULL);
 715                 }
 716                 file->SetPublishedED2K(true);
 717                 ++count;
 718         }
 719
 720         wxASSERT(count == limit);
 721
 722         CPacket* packet = new CPacket(files, OP_EDONKEYPROT, OP_OFFERFILES);
 723         // compress packet
 724         //   - this kind of data is highly compressable (N * (1 MD4 and at least 3 string meta data tags and 1 integer meta data tag))
 725         //   - the min. amount of data needed for one published file is ~100 bytes
 726         //   - this function is called once when connecting to a server and when a file becomes shareable - so, it's called rarely.
 727         //   - if the compressed size is still >= the original size, we send the uncompressed packet
 728         // therefor we always try to compress the packet
 729         if (server->GetTCPFlags() & SRV_TCPFLG_COMPRESSION){
 730                 packet->PackPacket();
 731         }
 732
 733         theStats::AddUpOverheadServer(packet->GetPacketSize());
 734         theApp->serverconnect->SendPacket(packet,true);
 735 }
 736
 737
 738 void CSharedFileList::CreateOfferedFilePacket(
 739         CKnownFile *cur_file,
 740         CMemFile *files,
 741         CServer *pServer,
 742         CUpDownClient *pClient) {
 743
 744         // This function is used for offering files to the local server and for sending
 745         // shared files to some other client. In each case we send our IP+Port only, if
 746         // we have a HighID.
 747
 748         wxASSERT(!(pClient && pServer));
 749
 750         cur_file->SetPublishedED2K(true);
 751         files->WriteHash(cur_file->GetFileHash());
 752
 753         uint32 nClientID = 0;
 754         uint16 nClientPort = 0;
 755
 756         if (pServer) {
 757                 if (pServer->GetTCPFlags() & SRV_TCPFLG_COMPRESSION) {
 758                         #define FILE_COMPLETE_ID                0xfbfbfbfb
 759                         #define FILE_COMPLETE_PORT      0xfbfb
 760                         #define FILE_INCOMPLETE_ID      0xfcfcfcfc
 761                         #define FILE_INCOMPLETE_PORT    0xfcfc
 762                         // complete   file: ip 251.251.251 (0xfbfbfbfb) port 0xfbfb
 763                         // incomplete file: op 252.252.252 (0xfcfcfcfc) port 0xfcfc
 764                         if (cur_file->GetStatus() == PS_COMPLETE) {
 765                                 nClientID = FILE_COMPLETE_ID;
 766                                 nClientPort = FILE_COMPLETE_PORT;
 767                         } else {
 768                                 nClientID = FILE_INCOMPLETE_ID;
 769                                 nClientPort = FILE_INCOMPLETE_PORT;
 770                         }
 771                 } else {
 772                         if (theApp->IsConnectedED2K() && !::IsLowID(theApp->GetED2KID())){
 773                                 nClientID = theApp->GetID();
 774                                 nClientPort = thePrefs::GetPort();
 775                         }
 776                 }
 777         } else {
 778                 // Do not merge this with the above case - this one
 779                 // also checks Kad status.
 780                 if (theApp->IsConnected() && !theApp->IsFirewalled()) {
 781                         nClientID = theApp->GetID();
 782                         nClientPort = thePrefs::GetPort();
 783                 }
 784         }
 785
 786         files->WriteUInt32(nClientID);
 787         files->WriteUInt16(nClientPort);
 788
 789         TagPtrList tags;
 790
 791         // The printable filename is used because it's destined for another user.
 792         tags.push_back(new CTagString(FT_FILENAME, cur_file->GetFileName().GetPrintable()));
 793
 794         if (pClient && pClient->GetVBTTags()) {
 795                 tags.push_back(new CTagVarInt(FT_FILESIZE, cur_file->GetFileSize()));
 796         } else {
 797                 if (!cur_file->IsLargeFile()){
 798                         tags.push_back(new CTagInt32(FT_FILESIZE, cur_file->GetFileSize()));
 799                 } else {
 800                         // Large file
 801                         // we send 2*32 bit tags to servers, but a real 64 bit tag to other clients.
 802                         if (pServer) {
 803                                 if (!pServer->SupportsLargeFilesTCP()){
 804                                         wxASSERT( false );
 805                                         tags.push_back(new CTagInt32(FT_FILESIZE, 0));
 806                                 }else {
 807                                         tags.push_back(new CTagInt32(FT_FILESIZE, (uint32)cur_file->GetFileSize()));
 808                                         tags.push_back(new CTagInt32(FT_FILESIZE_HI, (uint32)(cur_file->GetFileSize() >> 32)));
 809                                 }
 810                         } else {
 811                                 if (!pClient->SupportsLargeFiles()) {
 812                                         wxASSERT( false );
 813                                         tags.push_back(new CTagInt32(FT_FILESIZE, 0));
 814                                 } else {
 815                                         tags.push_back(new CTagInt64(FT_FILESIZE, cur_file->GetFileSize()));
 816                                 }
 817                         }
 818                 }
 819         }
 820
 821         if (cur_file->GetFileRating()) {
 822                 tags.push_back(new CTagVarInt(FT_FILERATING, cur_file->GetFileRating(), (pClient && pClient->GetVBTTags()) ? 0 : 32));
 823         }
 824
 825         // NOTE: Archives and CD-Images are published+searched with file type "Pro"
 826         bool bAddedFileType = false;
 827         if (pServer && (pServer->GetTCPFlags() & SRV_TCPFLG_TYPETAGINTEGER)) {
 828                 // Send integer file type tags to newer servers
 829                 EED2KFileType eFileType = GetED2KFileTypeSearchID(GetED2KFileTypeID(cur_file->GetFileName()));
 830                 if (eFileType >= ED2KFT_AUDIO && eFileType <= ED2KFT_CDIMAGE) {
 831                         tags.push_back(new CTagInt32(FT_FILETYPE, eFileType));
 832                         bAddedFileType = true;
 833                 }
 834         }
 835         if (!bAddedFileType) {
 836                 // Send string file type tags to:
 837                 //      - newer servers, in case there is no integer type available for the file type (e.g. emulecollection)
 838                 //      - older servers
 839                 //      - all clients
 840                 wxString strED2KFileType(GetED2KFileTypeSearchTerm(GetED2KFileTypeID(cur_file->GetFileName())));
 841                 if (!strED2KFileType.IsEmpty()) {
 842                         tags.push_back(new CTagString(FT_FILETYPE, strED2KFileType));
 843                 }
 844         }
 845
 846         // There, we could add MetaData info, if we ever get to have that.
 847
 848         EUtf8Str eStrEncode;
 849
 850         bool unicode_support =
 851                 // eservers that support UNICODE.
 852                 (pServer && (pServer->GetUnicodeSupport()))
 853                 ||
 854                 // clients that support unicode
 855                 (pClient && pClient->GetUnicodeSupport());
 856         eStrEncode = unicode_support ? utf8strRaw : utf8strNone;
 857
 858         files->WriteUInt32(tags.size());
 859
 860         // Sadly, eMule doesn't use a MISCOPTIONS flag on hello packet for this, so we
 861         // have to identify the support for new tags by version.
 862         bool new_ed2k =
 863                 // eMule client > 0.42f
 864                 (pClient && pClient->IsEmuleClient() && pClient->GetVersion()  >= MAKE_CLIENT_VERSION(0,42,7))
 865                 ||
 866                 // aMule >= 2.0.0rc8. Sadly, there's no way to check the rcN number, so I checked
 867                 // the rc8 changelog. On rc8 OSInfo was introduced, so...
 868                 (pClient && pClient->GetClientSoft() == SO_AMULE && !pClient->GetClientOSInfo().IsEmpty())
 869                 ||
 870                 // eservers use a flag for this, at least.
 871                 (pServer && (pServer->GetTCPFlags() & SRV_TCPFLG_NEWTAGS));
 872
 873         for (TagPtrList::iterator it = tags.begin(); it != tags.end(); ++it ) {
 874                 CTag* pTag = *it;
 875                 if (new_ed2k) {
 876                         pTag->WriteNewEd2kTag(files, eStrEncode);
 877                 } else {
 878                         pTag->WriteTagToFile(files, eStrEncode);
 879                 }
 880                 delete pTag;
 881         }
 882 }
 883
 884
 885 void CSharedFileList::Process()
 886 {
 887         Publish();
 888         if( !m_lastPublishED2KFlag || ( ::GetTickCount() - m_lastPublishED2K < ED2KREPUBLISHTIME ) ) {
 889                 return;
 890         }
 891         SendListToServer();
 892         m_lastPublishED2K = ::GetTickCount();
 893 }
 894
 895 void CSharedFileList::Publish()
 896 {
 897         // Variables to save cpu.
 898         unsigned int tNow = time(NULL);
 899         bool IsFirewalled = theApp->IsFirewalled();
 900
 901         if( Kademlia::CKademlia::IsConnected() && ( !IsFirewalled || ( IsFirewalled && theApp->clientlist->GetBuddyStatus() == Connected)) && GetCount() && Kademlia::CKademlia::GetPublish()) {
 902                 //We are connected to Kad. We are either open or have a buddy. And Kad is ready to start publishing.
 903
 904                 if( Kademlia::CKademlia::GetTotalStoreKey() < KADEMLIATOTALSTOREKEY) {
 905
 906                         //We are not at the max simultaneous keyword publishes
 907                         if (tNow >= m_keywords->GetNextPublishTime()) {
 908
 909                                 //Enough time has passed since last keyword publish
 910
 911                                 //Get the next keyword which has to be (re)-published
 912                                 CPublishKeyword* pPubKw = m_keywords->GetNextKeyword();
 913                                 if (pPubKw) {
 914
 915                                         //We have the next keyword to check if it can be published
 916
 917                                         //Debug check to make sure things are going well.
 918                                         wxASSERT( pPubKw->GetRefCount() != 0 );
 919
 920                                         if (tNow >= pPubKw->GetNextPublishTime()) {
 921                                                 //This keyword can be published.
 922                                                 Kademlia::CSearch* pSearch = Kademlia::CSearchManager::PrepareLookup(Kademlia::CSearch::STOREKEYWORD, false, pPubKw->GetKadID());
 923                                                 if (pSearch) {
 924                                                         //pSearch was created. Which means no search was already being done with this HashID.
 925                                                         //This also means that it was checked to see if network load wasn't a factor.
 926
 927                                                         //This sets the filename into the search object so we can show it in the gui.
 928                                                         pSearch->SetFileName(pPubKw->GetKeyword());
 929
 930                                                         //Add all file IDs which relate to the current keyword to be published
 931                                                         const KnownFileArray& aFiles = pPubKw->GetReferences();
 932                                                         uint32 count = 0;
 933                                                         for (unsigned int f = 0; f < aFiles.size(); ++f) {
 934
 935                                                                 //Only publish complete files as someone else should have the full file to publish these keywords.
 936                                                                 //As a side effect, this may help reduce people finding incomplete files in the network.
 937                                                                 if( !aFiles[f]->IsPartFile() ) {
 938                                                                         count++;
 939                                                                         pSearch->AddFileID(Kademlia::CUInt128(aFiles[f]->GetFileHash().GetHash()));
 940                                                                         if( count > 150 ) {
 941                                                                                 //We only publish up to 150 files per keyword publish then rotate the list.
 942                                                                                 pPubKw->RotateReferences(f);
 943                                                                                 break;
 944                                                                         }
 945                                                                 }
 946                                                         }
 947
 948                                                         if( count ) {
 949                                                                 //Start our keyword publish
 950                                                                 pPubKw->SetNextPublishTime(tNow+(KADEMLIAREPUBLISHTIMEK));
 951                                                                 pPubKw->IncPublishedCount();
 952                                                                 Kademlia::CSearchManager::StartSearch(pSearch);
 953                                                         } else {
 954                                                                 //There were no valid files to publish with this keyword.
 955                                                                 delete pSearch;
 956                                                         }
 957                                                 }
 958                                         }
 959                                 }
 960                                 m_keywords->SetNextPublishTime(KADEMLIAPUBLISHTIME+tNow);
 961                         }
 962                 }
 963
 964                 if( Kademlia::CKademlia::GetTotalStoreSrc() < KADEMLIATOTALSTORESRC) {
 965                         if(tNow >= m_lastPublishKadSrc) {
 966                                 if(m_currFileSrc > GetCount()) {
 967                                         m_currFileSrc = 0;
 968                                 }
 969                                 CKnownFile* pCurKnownFile = const_cast<CKnownFile*>(GetFileByIndex(m_currFileSrc));
 970                                 if(pCurKnownFile) {
 971                                         if(pCurKnownFile->PublishSrc()) {
 972                                                 Kademlia::CUInt128 kadFileID;
 973                                                 kadFileID.SetValueBE(pCurKnownFile->GetFileHash().GetHash());
 974                                                 if(Kademlia::CSearchManager::PrepareLookup(Kademlia::CSearch::STOREFILE, true, kadFileID )==NULL) {
 975                                                         pCurKnownFile->SetLastPublishTimeKadSrc(0,0);
 976                                                 }
 977                                         }
 978                                 }
 979                                 m_currFileSrc++;
 980
 981                                 // even if we did not publish a source, reset the timer so that this list is processed
 982                                 // only every KADEMLIAPUBLISHTIME seconds.
 983                                 m_lastPublishKadSrc = KADEMLIAPUBLISHTIME+tNow;
 984                         }
 985                 }
 986
 987                 if( Kademlia::CKademlia::GetTotalStoreNotes() < KADEMLIATOTALSTORENOTES) {
 988                         if(tNow >= m_lastPublishKadNotes) {
 989                                 if(m_currFileNotes > GetCount()) {
 990                                         m_currFileNotes = 0;
 991                                 }
 992                                 CKnownFile* pCurKnownFile = const_cast<CKnownFile*>(GetFileByIndex(m_currFileNotes));
 993                                 if(pCurKnownFile) {
 994                                         if(pCurKnownFile->PublishNotes()) {
 995                                                 Kademlia::CUInt128 kadFileID;
 996                                                 kadFileID.SetValueBE(pCurKnownFile->GetFileHash().GetHash());
 997                                                 if(Kademlia::CSearchManager::PrepareLookup(Kademlia::CSearch::STORENOTES, true, kadFileID )==NULL)
 998                                                         pCurKnownFile->SetLastPublishTimeKadNotes(0);
 999                                         }
1000                                 }
1001                                 m_currFileNotes++;
1002
1003                                 // even if we did not publish a source, reset the timer so that this list is processed
1004                                 // only every KADEMLIAPUBLISHTIME seconds.
1005                                 m_lastPublishKadNotes = KADEMLIAPUBLISHTIME+tNow;
1006                         }
1007                 }
1008         }
1009 }
1010
1011
1012 void CSharedFileList::AddKeywords(CKnownFile* pFile)
1013 {
1014         m_keywords->AddKeywords(pFile);
1015 }
1016
1017
1018 void CSharedFileList::RemoveKeywords(CKnownFile* pFile)
1019 {
1020         m_keywords->RemoveKeywords(pFile);
1021 }
1022
1023
1024 bool CSharedFileList::RenameFile(CKnownFile* file, const CPath& newName)
1025 {
1026         if (file->IsPartFile()) {
1027                 CPartFile* pfile = dynamic_cast<CPartFile*>(file);
1028
1029                 if (file->GetStatus() != PS_COMPLETING) {
1030                         pfile->SetFileName(newName);
1031                         pfile->SavePartFile();
1032
1033                         Notify_SharedFilesUpdateItem(file);
1034                         Notify_DownloadCtrlUpdateItem(file);
1035
1036                         return true;
1037                 }
1038         } else {
1039 //#warning Renaming of completed files causes problems on kad. Enable when reviewed.
1040 #if 0
1041                 wxString oldPath = JoinPaths(file->GetFilePath(), file->GetFileName());
1042                 wxString newPath = JoinPaths(file->GetFilePath(), newName);
1043
1044                 if (UTF8_MoveFile(oldPath, newPath)) {
1045                         RemoveKeywords(file);
1046                         file->SetFileName(newName);
1047                         AddKeywords(file);
1048                         theApp->knownfiles->Save();
1049                         UpdateItem(file);
1050                         RepublishFile(file);
1051
1052                         Notify_DownloadCtrlUpdateItem(file);
1053                         Notify_SharedFilesUpdateItem(file);
1054
1055                         return true;
1056                 }
1057 #endif
1058         }
1059
1060         return false;
1061 }
1062
1063
1064 void CSharedFileList::CheckAICHHashes(const std::list<CAICHHash>& hashes)
1065 {
1066         wxMutexLocker locker(list_mut);
1067
1068         // Now we check that all files which are in the sharedfilelist have a
1069         // corresponding hash in our list. Those how don't are queued for hashing.
1070         CKnownFileMap::iterator it = m_Files_map.begin();
1071         for (; it != m_Files_map.end(); ++it) {
1072                 const CKnownFile* file = it->second;
1073
1074                 if (file->IsPartFile() == false) {
1075                         CAICHHashSet* hashset = file->GetAICHHashset();
1076
1077                         if (hashset->GetStatus() == AICH_HASHSETCOMPLETE) {
1078                                 if (std::find(hashes.begin(), hashes.end(), hashset->GetMasterHash()) != hashes.end()) {
1079                                         continue;
1080                                 }
1081                         }
1082
1083                         hashset->SetStatus(AICH_ERROR);
1084
1085                         CThreadScheduler::AddTask(new CHashingTask(file));
1086                 }
1087         }
1088
1089 }
1090
1091 // File_checked_for_headers