src/SharedFileList.cpp

   1 //
   2 // This file is part of the aMule Project.
   3 //
   4 // Copyright (c) 2003-2008 aMule Team ( admin@amule.org / http://www.amule.org )
   5 // Copyright (c) 2002-2008 Merkur ( devs@emule-project.net / http://www.emule-project.net )
   6 //
   7 // Any parts of this program derived from the xMule, lMule or eMule project,
   8 // or contributed by third-party developers are copyrighted by their
   9 // respective authors.
  10 //
  11 // This program is free software; you can redistribute it and/or modify
  12 // it under the terms of the GNU General Public License as published by
  13 // the Free Software Foundation; either version 2 of the License, or
  14 // (at your option) any later version.
  15 //
  16 // This program is distributed in the hope that it will be useful,
  17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 // GNU General Public License for more details.
  20 //
  21 // You should have received a copy of the GNU General Public License
  22 // along with this program; if not, write to the Free Software
  23 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
  24 //
  25
  26 #include "SharedFileList.h"     // Interface declarations  // Do_not_auto_remove
  27
  28 #include <protocol/Protocols.h>
  29 #include <protocol/ed2k/ClientSoftware.h>
  30 #include <protocol/kad/Constants.h>
  31 #include <tags/FileTags.h>
  32
  33 #include <wx/utils.h>
  34
  35 #include "Packet.h"             // Needed for CPacket
  36 #include "MemFile.h"            // Needed for CMemFile
  37 #include "ServerConnect.h"      // Needed for CServerConnect
  38 #include "KnownFileList.h"      // Needed for CKnownFileList
  39 #include "ThreadTasks.h"        // Needed for CThreadScheduler and CHasherTask
  40 #include "Preferences.h"        // Needed for thePrefs
  41 #include "DownloadQueue.h"      // Needed for CDownloadQueue
  42 #include "amule.h"              // Needed for theApp
  43 #include "PartFile.h"           // Needed for PartFile
  44 #include "Server.h"             // Needed for CServer
  45 #include "updownclient.h"
  46 #include "Statistics.h"         // Needed for theStats
  47 #include "Logger.h"
  48 #include <common/Format.h>
  49 #include <common/FileFunctions.h>
  50 #include "GuiEvents.h"          // Needed for Notify_*
  51 #include "SHAHashSet.h"         // Needed for CAICHHash
  52
  53
  54 #include "kademlia/kademlia/Kademlia.h"
  55 #include "kademlia/kademlia/Search.h"
  56 #include "ClientList.h"
  57
  58 typedef std::deque<CKnownFile*> KnownFileArray;
  59
  60 ///////////////////////////////////////////////////////////////////////////////
  61 // CPublishKeyword
  62
  63 class CPublishKeyword
  64 {
  65 public:
  66         CPublishKeyword(const wxString& rstrKeyword)
  67         {
  68                 m_strKeyword = rstrKeyword;
  69                 // min. keyword char is allowed to be < 3 in some cases (see also 'CSearchManager::getWords')
  70                 //ASSERT( rstrKeyword.GetLength() >= 3 );
  71                 wxASSERT( !rstrKeyword.IsEmpty() );
  72                 KadGetKeywordHash(rstrKeyword, &m_nKadID);
  73                 SetNextPublishTime(0);
  74                 SetPublishedCount(0);
  75         }
  76
  77         const Kademlia::CUInt128& GetKadID() const { return m_nKadID; }
  78         const wxString& GetKeyword() const { return m_strKeyword; }
  79         int GetRefCount() const { return m_aFiles.size(); }
  80         const KnownFileArray& GetReferences() const { return m_aFiles; }
  81
  82         uint32 GetNextPublishTime() const { return m_tNextPublishTime; }
  83         void SetNextPublishTime(uint32 tNextPublishTime) { m_tNextPublishTime = tNextPublishTime; }
  84
  85         uint32 GetPublishedCount() const { return m_uPublishedCount; }
  86         void SetPublishedCount(uint32 uPublishedCount) { m_uPublishedCount = uPublishedCount; }
  87         void IncPublishedCount() { m_uPublishedCount++; }
  88
  89         bool AddRef(CKnownFile* pFile) {
  90                 if (std::find(m_aFiles.begin(), m_aFiles.end(), pFile) != m_aFiles.end()) {
  91                         wxFAIL;
  92                         return false;
  93                 }
  94                 m_aFiles.push_back(pFile);
  95                 return true;
  96         }
  97
  98         int RemoveRef(CKnownFile* pFile) {
  99                 KnownFileArray::iterator it = std::find(m_aFiles.begin(), m_aFiles.end(), pFile);
 100                 if (it != m_aFiles.end()) {
 101                         m_aFiles.erase(it);
 102                 }
 103                 return m_aFiles.size();
 104         }
 105
 106         void RemoveAllReferences() {
 107                 m_aFiles.clear();
 108         }
 109
 110         void RotateReferences(unsigned iRotateSize) {
 111                 wxCHECK_RET(m_aFiles.size(), wxT("RotateReferences: Rotating empty array"));
 112
 113                 unsigned shift = (iRotateSize % m_aFiles.size());
 114                 std::rotate(m_aFiles.begin(), m_aFiles.begin() + shift, m_aFiles.end());
 115         }
 116
 117 protected:
 118         wxString m_strKeyword;
 119         Kademlia::CUInt128 m_nKadID;
 120         uint32 m_tNextPublishTime;
 121         uint32 m_uPublishedCount;
 122         KnownFileArray m_aFiles;
 123 };
 124
 125
 126 ///////////////////////////////////////////////////////////////////////////////
 127 // CPublishKeywordList
 128
 129 class CPublishKeywordList
 130 {
 131 public:
 132         CPublishKeywordList();
 133         ~CPublishKeywordList();
 134
 135         void AddKeyword(const wxString& keyword, CKnownFile *file);
 136         void AddKeywords(CKnownFile* pFile);
 137         void RemoveKeyword(const wxString& keyword, CKnownFile *file);
 138         void RemoveKeywords(CKnownFile* pFile);
 139         void RemoveAllKeywords();
 140
 141         void RemoveAllKeywordReferences();
 142         void PurgeUnreferencedKeywords();
 143
 144         int GetCount() const { return m_lstKeywords.size(); }
 145
 146         CPublishKeyword* GetNextKeyword();
 147         void ResetNextKeyword();
 148
 149         uint32 GetNextPublishTime() const { return m_tNextPublishKeywordTime; }
 150         void SetNextPublishTime(uint32 tNextPublishKeywordTime) { m_tNextPublishKeywordTime = tNextPublishKeywordTime; }
 151
 152 protected:
 153         // can't use a CMap - too many disadvantages in processing the 'list'
 154         //CTypedPtrMap<CMapStringToPtr, CString, CPublishKeyword*> m_lstKeywords;
 155         typedef std::list<CPublishKeyword*> CKeyWordList;
 156         CKeyWordList m_lstKeywords;
 157         CKeyWordList::iterator m_posNextKeyword;
 158         uint32 m_tNextPublishKeywordTime;
 159
 160         CPublishKeyword* FindKeyword(const wxString& rstrKeyword, CKeyWordList::iterator* ppos = NULL);
 161 };
 162
 163 CPublishKeywordList::CPublishKeywordList()
 164 {
 165         ResetNextKeyword();
 166         SetNextPublishTime(0);
 167 }
 168
 169 CPublishKeywordList::~CPublishKeywordList()
 170 {
 171         RemoveAllKeywords();
 172 }
 173
 174 CPublishKeyword* CPublishKeywordList::GetNextKeyword()
 175 {
 176         if (m_posNextKeyword == m_lstKeywords.end()) {
 177                 m_posNextKeyword = m_lstKeywords.begin();
 178                 if (m_posNextKeyword == m_lstKeywords.end()) {
 179                         return NULL;
 180                 }
 181         }
 182         return *m_posNextKeyword++;
 183 }
 184
 185 void CPublishKeywordList::ResetNextKeyword()
 186 {
 187         m_posNextKeyword = m_lstKeywords.begin();
 188 }
 189
 190 CPublishKeyword* CPublishKeywordList::FindKeyword(const wxString& rstrKeyword, CKeyWordList::iterator* ppos)
 191 {
 192         CKeyWordList::iterator it = m_lstKeywords.begin();
 193         for (; it != m_lstKeywords.end(); ++it) {
 194                 CPublishKeyword* pPubKw = *it;
 195                 if (pPubKw->GetKeyword() == rstrKeyword) {
 196                         if (ppos) {
 197                                 (*ppos) = it;
 198                         }
 199
 200                         return pPubKw;
 201                 }
 202         }
 203
 204         return NULL;
 205 }
 206
 207 void CPublishKeywordList::AddKeyword(const wxString& keyword, CKnownFile *file)
 208 {
 209         CPublishKeyword* pubKw = FindKeyword(keyword);
 210         if (pubKw == NULL) {
 211                 pubKw = new CPublishKeyword(keyword);
 212                 m_lstKeywords.push_back(pubKw);
 213                 SetNextPublishTime(0);
 214         }
 215         pubKw->AddRef(file);
 216 }
 217
 218 void CPublishKeywordList::AddKeywords(CKnownFile* pFile)
 219 {
 220         const Kademlia::WordList& wordlist = pFile->GetKadKeywords();
 221
 222         Kademlia::WordList::const_iterator it;
 223         for (it = wordlist.begin(); it != wordlist.end(); ++it) {
 224                 AddKeyword(*it, pFile);
 225         }
 226 }
 227
 228 void CPublishKeywordList::RemoveKeyword(const wxString& keyword, CKnownFile *file)
 229 {
 230         CKeyWordList::iterator pos;
 231         CPublishKeyword* pubKw = FindKeyword(keyword, &pos);
 232         if (pubKw != NULL) {
 233                 if (pubKw->RemoveRef(file) == 0) {
 234                         if (pos == m_posNextKeyword) {
 235                                 ++m_posNextKeyword;
 236                         }
 237                         m_lstKeywords.erase(pos);
 238                         delete pubKw;
 239                         SetNextPublishTime(0);
 240                 }
 241         }
 242 }
 243
 244 void CPublishKeywordList::RemoveKeywords(CKnownFile* pFile)
 245 {
 246         const Kademlia::WordList& wordlist = pFile->GetKadKeywords();
 247         Kademlia::WordList::const_iterator it;
 248         for (it = wordlist.begin(); it != wordlist.end(); ++it) {
 249                 RemoveKeyword(*it, pFile);
 250         }
 251 }
 252
 253
 254 void CPublishKeywordList::RemoveAllKeywords()
 255 {
 256         DeleteContents(m_lstKeywords);
 257         ResetNextKeyword();
 258         SetNextPublishTime(0);
 259 }
 260
 261
 262 void CPublishKeywordList::RemoveAllKeywordReferences()
 263 {
 264         CKeyWordList::iterator it = m_lstKeywords.begin();
 265         for (; it != m_lstKeywords.end(); ++it) {
 266                 (*it)->RemoveAllReferences();
 267         }
 268 }
 269
 270
 271 void CPublishKeywordList::PurgeUnreferencedKeywords()
 272 {
 273         CKeyWordList::iterator it = m_lstKeywords.begin();
 274         while (it != m_lstKeywords.end()) {
 275                 CPublishKeyword* pPubKw = *it;
 276                 if (pPubKw->GetRefCount() == 0) {
 277                         if (it == m_posNextKeyword) {
 278                                 ++m_posNextKeyword;
 279                         }
 280                         m_lstKeywords.erase(it++);
 281                         delete pPubKw;
 282                         SetNextPublishTime(0);
 283                 } else {
 284                         ++it;
 285                 }
 286         }
 287 }
 288
 289
 290 CSharedFileList::CSharedFileList(CKnownFileList* in_filelist){
 291         filelist = in_filelist;
 292         reloading = false;
 293         m_lastPublishED2K = 0;
 294         m_lastPublishED2KFlag = true;
 295         /* Kad Stuff */
 296         m_keywords = new CPublishKeywordList;
 297         m_currFileSrc = 0;
 298         m_currFileNotes = 0;
 299         m_lastPublishKadSrc = 0;
 300         m_lastPublishKadNotes = 0;
 301         m_currFileKey = 0;
 302 }
 303
 304
 305 CSharedFileList::~CSharedFileList()
 306 {
 307         delete m_keywords;
 308 }
 309
 310
 311 void CSharedFileList::FindSharedFiles()
 312 {
 313         /* Abort loading if we are shutting down. */
 314         if(theApp->IsOnShutDown()) {
 315                 return;
 316         }
 317
 318         // Clear statistics.
 319         theStats::ClearSharedFilesInfo();
 320
 321         // Reload shareddir.dat
 322         theApp->glob_prefs->ReloadSharedFolders();
 323
 324         {
 325                 wxMutexLocker lock(list_mut);
 326                 m_Files_map.clear();
 327         }
 328
 329         // All part files are automatically shared.
 330         for ( uint32 i = 0; i < theApp->downloadqueue->GetFileCount(); ++i ) {
 331                 CPartFile* file = theApp->downloadqueue->GetFileByIndex( i );
 332
 333                 if ( file->GetStatus(true) == PS_READY ) {
 334                         AddLogLineNS(CFormat(_("Adding file %s to shares"))
 335                                 % file->GetFullName().GetPrintable());
 336                         AddFile(file);
 337                 }
 338         }
 339
 340         // Create a list of all shared paths and weed out duplicates.
 341         std::list<CPath> sharedPaths;
 342
 343         // Global incoming dir and all category incoming directories are automatically shared.
 344         sharedPaths.push_back(thePrefs::GetIncomingDir());
 345         for (unsigned int i = 1;i < theApp->glob_prefs->GetCatCount(); ++i) {
 346                 sharedPaths.push_back(theApp->glob_prefs->GetCatPath(i));
 347         }
 348
 349         const thePrefs::PathList& shared = theApp->glob_prefs->shareddir_list;
 350         sharedPaths.insert(sharedPaths.end(), shared.begin(), shared.end());
 351
 352         sharedPaths.sort();
 353         sharedPaths.unique();
 354
 355         filelist->PrepareIndex();
 356         unsigned addedFiles = 0;
 357         std::list<CPath>::iterator it = sharedPaths.begin();
 358         for (; it != sharedPaths.end(); ++it) {
 359                 addedFiles += AddFilesFromDirectory(*it);
 360         }
 361         filelist->ReleaseIndex();
 362
 363         if (addedFiles == 0) {
 364                 AddLogLineM(false, wxString::Format(wxPLURAL("Found %i known shared file", "Found %i known shared files", GetCount()), GetCount()));
 365
 366                 // Make sure the AICH-hashes are up to date.
 367                 CThreadScheduler::AddTask(new CAICHSyncTask());
 368         } else {
 369                 // New files, AICH thread will be run at the end of the hashing thread.
 370                 AddLogLineM(false, wxString::Format(wxPLURAL("Found %i known shared file, %i unknown", "Found %i known shared files, %i unknown", GetCount()), GetCount(), addedFiles));
 371         }
 372 }
 373
 374
 375 // Checks if the dir a is the same as b. If they are, then logs the message and returns true.
 376 bool CheckDirectory(const wxString& a, const CPath& b)
 377 {
 378         if (CPath(a).IsSameDir(b)) {
 379                 AddLogLineM(true, CFormat( _("ERROR: Attempted to share %s") ) % a);
 380
 381                 return true;
 382         }
 383
 384         return false;
 385 }
 386
 387
 388 unsigned CSharedFileList::AddFilesFromDirectory(const CPath& directory)
 389 {
 390         // Do not allow these folders to be shared:
 391         //  - The .aMule folder
 392         //  - The Temp folder
 393         //  - The users home-dir
 394         if (CheckDirectory(wxGetHomeDir(), directory)) {
 395                 return 0;
 396         } else if (CheckDirectory(theApp->ConfigDir, directory)) {
 397                 return 0;
 398         } else if (CheckDirectory(thePrefs::GetTempDir().GetRaw(), directory)) {
 399                 return 0;
 400         }
 401
 402         if (!directory.DirExists()) {
 403                 AddLogLineNS(CFormat(_("Shared directory not found, skipping: %s"))
 404                         % directory.GetPrintable());
 405
 406                 return 0;
 407         }
 408
 409         CDirIterator::FileType searchFor = CDirIterator::FileNoHidden;
 410         if (thePrefs::ShareHiddenFiles()) {
 411                  searchFor = CDirIterator::File;
 412         }
 413
 414         unsigned knownFiles = 0;
 415         unsigned addedFiles = 0;
 416
 417         CDirIterator SharedDir(directory);
 418
 419         CPath fname = SharedDir.GetFirstFile(searchFor);
 420         while (fname.IsOk()) {
 421                 CPath fullPath = directory.JoinPaths(fname);
 422
 423                 if (!fullPath.FileExists()) {
 424                         AddDebugLogLineM(false, logKnownFiles,
 425                                 CFormat(wxT("Shared file does not exist (possibly a broken link): %s")) % fullPath);
 426
 427                         fname = SharedDir.GetNextFile();
 428                         continue;
 429                 }
 430
 431                 AddDebugLogLineM(false, logKnownFiles,
 432                         CFormat(wxT("Found shared file: %s")) % fullPath);
 433
 434                 time_t fdate = CPath::GetModificationTime(fullPath);
 435                 sint64 fsize = fullPath.GetFileSize();
 436
 437                 // This will also catch files with too strict permissions.
 438                 if ((fdate == (time_t)-1) || (fsize == wxInvalidOffset)) {
 439                         AddDebugLogLineM(false, logKnownFiles,
 440                                 CFormat(wxT("Failed to retrive modification time or size for '%s', skipping.")) % fullPath);
 441
 442                         fname = SharedDir.GetNextFile();
 443                         continue;
 444                 }
 445
 446
 447                 CKnownFile* toadd = filelist->FindKnownFile(fname, fdate, fsize);
 448                 if (toadd) {
 449                         knownFiles++;
 450                         if (AddFile(toadd)) {
 451                                 AddDebugLogLineM(false, logKnownFiles,
 452                                         CFormat(wxT("Added known file '%s' to shares"))
 453                                                 % fname);
 454
 455                                 toadd->SetFilePath(directory);
 456                         } else {
 457                                 AddDebugLogLineM(false, logKnownFiles,
 458                                         CFormat(wxT("File already shared, skipping: %s"))
 459                                                 % fname);
 460                         }
 461                 } else {
 462                         //not in knownfilelist - start adding thread to hash file
 463                         AddDebugLogLineM(false, logKnownFiles,
 464                                 CFormat(wxT("Hashing new unknown shared file '%s'")) % fname);
 465
 466                         if (CThreadScheduler::AddTask(new CHashingTask(directory, fname))) {
 467                                 addedFiles++;
 468                         }
 469                 }
 470
 471                 fname = SharedDir.GetNextFile();
 472         }
 473
 474         if ((addedFiles == 0) && (knownFiles == 0)) {
 475                 AddLogLineNS(CFormat(_("No shareable files found in directory: %s"))
 476                         % directory.GetPrintable());
 477         }
 478
 479         return addedFiles;
 480 }
 481
 482
 483 bool CSharedFileList::AddFile(CKnownFile* pFile)
 484 {
 485         wxASSERT(pFile->GetHashCount() == pFile->GetED2KPartHashCount());
 486
 487         wxMutexLocker lock(list_mut);
 488
 489         CKnownFileMap::value_type entry(pFile->GetFileHash(), pFile);
 490         if (m_Files_map.insert(entry).second) {
 491                 /* Keywords to publish on Kad */
 492                 m_keywords->AddKeywords(pFile);
 493                 theStats::AddSharedFile(pFile->GetFileSize());
 494                 return true;
 495         }
 496         return false;
 497 }
 498
 499
 500 void CSharedFileList::SafeAddKFile(CKnownFile* toadd, bool bOnlyAdd)
 501 {
 502         // TODO: Check if the file is already known - only with another date
 503
 504         if (AddFile(toadd)) {
 505                 Notify_SharedFilesShowFile(toadd);
 506         }
 507
 508         if (!bOnlyAdd && theApp->IsConnectedED2K()) {
 509                 // Publishing of files is not anymore handled here.
 510                 // Instead, the timer does it by itself.
 511                 m_lastPublishED2KFlag = true;
 512         }
 513 }
 514
 515
 516 // removes first occurrence of 'toremove' in 'list'
 517 void CSharedFileList::RemoveFile(CKnownFile* toremove){
 518         Notify_SharedFilesRemoveFile(toremove);
 519         wxMutexLocker lock(list_mut);
 520         if (m_Files_map.erase(toremove->GetFileHash()) > 0) {
 521                 theStats::RemoveSharedFile(toremove->GetFileSize());
 522         }
 523         /* This file keywords must not be published to kad anymore */
 524         m_keywords->RemoveKeywords(toremove);
 525 }
 526
 527
 528 void CSharedFileList::Reload()
 529 {
 530         // Madcat - Disable reloading if reloading already in progress.
 531         // Kry - Fixed to let non-english language users use the 'Reload' button :P
 532         // deltaHF - removed the old ugly button and changed the code to use the new small one
 533         // Kry - bah, let's use a var.
 534         if (!reloading) {
 535                 reloading = true;
 536                 Notify_SharedFilesRemoveAllItems();
 537
 538                 /* All Kad keywords must be removed */
 539                 m_keywords->RemoveAllKeywordReferences();
 540
 541                 FindSharedFiles();
 542
 543                 /* And now the unreferenced keywords must be removed also */
 544                 m_keywords->PurgeUnreferencedKeywords();
 545
 546                 Notify_SharedFilesShowFileList();
 547
 548                 reloading = false;
 549         }
 550 }
 551
 552
 553 const CKnownFile *CSharedFileList::GetFileByIndex(unsigned int index) const
 554 {
 555         wxMutexLocker lock(list_mut);
 556         if ( index >= m_Files_map.size() ) {
 557                 return NULL;
 558         }
 559         CKnownFileMap::const_iterator pos = m_Files_map.begin();
 560         std::advance(pos, index);
 561         return pos->second;
 562 }
 563
 564
 565 CKnownFile*     CSharedFileList::GetFileByID(const CMD4Hash& filehash)
 566 {
 567         wxMutexLocker lock(list_mut);
 568         CKnownFileMap::iterator it = m_Files_map.find(filehash);
 569
 570         if ( it != m_Files_map.end() ) {
 571                 return it->second;
 572         } else {
 573                 return NULL;
 574         }
 575 }
 576
 577 short CSharedFileList::GetFilePriorityByID(const CMD4Hash& filehash)
 578 {
 579         CKnownFile* tocheck = GetFileByID(filehash);
 580         if (tocheck)
 581                 return tocheck->GetUpPriority();
 582         else
 583                 return -10;     // file doesn't exist
 584 }
 585
 586
 587 void CSharedFileList::CopyFileList(std::vector<CKnownFile*>& out_list) const
 588 {
 589         wxMutexLocker lock(list_mut);
 590
 591         out_list.reserve(m_Files_map.size());
 592         for (
 593                 CKnownFileMap::const_iterator it = m_Files_map.begin();
 594                 it != m_Files_map.end();
 595                 ++it
 596                 ) {
 597                 out_list.push_back(it->second);
 598         }
 599 }
 600
 601
 602
 603 void CSharedFileList::UpdateItem(CKnownFile* toupdate)
 604 {
 605         Notify_SharedFilesUpdateItem(toupdate);
 606 }
 607
 608
 609 void CSharedFileList::GetSharedFilesByDirectory(const wxString& directory,
 610                                 CKnownFilePtrList& list)
 611 {
 612         wxMutexLocker lock(list_mut);
 613
 614         const CPath dir = CPath(directory);
 615         for (CKnownFileMap::iterator pos = m_Files_map.begin();
 616              pos != m_Files_map.end(); ++pos ) {
 617                 CKnownFile *cur_file = pos->second;
 618
 619                 if (dir.IsSameDir(cur_file->GetFilePath())) {
 620                         continue;
 621                 }
 622
 623                 list.push_back(cur_file);
 624         }
 625 }
 626
 627 /* ---------------- Network ----------------- */
 628
 629 void CSharedFileList::ClearED2KPublishInfo(){
 630         CKnownFile* cur_file;
 631         m_lastPublishED2KFlag = true;
 632         wxMutexLocker lock(list_mut);
 633         for (CKnownFileMap::iterator pos = m_Files_map.begin(); pos != m_Files_map.end(); ++pos ) {
 634                 cur_file = pos->second;
 635                 cur_file->SetPublishedED2K(false);
 636         }
 637 }
 638
 639 void CSharedFileList::ClearKadSourcePublishInfo()
 640 {
 641         wxMutexLocker lock(list_mut);
 642         CKnownFile* cur_file;
 643         for (CKnownFileMap::iterator pos = m_Files_map.begin(); pos != m_Files_map.end(); ++pos ) {
 644                 cur_file = pos->second;
 645                 cur_file->SetLastPublishTimeKadSrc(0,0);
 646         }
 647 }
 648
 649 void CSharedFileList::RepublishFile(CKnownFile* pFile)
 650 {
 651         CServer* server = theApp->serverconnect->GetCurrentServer();
 652         if (server && (server->GetTCPFlags() & SRV_TCPFLG_COMPRESSION)) {
 653                 m_lastPublishED2KFlag = true;
 654                 pFile->SetPublishedED2K(false); // FIXME: this creates a wrong 'No' for the ed2k shared info in the listview until the file is shared again.
 655         }
 656 }
 657
 658 uint8 GetRealPrio(uint8 in)
 659 {
 660         switch(in) {
 661                 case 4 : return 0;
 662                 case 0 : return 1;
 663                 case 1 : return 2;
 664                 case 2 : return 3;
 665                 case 3 : return 4;
 666         }
 667         return 0;
 668 }
 669
 670 bool SortFunc( const CKnownFile* fileA, const CKnownFile* fileB )
 671 {
 672     return GetRealPrio(fileA->GetUpPriority()) < GetRealPrio(fileB->GetUpPriority());
 673 }
 674
 675 void CSharedFileList::SendListToServer(){
 676         std::vector<CKnownFile*> SortedList;
 677
 678         {
 679                 wxMutexLocker lock(list_mut);
 680
 681                 if (m_Files_map.empty() || !theApp->IsConnectedED2K() ) {
 682                         return;
 683                 }
 684
 685                 // Getting a sorted list of the non-published files.
 686                 SortedList.reserve( m_Files_map.size() );
 687
 688                 CKnownFileMap::iterator it = m_Files_map.begin();
 689                 for ( ; it != m_Files_map.end(); ++it ) {
 690                         if (!it->second->GetPublishedED2K()) {
 691                                 SortedList.push_back( it->second );
 692                         }
 693                 }
 694         }
 695
 696         std::sort( SortedList.begin(), SortedList.end(), SortFunc );
 697
 698         // Limits for the server.
 699
 700         CServer* server = theApp->serverconnect->GetCurrentServer();
 701
 702         uint32 limit = server ? server->GetSoftFiles() : 0;
 703         if( limit == 0 || limit > 200 ) {
 704                 limit = 200;
 705         }
 706
 707         if( (uint32)SortedList.size() < limit ) {
 708                 limit = SortedList.size();
 709                 if (limit == 0) {
 710                         m_lastPublishED2KFlag = false;
 711                         return;
 712                 }
 713         }
 714
 715         CMemFile files;
 716
 717         // Files sent.
 718         files.WriteUInt32(limit);
 719
 720         uint16 count = 0;
 721         // Add to packet
 722         std::vector<CKnownFile*>::iterator sorted_it = SortedList.begin();
 723         for ( ; (sorted_it != SortedList.end()) && (count < limit); ++sorted_it ) {
 724                 CKnownFile* file = *sorted_it;
 725                 if (!file->IsLargeFile() || (server && server->SupportsLargeFilesTCP())) {
 726                         CreateOfferedFilePacket(file, &files, server, NULL);
 727                 }
 728                 file->SetPublishedED2K(true);
 729                 ++count;
 730         }
 731
 732         wxASSERT(count == limit);
 733
 734         CPacket* packet = new CPacket(files, OP_EDONKEYPROT, OP_OFFERFILES);
 735         // compress packet
 736         //   - this kind of data is highly compressable (N * (1 MD4 and at least 3 string meta data tags and 1 integer meta data tag))
 737         //   - the min. amount of data needed for one published file is ~100 bytes
 738         //   - this function is called once when connecting to a server and when a file becomes shareable - so, it's called rarely.
 739         //   - if the compressed size is still >= the original size, we send the uncompressed packet
 740         // therefor we always try to compress the packet
 741         if (server->GetTCPFlags() & SRV_TCPFLG_COMPRESSION){
 742                 packet->PackPacket();
 743         }
 744
 745         theStats::AddUpOverheadServer(packet->GetPacketSize());
 746         theApp->serverconnect->SendPacket(packet,true);
 747 }
 748
 749
 750 void CSharedFileList::CreateOfferedFilePacket(
 751         CKnownFile *cur_file,
 752         CMemFile *files,
 753         CServer *pServer,
 754         CUpDownClient *pClient) {
 755
 756         // This function is used for offering files to the local server and for sending
 757         // shared files to some other client. In each case we send our IP+Port only, if
 758         // we have a HighID.
 759
 760         wxASSERT(!(pClient && pServer));
 761
 762         cur_file->SetPublishedED2K(true);
 763         files->WriteHash(cur_file->GetFileHash());
 764
 765         uint32 nClientID = 0;
 766         uint16 nClientPort = 0;
 767
 768         if (pServer) {
 769                 if (pServer->GetTCPFlags() & SRV_TCPFLG_COMPRESSION) {
 770                         #define FILE_COMPLETE_ID                0xfbfbfbfb
 771                         #define FILE_COMPLETE_PORT      0xfbfb
 772                         #define FILE_INCOMPLETE_ID      0xfcfcfcfc
 773                         #define FILE_INCOMPLETE_PORT    0xfcfc
 774                         // complete   file: ip 251.251.251 (0xfbfbfbfb) port 0xfbfb
 775                         // incomplete file: op 252.252.252 (0xfcfcfcfc) port 0xfcfc
 776                         if (cur_file->GetStatus() == PS_COMPLETE) {
 777                                 nClientID = FILE_COMPLETE_ID;
 778                                 nClientPort = FILE_COMPLETE_PORT;
 779                         } else {
 780                                 nClientID = FILE_INCOMPLETE_ID;
 781                                 nClientPort = FILE_INCOMPLETE_PORT;
 782                         }
 783                 } else {
 784                         if (theApp->IsConnectedED2K() && !::IsLowID(theApp->GetED2KID())){
 785                                 nClientID = theApp->GetID();
 786                                 nClientPort = thePrefs::GetPort();
 787                         }
 788                 }
 789         } else {
 790                 // Do not merge this with the above case - this one
 791                 // also checks Kad status.
 792                 if (theApp->IsConnected() && !theApp->IsFirewalled()) {
 793                         nClientID = theApp->GetID();
 794                         nClientPort = thePrefs::GetPort();
 795                 }
 796         }
 797
 798         files->WriteUInt32(nClientID);
 799         files->WriteUInt16(nClientPort);
 800
 801         TagPtrList tags;
 802
 803         // The printable filename is used because it's destined for another user.
 804         tags.push_back(new CTagString(FT_FILENAME, cur_file->GetFileName().GetPrintable()));
 805
 806         if (pClient && pClient->GetVBTTags()) {
 807                 tags.push_back(new CTagVarInt(FT_FILESIZE, cur_file->GetFileSize()));
 808         } else {
 809                 if (!cur_file->IsLargeFile()){
 810                         tags.push_back(new CTagInt32(FT_FILESIZE, cur_file->GetFileSize()));
 811                 } else {
 812                         // Large file
 813                         // we send 2*32 bit tags to servers, but a real 64 bit tag to other clients.
 814                         if (pServer) {
 815                                 if (!pServer->SupportsLargeFilesTCP()){
 816                                         wxFAIL;
 817                                         tags.push_back(new CTagInt32(FT_FILESIZE, 0));
 818                                 }else {
 819                                         tags.push_back(new CTagInt32(FT_FILESIZE, (uint32)cur_file->GetFileSize()));
 820                                         tags.push_back(new CTagInt32(FT_FILESIZE_HI, (uint32)(cur_file->GetFileSize() >> 32)));
 821                                 }
 822                         } else {
 823                                 if (!pClient->SupportsLargeFiles()) {
 824                                         wxFAIL;
 825                                         tags.push_back(new CTagInt32(FT_FILESIZE, 0));
 826                                 } else {
 827                                         tags.push_back(new CTagInt64(FT_FILESIZE, cur_file->GetFileSize()));
 828                                 }
 829                         }
 830                 }
 831         }
 832
 833         if (cur_file->GetFileRating()) {
 834                 tags.push_back(new CTagVarInt(FT_FILERATING, cur_file->GetFileRating(), (pClient && pClient->GetVBTTags()) ? 0 : 32));
 835         }
 836
 837         // NOTE: Archives and CD-Images are published+searched with file type "Pro"
 838         bool bAddedFileType = false;
 839         if (pServer && (pServer->GetTCPFlags() & SRV_TCPFLG_TYPETAGINTEGER)) {
 840                 // Send integer file type tags to newer servers
 841                 EED2KFileType eFileType = GetED2KFileTypeSearchID(GetED2KFileTypeID(cur_file->GetFileName()));
 842                 if (eFileType >= ED2KFT_AUDIO && eFileType <= ED2KFT_CDIMAGE) {
 843                         tags.push_back(new CTagInt32(FT_FILETYPE, eFileType));
 844                         bAddedFileType = true;
 845                 }
 846         }
 847         if (!bAddedFileType) {
 848                 // Send string file type tags to:
 849                 //      - newer servers, in case there is no integer type available for the file type (e.g. emulecollection)
 850                 //      - older servers
 851                 //      - all clients
 852                 wxString strED2KFileType(GetED2KFileTypeSearchTerm(GetED2KFileTypeID(cur_file->GetFileName())));
 853                 if (!strED2KFileType.IsEmpty()) {
 854                         tags.push_back(new CTagString(FT_FILETYPE, strED2KFileType));
 855                 }
 856         }
 857
 858         // There, we could add MetaData info, if we ever get to have that.
 859
 860         EUtf8Str eStrEncode;
 861
 862         bool unicode_support =
 863                 // eservers that support UNICODE.
 864                 (pServer && (pServer->GetUnicodeSupport()))
 865                 ||
 866                 // clients that support unicode
 867                 (pClient && pClient->GetUnicodeSupport());
 868         eStrEncode = unicode_support ? utf8strRaw : utf8strNone;
 869
 870         files->WriteUInt32(tags.size());
 871
 872         // Sadly, eMule doesn't use a MISCOPTIONS flag on hello packet for this, so we
 873         // have to identify the support for new tags by version.
 874         bool new_ed2k =
 875                 // eMule client > 0.42f
 876                 (pClient && pClient->IsEmuleClient() && pClient->GetVersion()  >= MAKE_CLIENT_VERSION(0,42,7))
 877                 ||
 878                 // aMule >= 2.0.0rc8. Sadly, there's no way to check the rcN number, so I checked
 879                 // the rc8 changelog. On rc8 OSInfo was introduced, so...
 880                 (pClient && pClient->GetClientSoft() == SO_AMULE && !pClient->GetClientOSInfo().IsEmpty())
 881                 ||
 882                 // eservers use a flag for this, at least.
 883                 (pServer && (pServer->GetTCPFlags() & SRV_TCPFLG_NEWTAGS));
 884
 885         for (TagPtrList::iterator it = tags.begin(); it != tags.end(); ++it ) {
 886                 CTag* pTag = *it;
 887                 if (new_ed2k) {
 888                         pTag->WriteNewEd2kTag(files, eStrEncode);
 889                 } else {
 890                         pTag->WriteTagToFile(files, eStrEncode);
 891                 }
 892                 delete pTag;
 893         }
 894 }
 895
 896
 897 void CSharedFileList::Process()
 898 {
 899         Publish();
 900         if( !m_lastPublishED2KFlag || ( ::GetTickCount() - m_lastPublishED2K < ED2KREPUBLISHTIME ) ) {
 901                 return;
 902         }
 903         SendListToServer();
 904         m_lastPublishED2K = ::GetTickCount();
 905 }
 906
 907 void CSharedFileList::Publish()
 908 {
 909         // Variables to save cpu.
 910         unsigned int tNow = time(NULL);
 911         bool IsFirewalled = theApp->IsFirewalled();
 912
 913         if( Kademlia::CKademlia::IsConnected() && ( !IsFirewalled || ( IsFirewalled && theApp->clientlist->GetBuddyStatus() == Connected)) && GetCount() && Kademlia::CKademlia::GetPublish()) {
 914                 //We are connected to Kad. We are either open or have a buddy. And Kad is ready to start publishing.
 915
 916                 if( Kademlia::CKademlia::GetTotalStoreKey() < KADEMLIATOTALSTOREKEY) {
 917
 918                         //We are not at the max simultaneous keyword publishes
 919                         if (tNow >= m_keywords->GetNextPublishTime()) {
 920
 921                                 //Enough time has passed since last keyword publish
 922
 923                                 //Get the next keyword which has to be (re)-published
 924                                 CPublishKeyword* pPubKw = m_keywords->GetNextKeyword();
 925                                 if (pPubKw) {
 926
 927                                         //We have the next keyword to check if it can be published
 928
 929                                         //Debug check to make sure things are going well.
 930                                         wxASSERT( pPubKw->GetRefCount() != 0 );
 931
 932                                         if (tNow >= pPubKw->GetNextPublishTime()) {
 933                                                 //This keyword can be published.
 934                                                 Kademlia::CSearch* pSearch = Kademlia::CSearchManager::PrepareLookup(Kademlia::CSearch::STOREKEYWORD, false, pPubKw->GetKadID());
 935                                                 if (pSearch) {
 936                                                         //pSearch was created. Which means no search was already being done with this HashID.
 937                                                         //This also means that it was checked to see if network load wasn't a factor.
 938
 939                                                         //This sets the filename into the search object so we can show it in the gui.
 940                                                         pSearch->SetFileName(pPubKw->GetKeyword());
 941
 942                                                         //Add all file IDs which relate to the current keyword to be published
 943                                                         const KnownFileArray& aFiles = pPubKw->GetReferences();
 944                                                         uint32 count = 0;
 945                                                         for (unsigned int f = 0; f < aFiles.size(); ++f) {
 946
 947                                                                 //Only publish complete files as someone else should have the full file to publish these keywords.
 948                                                                 //As a side effect, this may help reduce people finding incomplete files in the network.
 949                                                                 if( !aFiles[f]->IsPartFile() ) {
 950                                                                         count++;
 951                                                                         pSearch->AddFileID(Kademlia::CUInt128(aFiles[f]->GetFileHash().GetHash()));
 952                                                                         if( count > 150 ) {
 953                                                                                 //We only publish up to 150 files per keyword publish then rotate the list.
 954                                                                                 pPubKw->RotateReferences(f);
 955                                                                                 break;
 956                                                                         }
 957                                                                 }
 958                                                         }
 959
 960                                                         if( count ) {
 961                                                                 //Start our keyword publish
 962                                                                 pPubKw->SetNextPublishTime(tNow+(KADEMLIAREPUBLISHTIMEK));
 963                                                                 pPubKw->IncPublishedCount();
 964                                                                 Kademlia::CSearchManager::StartSearch(pSearch);
 965                                                         } else {
 966                                                                 //There were no valid files to publish with this keyword.
 967                                                                 delete pSearch;
 968                                                         }
 969                                                 }
 970                                         }
 971                                 }
 972                                 m_keywords->SetNextPublishTime(KADEMLIAPUBLISHTIME+tNow);
 973                         }
 974                 }
 975
 976                 if( Kademlia::CKademlia::GetTotalStoreSrc() < KADEMLIATOTALSTORESRC) {
 977                         if(tNow >= m_lastPublishKadSrc) {
 978                                 if(m_currFileSrc > GetCount()) {
 979                                         m_currFileSrc = 0;
 980                                 }
 981                                 CKnownFile* pCurKnownFile = const_cast<CKnownFile*>(GetFileByIndex(m_currFileSrc));
 982                                 if(pCurKnownFile) {
 983                                         if(pCurKnownFile->PublishSrc()) {
 984                                                 Kademlia::CUInt128 kadFileID;
 985                                                 kadFileID.SetValueBE(pCurKnownFile->GetFileHash().GetHash());
 986                                                 if(Kademlia::CSearchManager::PrepareLookup(Kademlia::CSearch::STOREFILE, true, kadFileID )==NULL) {
 987                                                         pCurKnownFile->SetLastPublishTimeKadSrc(0,0);
 988                                                 }
 989                                         }
 990                                 }
 991                                 m_currFileSrc++;
 992
 993                                 // even if we did not publish a source, reset the timer so that this list is processed
 994                                 // only every KADEMLIAPUBLISHTIME seconds.
 995                                 m_lastPublishKadSrc = KADEMLIAPUBLISHTIME+tNow;
 996                         }
 997                 }
 998
 999                 if( Kademlia::CKademlia::GetTotalStoreNotes() < KADEMLIATOTALSTORENOTES) {
1000                         if(tNow >= m_lastPublishKadNotes) {
1001                                 if(m_currFileNotes > GetCount()) {
1002                                         m_currFileNotes = 0;
1003                                 }
1004                                 CKnownFile* pCurKnownFile = const_cast<CKnownFile*>(GetFileByIndex(m_currFileNotes));
1005                                 if(pCurKnownFile) {
1006                                         if(pCurKnownFile->PublishNotes()) {
1007                                                 Kademlia::CUInt128 kadFileID;
1008                                                 kadFileID.SetValueBE(pCurKnownFile->GetFileHash().GetHash());
1009                                                 if(Kademlia::CSearchManager::PrepareLookup(Kademlia::CSearch::STORENOTES, true, kadFileID )==NULL)
1010                                                         pCurKnownFile->SetLastPublishTimeKadNotes(0);
1011                                         }
1012                                 }
1013                                 m_currFileNotes++;
1014
1015                                 // even if we did not publish a source, reset the timer so that this list is processed
1016                                 // only every KADEMLIAPUBLISHTIME seconds.
1017                                 m_lastPublishKadNotes = KADEMLIAPUBLISHTIME+tNow;
1018                         }
1019                 }
1020         }
1021 }
1022
1023
1024 void CSharedFileList::AddKeywords(CKnownFile* pFile)
1025 {
1026         m_keywords->AddKeywords(pFile);
1027 }
1028
1029
1030 void CSharedFileList::RemoveKeywords(CKnownFile* pFile)
1031 {
1032         m_keywords->RemoveKeywords(pFile);
1033 }
1034
1035
1036 bool CSharedFileList::RenameFile(CKnownFile* file, const CPath& newName)
1037 {
1038         if (file->IsPartFile()) {
1039                 CPartFile* pfile = dynamic_cast<CPartFile*>(file);
1040
1041                 if (file->GetStatus() != PS_COMPLETING) {
1042                         pfile->SetFileName(newName);
1043                         pfile->SavePartFile();
1044
1045                         Notify_SharedFilesUpdateItem(file);
1046                         Notify_DownloadCtrlUpdateItem(file);
1047
1048                         return true;
1049                 }
1050         } else {
1051                 CPath oldPath = file->GetFilePath().JoinPaths(file->GetFileName());
1052                 CPath newPath = file->GetFilePath().JoinPaths(newName);
1053
1054                 if (CPath::RenameFile(oldPath, newPath)) {
1055                         // Must create a copy of the word list because:
1056                         // 1) it will be reset on SetFileName()
1057                         // 2) we will want to edit it
1058                         Kademlia::WordList oldwords = file->GetKadKeywords();
1059                         file->SetFileName(newName);
1060                         theApp->knownfiles->Save();
1061                         UpdateItem(file);
1062                         RepublishFile(file);
1063
1064                         const Kademlia::WordList& newwords = file->GetKadKeywords();
1065                         Kademlia::WordList::iterator itold;
1066                         Kademlia::WordList::const_iterator itnew;
1067                         // compare keywords in old and new names
1068                         for (itnew = newwords.begin(); itnew != newwords.end(); ++itnew) {
1069                                 for (itold = oldwords.begin(); itold != oldwords.end(); ++itold) {
1070                                         if (*itold == *itnew) {
1071                                                 break;
1072                                         }
1073                                 }
1074                                 if (itold != oldwords.end()) {
1075                                         // Remove keyword from old name which also exist in new name
1076                                         oldwords.erase(itold);
1077                                 } else {
1078                                         // This is a new keyword not present in the old name
1079                                         m_keywords->AddKeyword(*itnew, file);
1080                                 }
1081                         }
1082                         // Remove all remaining old keywords not present in the new name
1083                         for (itold = oldwords.begin(); itold != oldwords.end(); ++itold) {
1084                                 m_keywords->RemoveKeyword(*itold, file);
1085                         }
1086
1087                         Notify_DownloadCtrlUpdateItem(file);
1088                         Notify_SharedFilesUpdateItem(file);
1089
1090                         return true;
1091                 }
1092         }
1093
1094         return false;
1095 }
1096
1097
1098 void CSharedFileList::CheckAICHHashes(const std::list<CAICHHash>& hashes)
1099 {
1100         wxMutexLocker locker(list_mut);
1101
1102         // Now we check that all files which are in the sharedfilelist have a
1103         // corresponding hash in our list. Those how don't are queued for hashing.
1104         CKnownFileMap::iterator it = m_Files_map.begin();
1105         for (; it != m_Files_map.end(); ++it) {
1106                 const CKnownFile* file = it->second;
1107
1108                 if (file->IsPartFile() == false) {
1109                         CAICHHashSet* hashset = file->GetAICHHashset();
1110
1111                         if (hashset->GetStatus() == AICH_HASHSETCOMPLETE) {
1112                                 if (std::find(hashes.begin(), hashes.end(), hashset->GetMasterHash()) != hashes.end()) {
1113                                         continue;
1114                                 }
1115                         }
1116
1117                         hashset->SetStatus(AICH_ERROR);
1118
1119                         CThreadScheduler::AddTask(new CHashingTask(file));
1120                 }
1121         }
1122
1123 }
1124
1125 // File_checked_for_headers