src/SharedFileList.cpp

   1 //
   2 // This file is part of the aMule Project.
   3 //
   4 // Copyright (c) 2003-2011 aMule Team ( admin@amule.org / http://www.amule.org )
   5 // Copyright (c) 2002-2011 Merkur ( devs@emule-project.net / http://www.emule-project.net )
   6 //
   7 // Any parts of this program derived from the xMule, lMule or eMule project,
   8 // or contributed by third-party developers are copyrighted by their
   9 // respective authors.
  10 //
  11 // This program is free software; you can redistribute it and/or modify
  12 // it under the terms of the GNU General Public License as published by
  13 // the Free Software Foundation; either version 2 of the License, or
  14 // (at your option) any later version.
  15 //
  16 // This program is distributed in the hope that it will be useful,
  17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 // GNU General Public License for more details.
  20 //
  21 // You should have received a copy of the GNU General Public License
  22 // along with this program; if not, write to the Free Software
  23 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
  24 //
  25
  26 #include "SharedFileList.h"     // Interface declarations  // Do_not_auto_remove
  27
  28 #include <protocol/Protocols.h>
  29 #include <protocol/kad/Constants.h>
  30 #include <tags/FileTags.h>
  31
  32 #include <wx/utils.h>
  33
  34 #include "Packet.h"             // Needed for CPacket
  35 #include "MemFile.h"            // Needed for CMemFile
  36 #include "ServerConnect.h"      // Needed for CServerConnect
  37 #include "KnownFileList.h"      // Needed for CKnownFileList
  38 #include "ThreadTasks.h"        // Needed for CThreadScheduler and CHasherTask
  39 #include "Preferences.h"        // Needed for thePrefs
  40 #include "DownloadQueue.h"      // Needed for CDownloadQueue
  41 #include "amule.h"              // Needed for theApp
  42 #include "PartFile.h"           // Needed for PartFile
  43 #include "Server.h"             // Needed for CServer
  44 #include "Statistics.h"         // Needed for theStats
  45 #include "Logger.h"
  46 #include <common/Format.h>
  47 #include <common/FileFunctions.h>
  48 #include "GuiEvents.h"          // Needed for Notify_*
  49 #include "SHAHashSet.h"         // Needed for CAICHHash
  50
  51
  52 #include "kademlia/kademlia/Kademlia.h"
  53 #include "kademlia/kademlia/Search.h"
  54 #include "ClientList.h"
  55
  56 typedef std::deque<CKnownFile*> KnownFileArray;
  57
  58 ///////////////////////////////////////////////////////////////////////////////
  59 // CPublishKeyword
  60
  61 class CPublishKeyword
  62 {
  63 public:
  64         CPublishKeyword(const wxString& rstrKeyword)
  65         {
  66                 m_strKeyword = rstrKeyword;
  67                 // min. keyword char is allowed to be < 3 in some cases (see also 'CSearchManager::getWords')
  68                 //ASSERT( rstrKeyword.GetLength() >= 3 );
  69                 wxASSERT( !rstrKeyword.IsEmpty() );
  70                 KadGetKeywordHash(rstrKeyword, &m_nKadID);
  71                 SetNextPublishTime(0);
  72                 SetPublishedCount(0);
  73         }
  74
  75         const Kademlia::CUInt128& GetKadID() const { return m_nKadID; }
  76         const wxString& GetKeyword() const { return m_strKeyword; }
  77         int GetRefCount() const { return m_aFiles.size(); }
  78         const KnownFileArray& GetReferences() const { return m_aFiles; }
  79
  80         uint32 GetNextPublishTime() const { return m_tNextPublishTime; }
  81         void SetNextPublishTime(uint32 tNextPublishTime) { m_tNextPublishTime = tNextPublishTime; }
  82
  83         uint32 GetPublishedCount() const { return m_uPublishedCount; }
  84         void SetPublishedCount(uint32 uPublishedCount) { m_uPublishedCount = uPublishedCount; }
  85         void IncPublishedCount() { m_uPublishedCount++; }
  86
  87         bool AddRef(CKnownFile* pFile) {
  88                 if (std::find(m_aFiles.begin(), m_aFiles.end(), pFile) != m_aFiles.end()) {
  89                         wxFAIL;
  90                         return false;
  91                 }
  92                 m_aFiles.push_back(pFile);
  93                 return true;
  94         }
  95
  96         int RemoveRef(CKnownFile* pFile) {
  97                 KnownFileArray::iterator it = std::find(m_aFiles.begin(), m_aFiles.end(), pFile);
  98                 if (it != m_aFiles.end()) {
  99                         m_aFiles.erase(it);
 100                 }
 101                 return m_aFiles.size();
 102         }
 103
 104         void RemoveAllReferences() {
 105                 m_aFiles.clear();
 106         }
 107
 108         void RotateReferences(unsigned iRotateSize) {
 109                 wxCHECK_RET(m_aFiles.size(), wxT("RotateReferences: Rotating empty array"));
 110
 111                 unsigned shift = (iRotateSize % m_aFiles.size());
 112                 std::rotate(m_aFiles.begin(), m_aFiles.begin() + shift, m_aFiles.end());
 113         }
 114
 115 protected:
 116         wxString m_strKeyword;
 117         Kademlia::CUInt128 m_nKadID;
 118         uint32 m_tNextPublishTime;
 119         uint32 m_uPublishedCount;
 120         KnownFileArray m_aFiles;
 121 };
 122
 123
 124 ///////////////////////////////////////////////////////////////////////////////
 125 // CPublishKeywordList
 126
 127 class CPublishKeywordList
 128 {
 129 public:
 130         CPublishKeywordList();
 131         ~CPublishKeywordList();
 132
 133         void AddKeyword(const wxString& keyword, CKnownFile *file);
 134         void AddKeywords(CKnownFile* pFile);
 135         void RemoveKeyword(const wxString& keyword, CKnownFile *file);
 136         void RemoveKeywords(CKnownFile* pFile);
 137         void RemoveAllKeywords();
 138
 139         void RemoveAllKeywordReferences();
 140         void PurgeUnreferencedKeywords();
 141
 142         int GetCount() const { return m_lstKeywords.size(); }
 143
 144         CPublishKeyword* GetNextKeyword();
 145         void ResetNextKeyword();
 146
 147         uint32 GetNextPublishTime() const { return m_tNextPublishKeywordTime; }
 148         void SetNextPublishTime(uint32 tNextPublishKeywordTime) { m_tNextPublishKeywordTime = tNextPublishKeywordTime; }
 149
 150 protected:
 151         // can't use a CMap - too many disadvantages in processing the 'list'
 152         //CTypedPtrMap<CMapStringToPtr, CString, CPublishKeyword*> m_lstKeywords;
 153         typedef std::list<CPublishKeyword*> CKeyWordList;
 154         CKeyWordList m_lstKeywords;
 155         CKeyWordList::iterator m_posNextKeyword;
 156         uint32 m_tNextPublishKeywordTime;
 157
 158         CPublishKeyword* FindKeyword(const wxString& rstrKeyword, CKeyWordList::iterator* ppos = NULL);
 159 };
 160
 161 CPublishKeywordList::CPublishKeywordList()
 162 {
 163         ResetNextKeyword();
 164         SetNextPublishTime(0);
 165 }
 166
 167 CPublishKeywordList::~CPublishKeywordList()
 168 {
 169         RemoveAllKeywords();
 170 }
 171
 172 CPublishKeyword* CPublishKeywordList::GetNextKeyword()
 173 {
 174         if (m_posNextKeyword == m_lstKeywords.end()) {
 175                 m_posNextKeyword = m_lstKeywords.begin();
 176                 if (m_posNextKeyword == m_lstKeywords.end()) {
 177                         return NULL;
 178                 }
 179         }
 180         return *m_posNextKeyword++;
 181 }
 182
 183 void CPublishKeywordList::ResetNextKeyword()
 184 {
 185         m_posNextKeyword = m_lstKeywords.begin();
 186 }
 187
 188 CPublishKeyword* CPublishKeywordList::FindKeyword(const wxString& rstrKeyword, CKeyWordList::iterator* ppos)
 189 {
 190         CKeyWordList::iterator it = m_lstKeywords.begin();
 191         for (; it != m_lstKeywords.end(); ++it) {
 192                 CPublishKeyword* pPubKw = *it;
 193                 if (pPubKw->GetKeyword() == rstrKeyword) {
 194                         if (ppos) {
 195                                 (*ppos) = it;
 196                         }
 197
 198                         return pPubKw;
 199                 }
 200         }
 201
 202         return NULL;
 203 }
 204
 205 void CPublishKeywordList::AddKeyword(const wxString& keyword, CKnownFile *file)
 206 {
 207         CPublishKeyword* pubKw = FindKeyword(keyword);
 208         if (pubKw == NULL) {
 209                 pubKw = new CPublishKeyword(keyword);
 210                 m_lstKeywords.push_back(pubKw);
 211                 SetNextPublishTime(0);
 212         }
 213         pubKw->AddRef(file);
 214 }
 215
 216 void CPublishKeywordList::AddKeywords(CKnownFile* pFile)
 217 {
 218         const Kademlia::WordList& wordlist = pFile->GetKadKeywords();
 219
 220         Kademlia::WordList::const_iterator it;
 221         for (it = wordlist.begin(); it != wordlist.end(); ++it) {
 222                 AddKeyword(*it, pFile);
 223         }
 224 }
 225
 226 void CPublishKeywordList::RemoveKeyword(const wxString& keyword, CKnownFile *file)
 227 {
 228         CKeyWordList::iterator pos;
 229         CPublishKeyword* pubKw = FindKeyword(keyword, &pos);
 230         if (pubKw != NULL) {
 231                 if (pubKw->RemoveRef(file) == 0) {
 232                         if (pos == m_posNextKeyword) {
 233                                 ++m_posNextKeyword;
 234                         }
 235                         m_lstKeywords.erase(pos);
 236                         delete pubKw;
 237                         SetNextPublishTime(0);
 238                 }
 239         }
 240 }
 241
 242 void CPublishKeywordList::RemoveKeywords(CKnownFile* pFile)
 243 {
 244         const Kademlia::WordList& wordlist = pFile->GetKadKeywords();
 245         Kademlia::WordList::const_iterator it;
 246         for (it = wordlist.begin(); it != wordlist.end(); ++it) {
 247                 RemoveKeyword(*it, pFile);
 248         }
 249 }
 250
 251
 252 void CPublishKeywordList::RemoveAllKeywords()
 253 {
 254         DeleteContents(m_lstKeywords);
 255         ResetNextKeyword();
 256         SetNextPublishTime(0);
 257 }
 258
 259
 260 void CPublishKeywordList::RemoveAllKeywordReferences()
 261 {
 262         CKeyWordList::iterator it = m_lstKeywords.begin();
 263         for (; it != m_lstKeywords.end(); ++it) {
 264                 (*it)->RemoveAllReferences();
 265         }
 266 }
 267
 268
 269 void CPublishKeywordList::PurgeUnreferencedKeywords()
 270 {
 271         CKeyWordList::iterator it = m_lstKeywords.begin();
 272         while (it != m_lstKeywords.end()) {
 273                 CPublishKeyword* pPubKw = *it;
 274                 if (pPubKw->GetRefCount() == 0) {
 275                         if (it == m_posNextKeyword) {
 276                                 ++m_posNextKeyword;
 277                         }
 278                         m_lstKeywords.erase(it++);
 279                         delete pPubKw;
 280                         SetNextPublishTime(0);
 281                 } else {
 282                         ++it;
 283                 }
 284         }
 285 }
 286
 287
 288 CSharedFileList::CSharedFileList(CKnownFileList* in_filelist){
 289         filelist = in_filelist;
 290         reloading = false;
 291         m_lastPublishED2K = 0;
 292         m_lastPublishED2KFlag = true;
 293         /* Kad Stuff */
 294         m_keywords = new CPublishKeywordList;
 295         m_currFileSrc = 0;
 296         m_currFileNotes = 0;
 297         m_lastPublishKadSrc = 0;
 298         m_lastPublishKadNotes = 0;
 299         m_currFileKey = 0;
 300 }
 301
 302
 303 CSharedFileList::~CSharedFileList()
 304 {
 305         delete m_keywords;
 306 }
 307
 308
 309 void CSharedFileList::FindSharedFiles()
 310 {
 311         /* Abort loading if we are shutting down. */
 312         if(theApp->IsOnShutDown()) {
 313                 return;
 314         }
 315
 316         // Clear statistics.
 317         theStats::ClearSharedFilesInfo();
 318
 319         // Reload shareddir.dat
 320         theApp->glob_prefs->ReloadSharedFolders();
 321
 322         {
 323                 wxMutexLocker lock(list_mut);
 324                 m_Files_map.clear();
 325         }
 326
 327         // All part files are automatically shared.
 328         for ( uint32 i = 0; i < theApp->downloadqueue->GetFileCount(); ++i ) {
 329                 CPartFile* file = theApp->downloadqueue->GetFileByIndex( i );
 330
 331                 if ( file->GetStatus(true) == PS_READY ) {
 332                         AddLogLineNS(CFormat(_("Adding file %s to shares"))
 333                                 % file->GetFullName().GetPrintable());
 334                         AddFile(file);
 335                 }
 336         }
 337
 338         // Create a list of all shared paths and weed out duplicates.
 339         std::list<CPath> sharedPaths;
 340
 341         // Global incoming dir and all category incoming directories are automatically shared.
 342         sharedPaths.push_back(thePrefs::GetIncomingDir());
 343         for (unsigned int i = 1;i < theApp->glob_prefs->GetCatCount(); ++i) {
 344                 sharedPaths.push_back(theApp->glob_prefs->GetCatPath(i));
 345         }
 346
 347         const thePrefs::PathList& shared = theApp->glob_prefs->shareddir_list;
 348         sharedPaths.insert(sharedPaths.end(), shared.begin(), shared.end());
 349
 350         sharedPaths.sort();
 351         sharedPaths.unique();
 352
 353         filelist->PrepareIndex();
 354         // Gathering is done in the foreground and can be slowed down severely by parallel background hashing.
 355         // So just store the hashing tasks for now.
 356         TaskList hashTasks;
 357         for (std::list<CPath>::iterator it = sharedPaths.begin(); it != sharedPaths.end(); ++it) {
 358                 AddFilesFromDirectory(*it, hashTasks);
 359         }
 360         filelist->ReleaseIndex();
 361
 362         // Now that the shared files are gathered feed the hashing tasks to the scheduler to start hashing.
 363         unsigned addedFiles = 0;
 364         for (TaskList::iterator it = hashTasks.begin(); it != hashTasks.end(); ++it) {
 365                 if (CThreadScheduler::AddTask(*it)) {
 366                         addedFiles++;
 367                 }
 368         }
 369
 370         if (addedFiles == 0) {
 371                 AddLogLineN(CFormat(wxPLURAL("Found %i known shared file", "Found %i known shared files", GetCount())) % GetCount());
 372
 373                 // Make sure the AICH-hashes are up to date.
 374                 CThreadScheduler::AddTask(new CAICHSyncTask());
 375         } else {
 376                 // New files, AICH thread will be run at the end of the hashing thread.
 377                 AddLogLineN(CFormat(wxPLURAL("Found %i known shared file, %i unknown", "Found %i known shared files, %i unknown", GetCount())) % GetCount() % addedFiles);
 378         }
 379 }
 380
 381
 382 // Checks if the dir a is the same as b. If they are, then logs the message and returns true.
 383 bool CheckDirectory(const wxString& a, const CPath& b)
 384 {
 385         if (CPath(a).IsSameDir(b)) {
 386                 AddLogLineC(CFormat( _("ERROR: Attempted to share %s") ) % a);
 387
 388                 return true;
 389         }
 390
 391         return false;
 392 }
 393
 394
 395 unsigned CSharedFileList::AddFilesFromDirectory(const CPath& directory, TaskList & hashTasks)
 396 {
 397         // Do not allow these folders to be shared:
 398         //  - The .aMule folder
 399         //  - The Temp folder
 400         //  - The users home-dir
 401         if (CheckDirectory(wxGetHomeDir(), directory)) {
 402                 return 0;
 403         } else if (CheckDirectory(theApp->ConfigDir, directory)) {
 404                 return 0;
 405         } else if (CheckDirectory(thePrefs::GetTempDir().GetRaw(), directory)) {
 406                 return 0;
 407         }
 408
 409         if (!directory.DirExists()) {
 410                 AddLogLineNS(CFormat(_("Shared directory not found, skipping: %s"))
 411                         % directory.GetPrintable());
 412
 413                 return 0;
 414         }
 415
 416         CDirIterator::FileType searchFor = CDirIterator::FileNoHidden;
 417         if (thePrefs::ShareHiddenFiles()) {
 418                  searchFor = CDirIterator::File;
 419         }
 420
 421         unsigned knownFiles = 0;
 422         unsigned addedFiles = 0;
 423
 424         CDirIterator SharedDir(directory);
 425
 426         for (CPath fname = SharedDir.GetFirstFile(searchFor); fname.IsOk(); fname = SharedDir.GetNextFile()) {
 427                 CPath fullPath = directory.JoinPaths(fname);
 428
 429                 if (!fullPath.FileExists()) {
 430                         AddDebugLogLineN(logKnownFiles,
 431                                 CFormat(wxT("Shared file does not exist (possibly a broken link): %s")) % fullPath);
 432                         continue;
 433                 }
 434
 435                 AddDebugLogLineN(logKnownFiles,
 436                         CFormat(wxT("Found shared file: %s")) % fullPath);
 437
 438                 time_t fdate = CPath::GetModificationTime(fullPath);
 439                 sint64 fsize = fullPath.GetFileSize();
 440
 441                 // This will also catch files with too strict permissions.
 442                 if ((fdate == (time_t)-1) || (fsize == wxInvalidOffset)) {
 443                         AddDebugLogLineN(logKnownFiles,
 444                                 CFormat(wxT("Failed to retrieve modification time or size for '%s', skipping.")) % fullPath);
 445                         continue;
 446                 }
 447
 448                 if (fsize == 0) {
 449                         AddDebugLogLineN(logKnownFiles,
 450                                 CFormat(wxT("Skip zero size file '%s'")) % fullPath);
 451                         continue;
 452                 }
 453
 454
 455                 CKnownFile* toadd = filelist->FindKnownFile(fname, fdate, fsize);
 456                 if (toadd) {
 457                         knownFiles++;
 458                         if (AddFile(toadd)) {
 459                                 AddDebugLogLineN(logKnownFiles,
 460                                         CFormat(wxT("Added known file '%s' to shares"))
 461                                                 % fname);
 462
 463                                 toadd->SetFilePath(directory);
 464                         } else {
 465                                 AddDebugLogLineN(logKnownFiles,
 466                                         CFormat(wxT("File already shared, skipping: %s"))
 467                                                 % fname);
 468                         }
 469                 } else {
 470                         //not in knownfilelist - start adding thread to hash file
 471                         AddDebugLogLineN(logKnownFiles,
 472                                 CFormat(wxT("Hashing new unknown shared file '%s'")) % fname);
 473
 474                         hashTasks.push_back(new CHashingTask(directory, fname));
 475                         addedFiles++;
 476                 }
 477         }
 478
 479         if ((addedFiles == 0) && (knownFiles == 0)) {
 480                 AddLogLineN(CFormat(_("No shareable files found in directory: %s"))
 481                         % directory.GetPrintable());
 482         }
 483
 484         return addedFiles;
 485 }
 486
 487
 488 bool CSharedFileList::AddFile(CKnownFile* pFile)
 489 {
 490         wxASSERT(pFile->GetHashCount() == pFile->GetED2KPartHashCount());
 491
 492         wxMutexLocker lock(list_mut);
 493
 494         CKnownFileMap::value_type entry(pFile->GetFileHash(), pFile);
 495         if (m_Files_map.insert(entry).second) {
 496                 /* Keywords to publish on Kad */
 497                 m_keywords->AddKeywords(pFile);
 498                 theStats::AddSharedFile(pFile->GetFileSize());
 499                 return true;
 500         }
 501         return false;
 502 }
 503
 504
 505 void CSharedFileList::SafeAddKFile(CKnownFile* toadd, bool bOnlyAdd)
 506 {
 507         // TODO: Check if the file is already known - only with another date
 508
 509         if (AddFile(toadd)) {
 510                 Notify_SharedFilesShowFile(toadd);
 511         }
 512
 513         if (!bOnlyAdd && theApp->IsConnectedED2K()) {
 514                 // Publishing of files is not anymore handled here.
 515                 // Instead, the timer does it by itself.
 516                 m_lastPublishED2KFlag = true;
 517         }
 518 }
 519
 520
 521 // removes first occurrence of 'toremove' in 'list'
 522 void CSharedFileList::RemoveFile(CKnownFile* toremove){
 523         Notify_SharedFilesRemoveFile(toremove);
 524         wxMutexLocker lock(list_mut);
 525         if (m_Files_map.erase(toremove->GetFileHash()) > 0) {
 526                 theStats::RemoveSharedFile(toremove->GetFileSize());
 527         }
 528         /* This file keywords must not be published to kad anymore */
 529         m_keywords->RemoveKeywords(toremove);
 530 }
 531
 532
 533 void CSharedFileList::Reload()
 534 {
 535         // Madcat - Disable reloading if reloading already in progress.
 536         // Kry - Fixed to let non-english language users use the 'Reload' button :P
 537         // deltaHF - removed the old ugly button and changed the code to use the new small one
 538         // Kry - bah, let's use a var.
 539         if (!reloading) {
 540                 AddDebugLogLineN(logKnownFiles, wxT("Reload shared files"));
 541                 reloading = true;
 542                 Notify_SharedFilesRemoveAllItems();
 543
 544                 /* All Kad keywords must be removed */
 545                 m_keywords->RemoveAllKeywordReferences();
 546
 547                 /* Public identifiers must be erased as they might be invalid now */
 548                 m_PublicSharedDirNames.clear();
 549
 550                 FindSharedFiles();
 551
 552                 /* And now the unreferenced keywords must be removed also */
 553                 m_keywords->PurgeUnreferencedKeywords();
 554
 555                 Notify_SharedFilesShowFileList();
 556
 557                 reloading = false;
 558         }
 559 }
 560
 561
 562 const CKnownFile *CSharedFileList::GetFileByIndex(unsigned int index) const
 563 {
 564         wxMutexLocker lock(list_mut);
 565         if ( index >= m_Files_map.size() ) {
 566                 return NULL;
 567         }
 568         CKnownFileMap::const_iterator pos = m_Files_map.begin();
 569         std::advance(pos, index);
 570         return pos->second;
 571 }
 572
 573
 574 CKnownFile*     CSharedFileList::GetFileByID(const CMD4Hash& filehash)
 575 {
 576         wxMutexLocker lock(list_mut);
 577         CKnownFileMap::iterator it = m_Files_map.find(filehash);
 578
 579         if ( it != m_Files_map.end() ) {
 580                 return it->second;
 581         } else {
 582                 return NULL;
 583         }
 584 }
 585
 586 short CSharedFileList::GetFilePriorityByID(const CMD4Hash& filehash)
 587 {
 588         CKnownFile* tocheck = GetFileByID(filehash);
 589         if (tocheck)
 590                 return tocheck->GetUpPriority();
 591         else
 592                 return -10;     // file doesn't exist
 593 }
 594
 595
 596 void CSharedFileList::CopyFileList(std::vector<CKnownFile*>& out_list) const
 597 {
 598         wxMutexLocker lock(list_mut);
 599
 600         out_list.reserve(m_Files_map.size());
 601         for (
 602                 CKnownFileMap::const_iterator it = m_Files_map.begin();
 603                 it != m_Files_map.end();
 604                 ++it
 605                 ) {
 606                 out_list.push_back(it->second);
 607         }
 608 }
 609
 610
 611
 612 void CSharedFileList::UpdateItem(CKnownFile* toupdate)
 613 {
 614         Notify_SharedFilesUpdateItem(toupdate);
 615 }
 616
 617
 618 void CSharedFileList::GetSharedFilesByDirectory(const wxString& directory,
 619                                 CKnownFilePtrList& list)
 620 {
 621         wxMutexLocker lock(list_mut);
 622
 623         const CPath dir = CPath(directory);
 624         for (CKnownFileMap::iterator pos = m_Files_map.begin();
 625              pos != m_Files_map.end(); ++pos ) {
 626                 CKnownFile *cur_file = pos->second;
 627
 628                 if (dir.IsSameDir(cur_file->GetFilePath())) {
 629                         continue;
 630                 }
 631
 632                 list.push_back(cur_file);
 633         }
 634 }
 635
 636 /* ---------------- Network ----------------- */
 637
 638 void CSharedFileList::ClearED2KPublishInfo(){
 639         CKnownFile* cur_file;
 640         m_lastPublishED2KFlag = true;
 641         wxMutexLocker lock(list_mut);
 642         for (CKnownFileMap::iterator pos = m_Files_map.begin(); pos != m_Files_map.end(); ++pos ) {
 643                 cur_file = pos->second;
 644                 cur_file->SetPublishedED2K(false);
 645         }
 646 }
 647
 648 void CSharedFileList::ClearKadSourcePublishInfo()
 649 {
 650         wxMutexLocker lock(list_mut);
 651         CKnownFile* cur_file;
 652         for (CKnownFileMap::iterator pos = m_Files_map.begin(); pos != m_Files_map.end(); ++pos ) {
 653                 cur_file = pos->second;
 654                 cur_file->SetLastPublishTimeKadSrc(0,0);
 655         }
 656 }
 657
 658 void CSharedFileList::RepublishFile(CKnownFile* pFile)
 659 {
 660         CServer* server = theApp->serverconnect->GetCurrentServer();
 661         if (server && (server->GetTCPFlags() & SRV_TCPFLG_COMPRESSION)) {
 662                 m_lastPublishED2KFlag = true;
 663                 pFile->SetPublishedED2K(false); // FIXME: this creates a wrong 'No' for the ed2k shared info in the listview until the file is shared again.
 664         }
 665 }
 666
 667 uint8 GetRealPrio(uint8 in)
 668 {
 669         switch(in) {
 670                 case 4 : return 0;
 671                 case 0 : return 1;
 672                 case 1 : return 2;
 673                 case 2 : return 3;
 674                 case 3 : return 4;
 675         }
 676         return 0;
 677 }
 678
 679 bool SortFunc( const CKnownFile* fileA, const CKnownFile* fileB )
 680 {
 681     return GetRealPrio(fileA->GetUpPriority()) < GetRealPrio(fileB->GetUpPriority());
 682 }
 683
 684 void CSharedFileList::SendListToServer(){
 685         std::vector<CKnownFile*> SortedList;
 686
 687         {
 688                 wxMutexLocker lock(list_mut);
 689
 690                 if (m_Files_map.empty() || !theApp->IsConnectedED2K() ) {
 691                         return;
 692                 }
 693
 694                 // Getting a sorted list of the non-published files.
 695                 SortedList.reserve( m_Files_map.size() );
 696
 697                 CKnownFileMap::iterator it = m_Files_map.begin();
 698                 for ( ; it != m_Files_map.end(); ++it ) {
 699                         if (!it->second->GetPublishedED2K()) {
 700                                 SortedList.push_back( it->second );
 701                         }
 702                 }
 703         }
 704
 705         std::sort( SortedList.begin(), SortedList.end(), SortFunc );
 706
 707         // Limits for the server.
 708
 709         CServer* server = theApp->serverconnect->GetCurrentServer();
 710
 711         uint32 limit = server ? server->GetSoftFiles() : 0;
 712         if( limit == 0 || limit > 200 ) {
 713                 limit = 200;
 714         }
 715
 716         if( (uint32)SortedList.size() < limit ) {
 717                 limit = SortedList.size();
 718                 if (limit == 0) {
 719                         m_lastPublishED2KFlag = false;
 720                         return;
 721                 }
 722         }
 723
 724         CMemFile files;
 725
 726         // Files sent.
 727         files.WriteUInt32(limit);
 728
 729         uint16 count = 0;
 730         // Add to packet
 731         std::vector<CKnownFile*>::iterator sorted_it = SortedList.begin();
 732         for ( ; (sorted_it != SortedList.end()) && (count < limit); ++sorted_it ) {
 733                 CKnownFile* file = *sorted_it;
 734                 if (!file->IsLargeFile() || (server && server->SupportsLargeFilesTCP())) {
 735                         file->CreateOfferedFilePacket(&files, server, NULL);
 736                 }
 737                 file->SetPublishedED2K(true);
 738                 ++count;
 739         }
 740
 741         wxASSERT(count == limit);
 742
 743         CPacket* packet = new CPacket(files, OP_EDONKEYPROT, OP_OFFERFILES);
 744         // compress packet
 745         //   - this kind of data is highly compressable (N * (1 MD4 and at least 3 string meta data tags and 1 integer meta data tag))
 746         //   - the min. amount of data needed for one published file is ~100 bytes
 747         //   - this function is called once when connecting to a server and when a file becomes shareable - so, it's called rarely.
 748         //   - if the compressed size is still >= the original size, we send the uncompressed packet
 749         // therefor we always try to compress the packet
 750         if (server->GetTCPFlags() & SRV_TCPFLG_COMPRESSION){
 751                 packet->PackPacket();
 752         }
 753
 754         theStats::AddUpOverheadServer(packet->GetPacketSize());
 755         theApp->serverconnect->SendPacket(packet,true);
 756 }
 757
 758
 759 void CSharedFileList::Process()
 760 {
 761         Publish();
 762         if( !m_lastPublishED2KFlag || ( ::GetTickCount() - m_lastPublishED2K < ED2KREPUBLISHTIME ) ) {
 763                 return;
 764         }
 765         SendListToServer();
 766         m_lastPublishED2K = ::GetTickCount();
 767 }
 768
 769 void CSharedFileList::Publish()
 770 {
 771         // Variables to save cpu.
 772         unsigned int tNow = time(NULL);
 773         bool IsFirewalled = theApp->IsFirewalled();
 774
 775         if( Kademlia::CKademlia::IsConnected() && ( !IsFirewalled || ( IsFirewalled && theApp->clientlist->GetBuddyStatus() == Connected)) && GetCount() && Kademlia::CKademlia::GetPublish()) {
 776                 //We are connected to Kad. We are either open or have a buddy. And Kad is ready to start publishing.
 777
 778                 if( Kademlia::CKademlia::GetTotalStoreKey() < KADEMLIATOTALSTOREKEY) {
 779
 780                         //We are not at the max simultaneous keyword publishes
 781                         if (tNow >= m_keywords->GetNextPublishTime()) {
 782
 783                                 //Enough time has passed since last keyword publish
 784
 785                                 //Get the next keyword which has to be (re)-published
 786                                 CPublishKeyword* pPubKw = m_keywords->GetNextKeyword();
 787                                 if (pPubKw) {
 788
 789                                         //We have the next keyword to check if it can be published
 790
 791                                         //Debug check to make sure things are going well.
 792                                         wxASSERT( pPubKw->GetRefCount() != 0 );
 793
 794                                         if (tNow >= pPubKw->GetNextPublishTime()) {
 795                                                 //This keyword can be published.
 796                                                 Kademlia::CSearch* pSearch = Kademlia::CSearchManager::PrepareLookup(Kademlia::CSearch::STOREKEYWORD, false, pPubKw->GetKadID());
 797                                                 if (pSearch) {
 798                                                         //pSearch was created. Which means no search was already being done with this HashID.
 799                                                         //This also means that it was checked to see if network load wasn't a factor.
 800
 801                                                         //This sets the filename into the search object so we can show it in the gui.
 802                                                         pSearch->SetFileName(pPubKw->GetKeyword());
 803
 804                                                         //Add all file IDs which relate to the current keyword to be published
 805                                                         const KnownFileArray& aFiles = pPubKw->GetReferences();
 806                                                         uint32 count = 0;
 807                                                         for (unsigned int f = 0; f < aFiles.size(); ++f) {
 808
 809                                                                 //Only publish complete files as someone else should have the full file to publish these keywords.
 810                                                                 //As a side effect, this may help reduce people finding incomplete files in the network.
 811                                                                 if( !aFiles[f]->IsPartFile() ) {
 812                                                                         count++;
 813                                                                         pSearch->AddFileID(Kademlia::CUInt128(aFiles[f]->GetFileHash().GetHash()));
 814                                                                         if( count > 150 ) {
 815                                                                                 //We only publish up to 150 files per keyword publish then rotate the list.
 816                                                                                 pPubKw->RotateReferences(f);
 817                                                                                 break;
 818                                                                         }
 819                                                                 }
 820                                                         }
 821
 822                                                         if( count ) {
 823                                                                 //Start our keyword publish
 824                                                                 pPubKw->SetNextPublishTime(tNow+(KADEMLIAREPUBLISHTIMEK));
 825                                                                 pPubKw->IncPublishedCount();
 826                                                                 Kademlia::CSearchManager::StartSearch(pSearch);
 827                                                         } else {
 828                                                                 //There were no valid files to publish with this keyword.
 829                                                                 delete pSearch;
 830                                                         }
 831                                                 }
 832                                         }
 833                                 }
 834                                 m_keywords->SetNextPublishTime(KADEMLIAPUBLISHTIME+tNow);
 835                         }
 836                 }
 837
 838                 if( Kademlia::CKademlia::GetTotalStoreSrc() < KADEMLIATOTALSTORESRC) {
 839                         if(tNow >= m_lastPublishKadSrc) {
 840                                 if(m_currFileSrc > GetCount()) {
 841                                         m_currFileSrc = 0;
 842                                 }
 843                                 CKnownFile* pCurKnownFile = const_cast<CKnownFile*>(GetFileByIndex(m_currFileSrc));
 844                                 if(pCurKnownFile) {
 845                                         if(pCurKnownFile->PublishSrc()) {
 846                                                 Kademlia::CUInt128 kadFileID;
 847                                                 kadFileID.SetValueBE(pCurKnownFile->GetFileHash().GetHash());
 848                                                 if(Kademlia::CSearchManager::PrepareLookup(Kademlia::CSearch::STOREFILE, true, kadFileID )==NULL) {
 849                                                         pCurKnownFile->SetLastPublishTimeKadSrc(0,0);
 850                                                 }
 851                                         }
 852                                 }
 853                                 m_currFileSrc++;
 854
 855                                 // even if we did not publish a source, reset the timer so that this list is processed
 856                                 // only every KADEMLIAPUBLISHTIME seconds.
 857                                 m_lastPublishKadSrc = KADEMLIAPUBLISHTIME+tNow;
 858                         }
 859                 }
 860
 861                 if( Kademlia::CKademlia::GetTotalStoreNotes() < KADEMLIATOTALSTORENOTES) {
 862                         if(tNow >= m_lastPublishKadNotes) {
 863                                 if(m_currFileNotes > GetCount()) {
 864                                         m_currFileNotes = 0;
 865                                 }
 866                                 CKnownFile* pCurKnownFile = const_cast<CKnownFile*>(GetFileByIndex(m_currFileNotes));
 867                                 if(pCurKnownFile) {
 868                                         if(pCurKnownFile->PublishNotes()) {
 869                                                 Kademlia::CUInt128 kadFileID;
 870                                                 kadFileID.SetValueBE(pCurKnownFile->GetFileHash().GetHash());
 871                                                 if(Kademlia::CSearchManager::PrepareLookup(Kademlia::CSearch::STORENOTES, true, kadFileID )==NULL)
 872                                                         pCurKnownFile->SetLastPublishTimeKadNotes(0);
 873                                         }
 874                                 }
 875                                 m_currFileNotes++;
 876
 877                                 // even if we did not publish a source, reset the timer so that this list is processed
 878                                 // only every KADEMLIAPUBLISHTIME seconds.
 879                                 m_lastPublishKadNotes = KADEMLIAPUBLISHTIME+tNow;
 880                         }
 881                 }
 882         }
 883 }
 884
 885
 886 void CSharedFileList::AddKeywords(CKnownFile* pFile)
 887 {
 888         m_keywords->AddKeywords(pFile);
 889 }
 890
 891
 892 void CSharedFileList::RemoveKeywords(CKnownFile* pFile)
 893 {
 894         m_keywords->RemoveKeywords(pFile);
 895 }
 896
 897
 898 bool CSharedFileList::RenameFile(CKnownFile* file, const CPath& newName)
 899 {
 900         if (file->IsPartFile()) {
 901                 CPartFile* pfile = dynamic_cast<CPartFile*>(file);
 902
 903                 if (file->GetStatus() != PS_COMPLETING) {
 904                         pfile->SetFileName(newName);
 905                         pfile->SavePartFile();
 906
 907                         Notify_SharedFilesUpdateItem(file);
 908                         Notify_DownloadCtrlUpdateItem(file);
 909
 910                         return true;
 911                 }
 912         } else {
 913                 CPath oldPath = file->GetFilePath().JoinPaths(file->GetFileName());
 914                 CPath newPath = file->GetFilePath().JoinPaths(newName);
 915
 916                 if (CPath::RenameFile(oldPath, newPath)) {
 917                         // Must create a copy of the word list because:
 918                         // 1) it will be reset on SetFileName()
 919                         // 2) we will want to edit it
 920                         Kademlia::WordList oldwords = file->GetKadKeywords();
 921                         file->SetFileName(newName);
 922                         theApp->knownfiles->Save();
 923                         UpdateItem(file);
 924                         RepublishFile(file);
 925
 926                         const Kademlia::WordList& newwords = file->GetKadKeywords();
 927                         Kademlia::WordList::iterator itold;
 928                         Kademlia::WordList::const_iterator itnew;
 929                         // compare keywords in old and new names
 930                         for (itnew = newwords.begin(); itnew != newwords.end(); ++itnew) {
 931                                 for (itold = oldwords.begin(); itold != oldwords.end(); ++itold) {
 932                                         if (*itold == *itnew) {
 933                                                 break;
 934                                         }
 935                                 }
 936                                 if (itold != oldwords.end()) {
 937                                         // Remove keyword from old name which also exist in new name
 938                                         oldwords.erase(itold);
 939                                 } else {
 940                                         // This is a new keyword not present in the old name
 941                                         m_keywords->AddKeyword(*itnew, file);
 942                                 }
 943                         }
 944                         // Remove all remaining old keywords not present in the new name
 945                         for (itold = oldwords.begin(); itold != oldwords.end(); ++itold) {
 946                                 m_keywords->RemoveKeyword(*itold, file);
 947                         }
 948
 949                         Notify_DownloadCtrlUpdateItem(file);
 950                         Notify_SharedFilesUpdateItem(file);
 951
 952                         return true;
 953                 }
 954         }
 955
 956         return false;
 957 }
 958
 959
 960 const CPath* CSharedFileList::GetDirForPublicSharedDirName(const wxString& strSharedDir) const
 961 {
 962         StringPathMap::const_iterator it = m_PublicSharedDirNames.find(strSharedDir);
 963
 964         if (it != m_PublicSharedDirNames.end()) {
 965                 return &(it->second);
 966         } else {
 967                 return NULL;
 968         }
 969 }
 970
 971
 972 wxString CSharedFileList::GetPublicSharedDirName(const CPath& dir)
 973 {
 974         // safety check: is the directory supposed to be shared after all?
 975         if (!IsShared(dir))     {
 976                 wxFAIL;
 977                 return wxT("");
 978         }
 979         // check if the public name for the directory is cached in our Map
 980         StringPathMap::const_iterator it;
 981         for (it = m_PublicSharedDirNames.begin(); it != m_PublicSharedDirNames.end(); ++it) {
 982                 if (it->second.IsSameDir(dir)) {
 983                         // public name for directory was determined earlier
 984                         return it->first;
 985                 }
 986         }
 987
 988         // we store the path separator (forward or back slash) for quick access
 989         wxChar cPathSepa = wxFileName::GetPathSeparator();
 990
 991         // determine and cache the public name for "dir" ...
 992         // We need to use the 'raw' filename, so the receiving client can recognize it.
 993         wxString strDirectoryTmp = dir.GetRaw();
 994         if (strDirectoryTmp.EndsWith(&cPathSepa)) {
 995                 strDirectoryTmp.RemoveLast();
 996         }
 997
 998         wxString strPublicName;
 999         int iPos;
1000         // check all the subdirectories in the path for being shared
1001         // the public name will consist of these concatenated
1002         while ((iPos = strDirectoryTmp.Find( cPathSepa, true )) != wxNOT_FOUND) {
1003                 strPublicName = strDirectoryTmp.Right(strDirectoryTmp.Length() - iPos) + strPublicName;
1004                 strDirectoryTmp.Truncate(iPos);
1005                 if (!IsShared(CPath(strDirectoryTmp)))
1006                         break;
1007         }
1008         if (!strPublicName.IsEmpty()) {
1009                 // remove first path separator ???
1010                 wxASSERT( strPublicName.GetChar(0) == cPathSepa );
1011                 strPublicName = strPublicName.Right(strPublicName.Length() - 1);
1012         } else {
1013                 // must be a rootdirectory on Windos
1014                 wxASSERT( strDirectoryTmp.Length() == 2 );
1015                 strPublicName = strDirectoryTmp;
1016         }
1017         // we have the name, make sure it is unique by appending an index if necessary
1018         if (m_PublicSharedDirNames.find(strPublicName) != m_PublicSharedDirNames.end()) {
1019                 wxString strUniquePublicName;
1020                 for (iPos = 2; ; ++iPos) {
1021                         strUniquePublicName = CFormat(wxT("%s_%i")) % strPublicName % iPos;
1022
1023                         if (m_PublicSharedDirNames.find(strUniquePublicName) == m_PublicSharedDirNames.end()) {
1024                                 AddDebugLogLineN(logClient, CFormat(wxT("Using public name '%s' for directory '%s'"))
1025                                                             % strUniquePublicName
1026                                                             % dir.GetPrintable());
1027                                 m_PublicSharedDirNames.insert(std::pair<wxString, CPath> (strUniquePublicName, dir));
1028                                 return strUniquePublicName;
1029                         }
1030                         // This is from eMule and it checks if there are more than 200 shared folders with the same public name.
1031                         // The condition can be true if many shared subfolders with the same name exist in folders that are not
1032                         // shared. So they get the names of each shared subfolders concatenated. But those might all be the same!
1033                         // It's here for safety reasons so we should not run out of memory.
1034                         else if (iPos > 200)  // Only 200 identical names are indexed.
1035                         {
1036                                 wxASSERT( false );
1037                                 return wxT("");
1038                         }
1039                 }
1040         } else {
1041                 AddDebugLogLineN(logClient, CFormat(wxT("Using public name '%s' for directory '%s'")) % strPublicName % dir.GetPrintable());
1042                 m_PublicSharedDirNames.insert(std::pair<wxString, CPath> (strPublicName, dir));
1043                 return strPublicName;
1044         }
1045 }
1046
1047
1048 bool CSharedFileList::IsShared(const CPath& path) const
1049 {
1050         if( path.IsDir(CPath::exists) ) {
1051                 // check if it's a shared folder
1052                 const unsigned folderCount = theApp->glob_prefs->shareddir_list.size();
1053                 for (unsigned i = 0; i < folderCount; ++i) {
1054                         if (path.IsSameDir(theApp->glob_prefs->shareddir_list[i])) {
1055                                 return true;
1056                         }
1057                 }
1058
1059                 // check if it's one of the categories folders (category 0 = incoming)
1060                 for (unsigned i = 0; i < theApp->glob_prefs->GetCatCount(); ++i) {
1061                         if (path.IsSameDir(theApp->glob_prefs->GetCategory(i)->path)) {
1062                                 return true;
1063                         }
1064                 }
1065         }
1066
1067         return false;
1068 }
1069
1070
1071 void CSharedFileList::CheckAICHHashes(const std::list<CAICHHash>& hashes)
1072 {
1073         wxMutexLocker locker(list_mut);
1074
1075         // Now we check that all files which are in the sharedfilelist have a
1076         // corresponding hash in our list. Those how don't are queued for hashing.
1077         CKnownFileMap::iterator it = m_Files_map.begin();
1078         for (; it != m_Files_map.end(); ++it) {
1079                 const CKnownFile* file = it->second;
1080
1081                 if (file->IsPartFile() == false) {
1082                         CAICHHashSet* hashset = file->GetAICHHashset();
1083
1084                         if (hashset->GetStatus() == AICH_HASHSETCOMPLETE) {
1085                                 if (std::find(hashes.begin(), hashes.end(), hashset->GetMasterHash()) != hashes.end()) {
1086                                         continue;
1087                                 }
1088                         }
1089
1090                         hashset->SetStatus(AICH_ERROR);
1091
1092                         CThreadScheduler::AddTask(new CHashingTask(file));
1093                 }
1094         }
1095
1096 }
1097
1098 // File_checked_for_headers