xbmc/filesystem/DirectoryCache.cpp

   1 /*
   2  *  Copyright (C) 2005-2018 Team Kodi
   3  *  This file is part of Kodi - https://kodi.tv
   4  *
   5  *  SPDX-License-Identifier: GPL-2.0-or-later
   6  *  See LICENSES/README.md for more information.
   7  */
   8
   9 #include "DirectoryCache.h"
  10
  11 #include "Directory.h"
  12 #include "FileItem.h"
  13 #include "URL.h"
  14 #include "utils/StringUtils.h"
  15 #include "utils/URIUtils.h"
  16 #include "utils/log.h"
  17
  18 #include <algorithm>
  19 #include <climits>
  20 #include <mutex>
  21
  22 // Maximum number of directories to keep in our cache
  23 #define MAX_CACHED_DIRS 50
  24
  25 using namespace XFILE;
  26
  27 CDirectoryCache::CDir::CDir(DIR_CACHE_TYPE cacheType) : m_Items(std::make_unique<CFileItemList>())
  28 {
  29   m_cacheType = cacheType;
  30   m_lastAccess = 0;
  31   m_Items->SetIgnoreURLOptions(true);
  32   m_Items->SetFastLookup(true);
  33 }
  34
  35 CDirectoryCache::CDir::~CDir() = default;
  36
  37 void CDirectoryCache::CDir::SetLastAccess(unsigned int &accessCounter)
  38 {
  39   m_lastAccess = accessCounter++;
  40 }
  41
  42 CDirectoryCache::CDirectoryCache(void)
  43 {
  44   m_accessCounter = 0;
  45 #ifdef _DEBUG
  46   m_cacheHits = 0;
  47   m_cacheMisses = 0;
  48 #endif
  49 }
  50
  51 CDirectoryCache::~CDirectoryCache(void) = default;
  52
  53 bool CDirectoryCache::GetDirectory(const std::string& strPath, CFileItemList &items, bool retrieveAll)
  54 {
  55   std::unique_lock<CCriticalSection> lock(m_cs);
  56
  57   // Get rid of any URL options, else the compare may be wrong
  58   std::string storedPath = CURL(strPath).GetWithoutOptions();
  59   URIUtils::RemoveSlashAtEnd(storedPath);
  60
  61   auto i = m_cache.find(storedPath);
  62   if (i != m_cache.end())
  63   {
  64     CDir& dir = i->second;
  65     if (dir.m_cacheType == XFILE::DIR_CACHE_ALWAYS ||
  66         (dir.m_cacheType == XFILE::DIR_CACHE_ONCE && retrieveAll))
  67     {
  68       items.Copy(*dir.m_Items);
  69       dir.SetLastAccess(m_accessCounter);
  70 #ifdef _DEBUG
  71       m_cacheHits+=items.Size();
  72 #endif
  73       return true;
  74     }
  75   }
  76   return false;
  77 }
  78
  79 void CDirectoryCache::SetDirectory(const std::string& strPath, const CFileItemList &items, DIR_CACHE_TYPE cacheType)
  80 {
  81   if (cacheType == DIR_CACHE_NEVER)
  82     return; // nothing to do
  83
  84   // caches the given directory using a copy of the items, rather than the items
  85   // themselves.  The reason we do this is because there is often some further
  86   // processing on the items (stacking, transparent rars/zips for instance) that
  87   // alters the URL of the items.  If we shared the pointers, we'd have problems
  88   // as the URLs in the cache would have changed, so things such as
  89   // CDirectoryCache::FileExists() would fail for files that really do exist (just their
  90   // URL's have been altered).  This is called from CFile::Exists() which causes
  91   // all sorts of hassles.
  92   // IDEALLY, any further processing on the item would actually create a new item
  93   // instead of altering it, but we can't really enforce that in an easy way, so
  94   // this is the best solution for now.
  95   std::unique_lock<CCriticalSection> lock(m_cs);
  96
  97   // Get rid of any URL options, else the compare may be wrong
  98   std::string storedPath = CURL(strPath).GetWithoutOptions();
  99   URIUtils::RemoveSlashAtEnd(storedPath);
 100
 101   ClearDirectory(storedPath);
 102
 103   CheckIfFull();
 104
 105   CDir dir(cacheType);
 106   dir.m_Items->Copy(items);
 107   dir.SetLastAccess(m_accessCounter);
 108   m_cache.emplace(storedPath, std::move(dir));
 109 }
 110
 111 void CDirectoryCache::ClearFile(const std::string& strFile)
 112 {
 113   // Get rid of any URL options, else the compare may be wrong
 114   std::string strFile2 = CURL(strFile).GetWithoutOptions();
 115   URIUtils::RemoveSlashAtEnd(strFile2);
 116
 117   ClearDirectory(URIUtils::GetDirectory(strFile2));
 118 }
 119
 120 void CDirectoryCache::ClearDirectory(const std::string& strPath)
 121 {
 122   std::unique_lock<CCriticalSection> lock(m_cs);
 123
 124   // Get rid of any URL options, else the compare may be wrong
 125   std::string storedPath = CURL(strPath).GetWithoutOptions();
 126   URIUtils::RemoveSlashAtEnd(storedPath);
 127
 128   m_cache.erase(storedPath);
 129 }
 130
 131 void CDirectoryCache::ClearSubPaths(const std::string& strPath)
 132 {
 133   std::unique_lock<CCriticalSection> lock(m_cs);
 134
 135   // Get rid of any URL options, else the compare may be wrong
 136   std::string storedPath = CURL(strPath).GetWithoutOptions();
 137
 138   auto i = m_cache.begin();
 139   while (i != m_cache.end())
 140   {
 141     if (URIUtils::PathHasParent(i->first, storedPath))
 142       m_cache.erase(i++);
 143     else
 144       i++;
 145   }
 146 }
 147
 148 void CDirectoryCache::AddFile(const std::string& strFile)
 149 {
 150   std::unique_lock<CCriticalSection> lock(m_cs);
 151
 152   // Get rid of any URL options, else the compare may be wrong
 153   std::string strPath = URIUtils::GetDirectory(CURL(strFile).GetWithoutOptions());
 154   URIUtils::RemoveSlashAtEnd(strPath);
 155
 156   auto i = m_cache.find(strPath);
 157   if (i != m_cache.end())
 158   {
 159     CDir& dir = i->second;
 160     CFileItemPtr item(new CFileItem(strFile, false));
 161     dir.m_Items->Add(item);
 162     dir.SetLastAccess(m_accessCounter);
 163   }
 164 }
 165
 166 bool CDirectoryCache::FileExists(const std::string& strFile, bool& bInCache)
 167 {
 168   std::unique_lock<CCriticalSection> lock(m_cs);
 169   bInCache = false;
 170
 171   // Get rid of any URL options, else the compare may be wrong
 172   std::string strPath = CURL(strFile).GetWithoutOptions();
 173   URIUtils::RemoveSlashAtEnd(strPath);
 174   std::string storedPath = URIUtils::GetDirectory(strPath);
 175   URIUtils::RemoveSlashAtEnd(storedPath);
 176
 177   auto i = m_cache.find(storedPath);
 178   if (i != m_cache.end())
 179   {
 180     bInCache = true;
 181     CDir& dir = i->second;
 182     dir.SetLastAccess(m_accessCounter);
 183 #ifdef _DEBUG
 184     m_cacheHits++;
 185 #endif
 186     return (URIUtils::PathEquals(strPath, storedPath) || dir.m_Items->Contains(strFile));
 187   }
 188 #ifdef _DEBUG
 189   m_cacheMisses++;
 190 #endif
 191   return false;
 192 }
 193
 194 void CDirectoryCache::Clear()
 195 {
 196   // this routine clears everything
 197   std::unique_lock<CCriticalSection> lock(m_cs);
 198   m_cache.clear();
 199 }
 200
 201 void CDirectoryCache::InitCache(const std::set<std::string>& dirs)
 202 {
 203   for (const std::string& strDir : dirs)
 204   {
 205     CFileItemList items;
 206     CDirectory::GetDirectory(strDir, items, "", DIR_FLAG_NO_FILE_DIRS);
 207     items.Clear();
 208   }
 209 }
 210
 211 void CDirectoryCache::ClearCache(std::set<std::string>& dirs)
 212 {
 213   auto i = m_cache.begin();
 214   while (i != m_cache.end())
 215   {
 216     if (dirs.find(i->first) != dirs.end())
 217       m_cache.erase(i++);
 218     else
 219       i++;
 220   }
 221 }
 222
 223 void CDirectoryCache::CheckIfFull()
 224 {
 225   std::unique_lock<CCriticalSection> lock(m_cs);
 226
 227   // find the last accessed folder, and remove if the number of cached folders is too many
 228   auto lastAccessed = m_cache.end();
 229   unsigned int numCached = 0;
 230   for (auto i = m_cache.begin(); i != m_cache.end(); i++)
 231   {
 232     // ensure dirs that are always cached aren't cleared
 233     if (i->second.m_cacheType != DIR_CACHE_ALWAYS)
 234     {
 235       if (lastAccessed == m_cache.end() ||
 236           i->second.GetLastAccess() < lastAccessed->second.GetLastAccess())
 237         lastAccessed = i;
 238       numCached++;
 239     }
 240   }
 241   if (lastAccessed != m_cache.end() && numCached >= MAX_CACHED_DIRS)
 242     m_cache.erase(lastAccessed);
 243 }
 244
 245 #ifdef _DEBUG
 246 void CDirectoryCache::PrintStats() const
 247 {
 248   std::unique_lock<CCriticalSection> lock(m_cs);
 249   CLog::Log(LOGDEBUG, "{} - total of {} cache hits, and {} cache misses", __FUNCTION__, m_cacheHits,
 250             m_cacheMisses);
 251   // run through and find the oldest and the number of items cached
 252   unsigned int oldest = UINT_MAX;
 253   unsigned int numItems = 0;
 254   unsigned int numDirs = 0;
 255   for (auto i = m_cache.begin(); i != m_cache.end(); i++)
 256   {
 257     const CDir& dir = i->second;
 258     oldest = std::min(oldest, dir.GetLastAccess());
 259     numItems += dir.m_Items->Size();
 260     numDirs++;
 261   }
 262   CLog::Log(LOGDEBUG, "{} - {} folders cached, with {} items total.  Oldest is {}, current is {}",
 263             __FUNCTION__, numDirs, numItems, oldest, m_accessCounter);
 264 }
 265 #endif