xbmc/TextureCache.cpp

   1 /*
   2  *  Copyright (C) 2005-2018 Team Kodi
   3  *  This file is part of Kodi - https://kodi.tv
   4  *
   5  *  SPDX-License-Identifier: GPL-2.0-or-later
   6  *  See LICENSES/README.md for more information.
   7  */
   8
   9 #include "TextureCache.h"
  10
  11 #include "ServiceBroker.h"
  12 #include "TextureCacheJob.h"
  13 #include "URL.h"
  14 #include "commons/ilog.h"
  15 #include "dialogs/GUIDialogProgress.h"
  16 #include "filesystem/File.h"
  17 #include "filesystem/IFileTypes.h"
  18 #include "guilib/GUIComponent.h"
  19 #include "guilib/GUIWindowManager.h"
  20 #include "guilib/Texture.h"
  21 #include "imagefiles/ImageCacheCleaner.h"
  22 #include "imagefiles/ImageFileURL.h"
  23 #include "profiles/ProfileManager.h"
  24 #include "settings/SettingsComponent.h"
  25 #include "utils/Crc32.h"
  26 #include "utils/Job.h"
  27 #include "utils/JobManager.h"
  28 #include "utils/StringUtils.h"
  29 #include "utils/URIUtils.h"
  30 #include "utils/log.h"
  31
  32 #include <chrono>
  33 #include <exception>
  34 #include <mutex>
  35 #include <optional>
  36 #include <string.h>
  37
  38 using namespace XFILE;
  39 using namespace std::chrono_literals;
  40
  41 CTextureCache::CTextureCache()
  42   : CJobQueue(false, 1, CJob::PRIORITY_LOW_PAUSABLE), m_cleanTimer{[this]() { CleanTimer(); }}
  43 {
  44 }
  45
  46 CTextureCache::~CTextureCache() = default;
  47
  48 void CTextureCache::Initialize()
  49 {
  50   m_cleanTimer.Start(60s);
  51   std::unique_lock<CCriticalSection> lock(m_databaseSection);
  52   if (!m_database.IsOpen())
  53     m_database.Open();
  54 }
  55
  56 void CTextureCache::Deinitialize()
  57 {
  58   CancelJobs();
  59
  60   std::unique_lock<CCriticalSection> lock(m_databaseSection);
  61   m_database.Close();
  62 }
  63
  64 bool CTextureCache::IsCachedImage(const std::string &url) const
  65 {
  66   if (url.empty())
  67     return false;
  68
  69   if (!CURL::IsFullPath(url))
  70     return true;
  71
  72   const std::shared_ptr<CProfileManager> profileManager = CServiceBroker::GetSettingsComponent()->GetProfileManager();
  73
  74   return URIUtils::PathHasParent(url, "special://skin", true) ||
  75       URIUtils::PathHasParent(url, "special://temp", true) ||
  76       URIUtils::PathHasParent(url, "resource://", true) ||
  77       URIUtils::PathHasParent(url, "androidapp://", true)   ||
  78       URIUtils::PathHasParent(url, profileManager->GetThumbnailsFolder(), true);
  79 }
  80
  81 bool CTextureCache::HasCachedImage(const std::string &url)
  82 {
  83   CTextureDetails details;
  84   std::string cachedImage(GetCachedImage(url, details));
  85   return (!cachedImage.empty() && cachedImage != url);
  86 }
  87
  88 std::string CTextureCache::GetCachedImage(const std::string &image, CTextureDetails &details, bool trackUsage)
  89 {
  90   std::string url = IMAGE_FILES::ToCacheKey(image);
  91   if (url.empty())
  92     return "";
  93   if (IsCachedImage(url))
  94     return url;
  95
  96   // lookup the item in the database
  97   if (GetCachedTexture(url, details))
  98   {
  99     if (details.file.empty())
 100       return {};
 101
 102     if (trackUsage)
 103       IncrementUseCount(details);
 104     return GetCachedPath(details.file);
 105   }
 106   return "";
 107 }
 108
 109 std::string CTextureCache::CheckCachedImage(const std::string &url, bool &needsRecaching)
 110 {
 111   CTextureDetails details;
 112   std::string path(GetCachedImage(url, details, true));
 113   needsRecaching = !details.hash.empty();
 114   if (!path.empty())
 115     return path;
 116   return "";
 117 }
 118
 119 void CTextureCache::BackgroundCacheImage(const std::string &url)
 120 {
 121   if (url.empty())
 122     return;
 123
 124   CTextureDetails details;
 125   std::string path(GetCachedImage(url, details));
 126   if (!path.empty() && details.hash.empty())
 127     return; // image is already cached and doesn't need to be checked further
 128
 129   path = IMAGE_FILES::ToCacheKey(url);
 130   if (path.empty())
 131     return;
 132
 133   // needs (re)caching
 134   AddJob(new CTextureCacheJob(path, details.hash));
 135 }
 136
 137 bool CTextureCache::StartCacheImage(const std::string& image)
 138 {
 139   std::unique_lock<CCriticalSection> lock(m_processingSection);
 140   std::set<std::string>::iterator i = m_processinglist.find(image);
 141   if (i == m_processinglist.end())
 142   {
 143     m_processinglist.insert(image);
 144     return true;
 145   }
 146   return false;
 147 }
 148
 149 std::string CTextureCache::CacheImage(const std::string& image,
 150                                       std::unique_ptr<CTexture>* texture /*= nullptr*/,
 151                                       CTextureDetails* details /*= nullptr*/)
 152 {
 153   std::string url = IMAGE_FILES::ToCacheKey(image);
 154   if (url.empty())
 155     return "";
 156
 157   std::unique_lock<CCriticalSection> lock(m_processingSection);
 158   if (m_processinglist.find(url) == m_processinglist.end())
 159   {
 160     m_processinglist.insert(url);
 161     lock.unlock();
 162     // cache the texture directly
 163     CTextureCacheJob job(url);
 164     bool success = job.CacheTexture(texture);
 165     OnCachingComplete(success, &job);
 166     if (success && details)
 167       *details = job.m_details;
 168     return success ? GetCachedPath(job.m_details.file) : "";
 169   }
 170   lock.unlock();
 171
 172   // wait for currently processing job to end.
 173   while (true)
 174   {
 175     m_completeEvent.Wait(1000ms);
 176     {
 177       std::unique_lock<CCriticalSection> lock(m_processingSection);
 178       if (m_processinglist.find(url) == m_processinglist.end())
 179         break;
 180     }
 181   }
 182   CTextureDetails tempDetails;
 183   if (!details)
 184     details = &tempDetails;
 185
 186   std::string cachedpath = GetCachedImage(url, *details, true);
 187   if (!cachedpath.empty())
 188   {
 189     if (texture)
 190       *texture = CTexture::LoadFromFile(cachedpath, 0, 0);
 191   }
 192   else
 193   {
 194     CLog::Log(LOGDEBUG, "CTextureCache::{} - Return NULL texture because cache is not ready",
 195               __FUNCTION__);
 196   }
 197
 198   return cachedpath;
 199 }
 200
 201 bool CTextureCache::CacheImage(const std::string &image, CTextureDetails &details)
 202 {
 203   std::string path = GetCachedImage(image, details);
 204   if (path.empty()) // not cached
 205     path = CacheImage(image, NULL, &details);
 206
 207   return !path.empty();
 208 }
 209
 210 void CTextureCache::ClearCachedImage(const std::string& image, bool deleteSource /*= false */)
 211 {
 212   //! @todo This can be removed when the texture cache covers everything.
 213   const std::string url = IMAGE_FILES::ToCacheKey(image);
 214   std::string path = deleteSource ? url : "";
 215   std::string cachedFile;
 216   if (ClearCachedTexture(url, cachedFile))
 217     path = GetCachedPath(cachedFile);
 218   if (CFile::Exists(path))
 219     CFile::Delete(path);
 220   path = URIUtils::ReplaceExtension(path, ".dds");
 221   if (CFile::Exists(path))
 222     CFile::Delete(path);
 223 }
 224
 225 bool CTextureCache::ClearCachedImage(int id)
 226 {
 227   std::string cachedFile;
 228   if (ClearCachedTexture(id, cachedFile))
 229   {
 230     cachedFile = GetCachedPath(cachedFile);
 231     if (CFile::Exists(cachedFile))
 232       CFile::Delete(cachedFile);
 233     cachedFile = URIUtils::ReplaceExtension(cachedFile, ".dds");
 234     if (CFile::Exists(cachedFile))
 235       CFile::Delete(cachedFile);
 236     return true;
 237   }
 238   return false;
 239 }
 240
 241 bool CTextureCache::GetCachedTexture(const std::string &url, CTextureDetails &details)
 242 {
 243   std::unique_lock<CCriticalSection> lock(m_databaseSection);
 244   return m_database.GetCachedTexture(url, details);
 245 }
 246
 247 bool CTextureCache::AddCachedTexture(const std::string &url, const CTextureDetails &details)
 248 {
 249   std::unique_lock<CCriticalSection> lock(m_databaseSection);
 250   return m_database.AddCachedTexture(url, details);
 251 }
 252
 253 void CTextureCache::IncrementUseCount(const CTextureDetails &details)
 254 {
 255   static const size_t count_before_update = 100;
 256   std::unique_lock<CCriticalSection> lock(m_useCountSection);
 257   m_useCounts.reserve(count_before_update);
 258   m_useCounts.push_back(details);
 259   if (m_useCounts.size() >= count_before_update)
 260   {
 261     AddJob(new CTextureUseCountJob(m_useCounts));
 262     m_useCounts.clear();
 263   }
 264 }
 265
 266 bool CTextureCache::SetCachedTextureValid(const std::string &url, bool updateable)
 267 {
 268   std::unique_lock<CCriticalSection> lock(m_databaseSection);
 269   return m_database.SetCachedTextureValid(url, updateable);
 270 }
 271
 272 bool CTextureCache::ClearCachedTexture(const std::string &url, std::string &cachedURL)
 273 {
 274   std::unique_lock<CCriticalSection> lock(m_databaseSection);
 275   return m_database.ClearCachedTexture(url, cachedURL);
 276 }
 277
 278 bool CTextureCache::ClearCachedTexture(int id, std::string &cachedURL)
 279 {
 280   std::unique_lock<CCriticalSection> lock(m_databaseSection);
 281   return m_database.ClearCachedTexture(id, cachedURL);
 282 }
 283
 284 std::string CTextureCache::GetCacheFile(const std::string &url)
 285 {
 286   auto crc = Crc32::ComputeFromLowerCase(url);
 287   std::string hex = StringUtils::Format("{:08x}", crc);
 288   std::string hash = StringUtils::Format("{}/{}", hex[0], hex.c_str());
 289   return hash;
 290 }
 291
 292 std::string CTextureCache::GetCachedPath(const std::string &file)
 293 {
 294   const std::shared_ptr<CProfileManager> profileManager = CServiceBroker::GetSettingsComponent()->GetProfileManager();
 295
 296   return URIUtils::AddFileToFolder(profileManager->GetThumbnailsFolder(), file);
 297 }
 298
 299 void CTextureCache::OnCachingComplete(bool success, CTextureCacheJob *job)
 300 {
 301   if (success)
 302   {
 303     if (job->m_details.hashRevalidated)
 304       SetCachedTextureValid(job->m_url, job->m_details.updateable);
 305     else
 306       AddCachedTexture(job->m_url, job->m_details);
 307   }
 308
 309   { // remove from our processing list
 310     std::unique_lock<CCriticalSection> lock(m_processingSection);
 311     std::set<std::string>::iterator i = m_processinglist.find(job->m_url);
 312     if (i != m_processinglist.end())
 313       m_processinglist.erase(i);
 314   }
 315
 316   m_completeEvent.Set();
 317 }
 318
 319 void CTextureCache::OnJobComplete(unsigned int jobID, bool success, CJob *job)
 320 {
 321   if (strcmp(job->GetType(), kJobTypeCacheImage) == 0)
 322     OnCachingComplete(success, static_cast<CTextureCacheJob*>(job));
 323   return CJobQueue::OnJobComplete(jobID, success, job);
 324 }
 325
 326 bool CTextureCache::Export(const std::string &image, const std::string &destination, bool overwrite)
 327 {
 328   CTextureDetails details;
 329   std::string cachedImage(GetCachedImage(image, details));
 330   if (!cachedImage.empty())
 331   {
 332     std::string dest = destination + URIUtils::GetExtension(cachedImage);
 333     if (overwrite || !CFile::Exists(dest))
 334     {
 335       if (CFile::Copy(cachedImage, dest))
 336         return true;
 337       CLog::Log(LOGERROR, "{} failed exporting '{}' to '{}'", __FUNCTION__, cachedImage, dest);
 338     }
 339   }
 340   return false;
 341 }
 342
 343 bool CTextureCache::Export(const std::string &image, const std::string &destination)
 344 {
 345   CTextureDetails details;
 346   std::string cachedImage(GetCachedImage(image, details));
 347   if (!cachedImage.empty())
 348   {
 349     if (CFile::Copy(cachedImage, destination))
 350       return true;
 351     CLog::Log(LOGERROR, "{} failed exporting '{}' to '{}'", __FUNCTION__, cachedImage, destination);
 352   }
 353   return false;
 354 }
 355
 356 bool CTextureCache::CleanAllUnusedImages()
 357 {
 358   if (m_cleaningInProgress.test_and_set())
 359     return false;
 360
 361   auto progress = CServiceBroker::GetGUI()->GetWindowManager().GetWindow<CGUIDialogProgress>(
 362       WINDOW_DIALOG_PROGRESS);
 363   if (progress)
 364   {
 365     progress->SetHeading(CVariant{14281}); //"Clean image cache"
 366     progress->SetText(CVariant{313}); //"Cleaning database"
 367     progress->SetPercentage(0);
 368     progress->Open();
 369     progress->ShowProgressBar(true);
 370   }
 371
 372   bool failure = false;
 373   CServiceBroker::GetJobManager()->Submit([this, progress, &failure]()
 374                                           { failure = CleanAllUnusedImagesJob(progress); });
 375
 376   // Wait for clean to complete or be canceled, but render every 10ms so that the
 377   // pointer movements work on dialog even when clean is reporting progress infrequently
 378   if (progress)
 379     progress->Wait();
 380
 381   m_cleaningInProgress.clear();
 382   return !failure;
 383 }
 384
 385 bool CTextureCache::CleanAllUnusedImagesJob(CGUIDialogProgress* progress)
 386 {
 387   auto cleaner = IMAGE_FILES::CImageCacheCleaner::Create();
 388   if (!cleaner)
 389   {
 390     if (progress)
 391       progress->Close();
 392     return false;
 393   }
 394
 395   const unsigned int cleanAmount = 1000000;
 396   const auto result = cleaner->ScanOldestCache(cleanAmount);
 397   if (progress && progress->IsCanceled())
 398   {
 399     progress->Close();
 400     return false;
 401   }
 402
 403   const auto total = result.imagesToClean.size();
 404   unsigned int current = 0;
 405   for (const auto& image : result.imagesToClean)
 406   {
 407     ClearCachedImage(image);
 408     if (progress)
 409     {
 410       if (progress->IsCanceled())
 411       {
 412         progress->Close();
 413         return false;
 414       }
 415       int percentage = static_cast<unsigned long long>(current) * 100 / total;
 416       if (progress->GetPercentage() != percentage)
 417       {
 418         progress->SetPercentage(percentage);
 419         progress->Progress();
 420       }
 421       current++;
 422     }
 423   }
 424
 425   if (progress)
 426     progress->Close();
 427   return true;
 428 }
 429
 430 void CTextureCache::CleanTimer()
 431 {
 432   CServiceBroker::GetJobManager()->Submit(
 433       [this]()
 434       {
 435         auto next = m_cleaningInProgress.test_and_set() ? std::chrono::hours(1) : ScanOldestCache();
 436         m_cleaningInProgress.clear();
 437         m_cleanTimer.Start(next);
 438       },
 439       CJob::PRIORITY_LOW_PAUSABLE);
 440 }
 441
 442 std::chrono::milliseconds CTextureCache::ScanOldestCache()
 443 {
 444   auto cleaner = IMAGE_FILES::CImageCacheCleaner::Create();
 445   if (!cleaner)
 446     return std::chrono::hours(1);
 447
 448   const unsigned int cleanAmount = 1000;
 449   const auto result = cleaner->ScanOldestCache(cleanAmount);
 450   for (const auto& image : result.imagesToClean)
 451   {
 452     ClearCachedImage(image);
 453   }
 454
 455   // update in the next 6 - 48 hours depending on number of items processed
 456   const auto minTime = 6;
 457   const auto maxTime = 24;
 458   const auto zeroItemTime = 48;
 459   const unsigned int next =
 460       result.processedCount == 0
 461           ? zeroItemTime
 462           : minTime + (maxTime - minTime) *
 463                           (1 - (static_cast<float>(result.processedCount) / cleanAmount));
 464   CLog::LogF(LOGDEBUG, "scheduling the next image cache cleaning in {} hours", next);
 465   return std::chrono::hours(next);
 466 }