xbmc/filesystem/MultiPathDirectory.cpp

   1 /*
   2  *  Copyright (C) 2005-2018 Team Kodi
   3  *  This file is part of Kodi - https://kodi.tv
   4  *
   5  *  SPDX-License-Identifier: GPL-2.0-or-later
   6  *  See LICENSES/README.md for more information.
   7  */
   8
   9 #include "MultiPathDirectory.h"
  10
  11 #include "Directory.h"
  12 #include "FileItem.h"
  13 #include "ServiceBroker.h"
  14 #include "URL.h"
  15 #include "Util.h"
  16 #include "dialogs/GUIDialogProgress.h"
  17 #include "guilib/GUIComponent.h"
  18 #include "guilib/GUIWindowManager.h"
  19 #include "threads/SystemClock.h"
  20 #include "utils/StringUtils.h"
  21 #include "utils/URIUtils.h"
  22 #include "utils/Variant.h"
  23 #include "utils/log.h"
  24
  25 using namespace XFILE;
  26
  27 using namespace std::chrono_literals;
  28
  29 //
  30 // multipath://{path1}/{path2}/{path3}/.../{path-N}
  31 //
  32 // unlike the older virtualpath:// protocol, sub-folders are combined together into a new
  33 // multipath:// style url.
  34 //
  35
  36 CMultiPathDirectory::CMultiPathDirectory() = default;
  37
  38 CMultiPathDirectory::~CMultiPathDirectory() = default;
  39
  40 bool CMultiPathDirectory::GetDirectory(const CURL& url, CFileItemList &items)
  41 {
  42   CLog::Log(LOGDEBUG, "CMultiPathDirectory::GetDirectory({})", url.GetRedacted());
  43
  44   std::vector<std::string> vecPaths;
  45   if (!GetPaths(url, vecPaths))
  46     return false;
  47
  48   XbmcThreads::EndTime<> progressTime(3000ms); // 3 seconds before showing progress bar
  49   CGUIDialogProgress* dlgProgress = NULL;
  50
  51   unsigned int iFailures = 0;
  52   for (unsigned int i = 0; i < vecPaths.size(); ++i)
  53   {
  54     // show the progress dialog if we have passed our time limit
  55     if (progressTime.IsTimePast() && !dlgProgress)
  56     {
  57       dlgProgress = CServiceBroker::GetGUI()->GetWindowManager().GetWindow<CGUIDialogProgress>(WINDOW_DIALOG_PROGRESS);
  58       if (dlgProgress)
  59       {
  60         dlgProgress->SetHeading(CVariant{15310});
  61         dlgProgress->SetLine(0, CVariant{15311});
  62         dlgProgress->SetLine(1, CVariant{""});
  63         dlgProgress->SetLine(2, CVariant{""});
  64         dlgProgress->Open();
  65         dlgProgress->ShowProgressBar(true);
  66         dlgProgress->SetProgressMax((int)vecPaths.size()*2);
  67         dlgProgress->Progress();
  68       }
  69     }
  70     if (dlgProgress)
  71     {
  72       CURL url(vecPaths[i]);
  73       dlgProgress->SetLine(1, CVariant{url.GetWithoutUserDetails()});
  74       dlgProgress->SetProgressAdvance();
  75       dlgProgress->Progress();
  76     }
  77
  78     CFileItemList tempItems;
  79     CLog::Log(LOGDEBUG, "Getting Directory ({})", CURL::GetRedacted(vecPaths[i]));
  80     if (CDirectory::GetDirectory(vecPaths[i], tempItems, m_strFileMask, m_flags))
  81       items.Append(tempItems);
  82     else
  83     {
  84       CLog::Log(LOGERROR, "Error Getting Directory ({})", CURL::GetRedacted(vecPaths[i]));
  85       iFailures++;
  86     }
  87
  88     if (dlgProgress)
  89     {
  90       dlgProgress->SetProgressAdvance();
  91       dlgProgress->Progress();
  92     }
  93   }
  94
  95   if (dlgProgress)
  96     dlgProgress->Close();
  97
  98   if (iFailures == vecPaths.size())
  99     return false;
 100
 101   // merge like-named folders into a sub multipath:// style url
 102   MergeItems(items);
 103
 104   return true;
 105 }
 106
 107 bool CMultiPathDirectory::Exists(const CURL& url)
 108 {
 109   CLog::Log(LOGDEBUG, "Testing Existence ({})", url.GetRedacted());
 110
 111   std::vector<std::string> vecPaths;
 112   if (!GetPaths(url, vecPaths))
 113     return false;
 114
 115   for (unsigned int i = 0; i < vecPaths.size(); ++i)
 116   {
 117     CLog::Log(LOGDEBUG, "Testing Existence ({})", CURL::GetRedacted(vecPaths[i]));
 118     if (CDirectory::Exists(vecPaths[i]))
 119       return true;
 120   }
 121   return false;
 122 }
 123
 124 bool CMultiPathDirectory::Remove(const CURL& url)
 125 {
 126   std::vector<std::string> vecPaths;
 127   if (!GetPaths(url, vecPaths))
 128     return false;
 129
 130   bool success = false;
 131   for (unsigned int i = 0; i < vecPaths.size(); ++i)
 132   {
 133     if (CDirectory::Remove(vecPaths[i]))
 134       success = true;
 135   }
 136   return success;
 137 }
 138
 139 std::string CMultiPathDirectory::GetFirstPath(const std::string &strPath)
 140 {
 141   size_t pos = strPath.find('/', 12);
 142   if (pos != std::string::npos)
 143     return CURL::Decode(strPath.substr(12, pos - 12));
 144   return "";
 145 }
 146
 147 bool CMultiPathDirectory::GetPaths(const CURL& url, std::vector<std::string>& vecPaths)
 148 {
 149   const std::string pathToUrl(url.Get());
 150   return GetPaths(pathToUrl, vecPaths);
 151 }
 152
 153 bool CMultiPathDirectory::GetPaths(const std::string& path, std::vector<std::string>& paths)
 154 {
 155   paths.clear();
 156
 157   // remove multipath:// from path and any trailing / (so that the last path doesn't get any more than it originally had)
 158   std::string path1 = path.substr(12);
 159   path1.erase(path1.find_last_not_of('/')+1);
 160
 161   // split on "/"
 162   std::vector<std::string> temp = StringUtils::Split(path1, '/');
 163   if (temp.empty())
 164     return false;
 165
 166   // URL decode each item
 167   paths.resize(temp.size());
 168   std::transform(temp.begin(), temp.end(), paths.begin(), CURL::Decode);
 169   return true;
 170 }
 171
 172 bool CMultiPathDirectory::HasPath(const std::string& strPath, const std::string& strPathToFind)
 173 {
 174   // remove multipath:// from path and any trailing / (so that the last path doesn't get any more than it originally had)
 175   std::string strPath1 = strPath.substr(12);
 176   URIUtils::RemoveSlashAtEnd(strPath1);
 177
 178   // split on "/"
 179   std::vector<std::string> vecTemp = StringUtils::Split(strPath1, '/');
 180   if (vecTemp.empty())
 181     return false;
 182
 183   // check each item
 184   for (unsigned int i = 0; i < vecTemp.size(); i++)
 185   {
 186     if (CURL::Decode(vecTemp[i]) == strPathToFind)
 187       return true;
 188   }
 189   return false;
 190 }
 191
 192 std::string CMultiPathDirectory::ConstructMultiPath(const CFileItemList& items, const std::vector<int> &stack)
 193 {
 194   // we replace all instances of comma's with double comma's, then separate
 195   // the paths using " , "
 196   //CLog::Log(LOGDEBUG, "Building multipath");
 197   std::string newPath = "multipath://";
 198   //CLog::Log(LOGDEBUG, "-- adding path: {}", strPath);
 199   for (unsigned int i = 0; i < stack.size(); ++i)
 200     AddToMultiPath(newPath, items[stack[i]]->GetPath());
 201
 202   //CLog::Log(LOGDEBUG, "Final path: {}", newPath);
 203   return newPath;
 204 }
 205
 206 void CMultiPathDirectory::AddToMultiPath(std::string& strMultiPath, const std::string& strPath)
 207 {
 208   URIUtils::AddSlashAtEnd(strMultiPath);
 209   //CLog::Log(LOGDEBUG, "-- adding path: {}", strPath);
 210   strMultiPath += CURL::Encode(strPath);
 211   strMultiPath += "/";
 212 }
 213
 214 std::string CMultiPathDirectory::ConstructMultiPath(const std::vector<std::string> &vecPaths)
 215 {
 216   // we replace all instances of comma's with double comma's, then separate
 217   // the paths using " , "
 218   //CLog::Log(LOGDEBUG, "Building multipath");
 219   std::string newPath = "multipath://";
 220   //CLog::Log(LOGDEBUG, "-- adding path: {}", strPath);
 221   for (std::vector<std::string>::const_iterator path = vecPaths.begin(); path != vecPaths.end(); ++path)
 222     AddToMultiPath(newPath, *path);
 223   //CLog::Log(LOGDEBUG, "Final path: {}", newPath);
 224   return newPath;
 225 }
 226
 227 std::string CMultiPathDirectory::ConstructMultiPath(const std::set<std::string> &setPaths)
 228 {
 229   std::string newPath = "multipath://";
 230   for (const std::string& path : setPaths)
 231     AddToMultiPath(newPath, path);
 232
 233   return newPath;
 234 }
 235
 236 void CMultiPathDirectory::MergeItems(CFileItemList &items)
 237 {
 238   CLog::Log(LOGDEBUG, "CMultiPathDirectory::MergeItems, items = {}", items.Size());
 239   auto start = std::chrono::steady_clock::now();
 240   if (items.Size() == 0)
 241     return;
 242   // sort items by label
 243   // folders are before files in this sort method
 244   items.Sort(SortByLabel, SortOrderAscending);
 245   int i = 0;
 246
 247   // if first item in the sorted list is a file, just abort
 248   if (!items.Get(i)->m_bIsFolder)
 249     return;
 250
 251   while (i + 1 < items.Size())
 252   {
 253     // there are no more folders left, so exit the loop
 254     CFileItemPtr pItem1 = items.Get(i);
 255     if (!pItem1->m_bIsFolder)
 256       break;
 257
 258     std::vector<int> stack;
 259     stack.push_back(i);
 260     CLog::Log(LOGDEBUG, "Testing path: [{:03}] {}", i, CURL::GetRedacted(pItem1->GetPath()));
 261
 262     int j = i + 1;
 263     do
 264     {
 265       CFileItemPtr pItem2 = items.Get(j);
 266       if (pItem2->GetLabel() != pItem1->GetLabel())
 267         break;
 268
 269       // ignore any filefolders which may coincidently have
 270       // the same label as a true folder
 271       if (!pItem2->IsFileFolder())
 272       {
 273         stack.push_back(j);
 274         CLog::Log(LOGDEBUG, "  Adding path: [{:03}] {}", j, CURL::GetRedacted(pItem2->GetPath()));
 275       }
 276       j++;
 277     }
 278     while (j < items.Size());
 279
 280     // do we have anything to combine?
 281     if (stack.size() > 1)
 282     {
 283       // we have a multipath so remove the items and add the new item
 284       std::string newPath = ConstructMultiPath(items, stack);
 285       for (unsigned int k = stack.size() - 1; k > 0; --k)
 286         items.Remove(stack[k]);
 287       pItem1->SetPath(newPath);
 288       CLog::Log(LOGDEBUG, "  New path: {}", CURL::GetRedacted(pItem1->GetPath()));
 289     }
 290
 291     i++;
 292   }
 293
 294   auto end = std::chrono::steady_clock::now();
 295   auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
 296
 297   CLog::Log(LOGDEBUG, "CMultiPathDirectory::MergeItems, items = {},  took {} ms", items.Size(),
 298             duration.count());
 299 }
 300
 301 bool CMultiPathDirectory::SupportsWriteFileOperations(const std::string &strPath)
 302 {
 303   std::vector<std::string> paths;
 304   GetPaths(strPath, paths);
 305   for (unsigned int i = 0; i < paths.size(); ++i)
 306     if (CUtil::SupportsWriteFileOperations(paths[i]))
 307       return true;
 308   return false;
 309 }