xbmc/filesystem/MultiPathDirectory.cpp

   1 /*
   2  *  Copyright (C) 2005-2018 Team Kodi
   3  *  This file is part of Kodi - https://kodi.tv
   4  *
   5  *  SPDX-License-Identifier: GPL-2.0-or-later
   6  *  See LICENSES/README.md for more information.
   7  */
   8
   9 #include "MultiPathDirectory.h"
  10
  11 #include "Directory.h"
  12 #include "FileItem.h"
  13 #include "FileItemList.h"
  14 #include "ServiceBroker.h"
  15 #include "URL.h"
  16 #include "Util.h"
  17 #include "dialogs/GUIDialogProgress.h"
  18 #include "guilib/GUIComponent.h"
  19 #include "guilib/GUIWindowManager.h"
  20 #include "threads/SystemClock.h"
  21 #include "utils/StringUtils.h"
  22 #include "utils/URIUtils.h"
  23 #include "utils/Variant.h"
  24 #include "utils/log.h"
  25
  26 using namespace XFILE;
  27
  28 using namespace std::chrono_literals;
  29
  30 //
  31 // multipath://{path1}/{path2}/{path3}/.../{path-N}
  32 //
  33 // unlike the older virtualpath:// protocol, sub-folders are combined together into a new
  34 // multipath:// style url.
  35 //
  36
  37 CMultiPathDirectory::CMultiPathDirectory() = default;
  38
  39 CMultiPathDirectory::~CMultiPathDirectory() = default;
  40
  41 bool CMultiPathDirectory::GetDirectory(const CURL& url, CFileItemList &items)
  42 {
  43   CLog::Log(LOGDEBUG, "CMultiPathDirectory::GetDirectory({})", url.GetRedacted());
  44
  45   std::vector<std::string> vecPaths;
  46   if (!GetPaths(url, vecPaths))
  47     return false;
  48
  49   XbmcThreads::EndTime<> progressTime(3000ms); // 3 seconds before showing progress bar
  50   CGUIDialogProgress* dlgProgress = NULL;
  51
  52   unsigned int iFailures = 0;
  53   for (unsigned int i = 0; i < vecPaths.size(); ++i)
  54   {
  55     // show the progress dialog if we have passed our time limit
  56     if (progressTime.IsTimePast() && !dlgProgress)
  57     {
  58       dlgProgress = CServiceBroker::GetGUI()->GetWindowManager().GetWindow<CGUIDialogProgress>(WINDOW_DIALOG_PROGRESS);
  59       if (dlgProgress)
  60       {
  61         dlgProgress->SetHeading(CVariant{15310});
  62         dlgProgress->SetLine(0, CVariant{15311});
  63         dlgProgress->SetLine(1, CVariant{""});
  64         dlgProgress->SetLine(2, CVariant{""});
  65         dlgProgress->Open();
  66         dlgProgress->ShowProgressBar(true);
  67         dlgProgress->SetProgressMax((int)vecPaths.size()*2);
  68         dlgProgress->Progress();
  69       }
  70     }
  71     if (dlgProgress)
  72     {
  73       CURL url(vecPaths[i]);
  74       dlgProgress->SetLine(1, CVariant{url.GetWithoutUserDetails()});
  75       dlgProgress->SetProgressAdvance();
  76       dlgProgress->Progress();
  77     }
  78
  79     CFileItemList tempItems;
  80     CLog::Log(LOGDEBUG, "Getting Directory ({})", CURL::GetRedacted(vecPaths[i]));
  81     if (CDirectory::GetDirectory(vecPaths[i], tempItems, m_strFileMask, m_flags))
  82       items.Append(tempItems);
  83     else
  84     {
  85       CLog::Log(LOGERROR, "Error Getting Directory ({})", CURL::GetRedacted(vecPaths[i]));
  86       iFailures++;
  87     }
  88
  89     if (dlgProgress)
  90     {
  91       dlgProgress->SetProgressAdvance();
  92       dlgProgress->Progress();
  93     }
  94   }
  95
  96   if (dlgProgress)
  97     dlgProgress->Close();
  98
  99   if (iFailures == vecPaths.size())
 100     return false;
 101
 102   // merge like-named folders into a sub multipath:// style url
 103   MergeItems(items);
 104
 105   return true;
 106 }
 107
 108 bool CMultiPathDirectory::Exists(const CURL& url)
 109 {
 110   CLog::Log(LOGDEBUG, "Testing Existence ({})", url.GetRedacted());
 111
 112   std::vector<std::string> vecPaths;
 113   if (!GetPaths(url, vecPaths))
 114     return false;
 115
 116   for (unsigned int i = 0; i < vecPaths.size(); ++i)
 117   {
 118     CLog::Log(LOGDEBUG, "Testing Existence ({})", CURL::GetRedacted(vecPaths[i]));
 119     if (CDirectory::Exists(vecPaths[i]))
 120       return true;
 121   }
 122   return false;
 123 }
 124
 125 bool CMultiPathDirectory::Remove(const CURL& url)
 126 {
 127   std::vector<std::string> vecPaths;
 128   if (!GetPaths(url, vecPaths))
 129     return false;
 130
 131   bool success = false;
 132   for (unsigned int i = 0; i < vecPaths.size(); ++i)
 133   {
 134     if (CDirectory::Remove(vecPaths[i]))
 135       success = true;
 136   }
 137   return success;
 138 }
 139
 140 std::string CMultiPathDirectory::GetFirstPath(const std::string &strPath)
 141 {
 142   size_t pos = strPath.find('/', 12);
 143   if (pos != std::string::npos)
 144     return CURL::Decode(strPath.substr(12, pos - 12));
 145   return "";
 146 }
 147
 148 bool CMultiPathDirectory::GetPaths(const CURL& url, std::vector<std::string>& vecPaths)
 149 {
 150   const std::string pathToUrl(url.Get());
 151   return GetPaths(pathToUrl, vecPaths);
 152 }
 153
 154 bool CMultiPathDirectory::GetPaths(const std::string& path, std::vector<std::string>& paths)
 155 {
 156   paths.clear();
 157
 158   // remove multipath:// from path and any trailing / (so that the last path doesn't get any more than it originally had)
 159   std::string path1 = path.substr(12);
 160   path1.erase(path1.find_last_not_of('/')+1);
 161
 162   // split on "/"
 163   std::vector<std::string> temp = StringUtils::Split(path1, '/');
 164   if (temp.empty())
 165     return false;
 166
 167   // URL decode each item
 168   paths.resize(temp.size());
 169   std::transform(temp.begin(), temp.end(), paths.begin(), CURL::Decode);
 170   return true;
 171 }
 172
 173 bool CMultiPathDirectory::HasPath(const std::string& strPath, const std::string& strPathToFind)
 174 {
 175   // remove multipath:// from path and any trailing / (so that the last path doesn't get any more than it originally had)
 176   std::string strPath1 = strPath.substr(12);
 177   URIUtils::RemoveSlashAtEnd(strPath1);
 178
 179   // split on "/"
 180   std::vector<std::string> vecTemp = StringUtils::Split(strPath1, '/');
 181   if (vecTemp.empty())
 182     return false;
 183
 184   // check each item
 185   for (unsigned int i = 0; i < vecTemp.size(); i++)
 186   {
 187     if (CURL::Decode(vecTemp[i]) == strPathToFind)
 188       return true;
 189   }
 190   return false;
 191 }
 192
 193 std::string CMultiPathDirectory::ConstructMultiPath(const CFileItemList& items, const std::vector<int> &stack)
 194 {
 195   // we replace all instances of comma's with double comma's, then separate
 196   // the paths using " , "
 197   //CLog::Log(LOGDEBUG, "Building multipath");
 198   std::string newPath = "multipath://";
 199   //CLog::Log(LOGDEBUG, "-- adding path: {}", strPath);
 200   for (unsigned int i = 0; i < stack.size(); ++i)
 201     AddToMultiPath(newPath, items[stack[i]]->GetPath());
 202
 203   //CLog::Log(LOGDEBUG, "Final path: {}", newPath);
 204   return newPath;
 205 }
 206
 207 void CMultiPathDirectory::AddToMultiPath(std::string& strMultiPath, const std::string& strPath)
 208 {
 209   URIUtils::AddSlashAtEnd(strMultiPath);
 210   //CLog::Log(LOGDEBUG, "-- adding path: {}", strPath);
 211   strMultiPath += CURL::Encode(strPath);
 212   strMultiPath += "/";
 213 }
 214
 215 std::string CMultiPathDirectory::ConstructMultiPath(const std::vector<std::string> &vecPaths)
 216 {
 217   // we replace all instances of comma's with double comma's, then separate
 218   // the paths using " , "
 219   //CLog::Log(LOGDEBUG, "Building multipath");
 220   std::string newPath = "multipath://";
 221   //CLog::Log(LOGDEBUG, "-- adding path: {}", strPath);
 222   for (std::vector<std::string>::const_iterator path = vecPaths.begin(); path != vecPaths.end(); ++path)
 223     AddToMultiPath(newPath, *path);
 224   //CLog::Log(LOGDEBUG, "Final path: {}", newPath);
 225   return newPath;
 226 }
 227
 228 std::string CMultiPathDirectory::ConstructMultiPath(const std::set<std::string> &setPaths)
 229 {
 230   std::string newPath = "multipath://";
 231   for (const std::string& path : setPaths)
 232     AddToMultiPath(newPath, path);
 233
 234   return newPath;
 235 }
 236
 237 void CMultiPathDirectory::MergeItems(CFileItemList &items)
 238 {
 239   CLog::Log(LOGDEBUG, "CMultiPathDirectory::MergeItems, items = {}", items.Size());
 240   auto start = std::chrono::steady_clock::now();
 241   if (items.Size() == 0)
 242     return;
 243   // sort items by label
 244   // folders are before files in this sort method
 245   items.Sort(SortByLabel, SortOrderAscending);
 246   int i = 0;
 247
 248   // if first item in the sorted list is a file, just abort
 249   if (!items.Get(i)->m_bIsFolder)
 250     return;
 251
 252   while (i + 1 < items.Size())
 253   {
 254     // there are no more folders left, so exit the loop
 255     CFileItemPtr pItem1 = items.Get(i);
 256     if (!pItem1->m_bIsFolder)
 257       break;
 258
 259     std::vector<int> stack;
 260     stack.push_back(i);
 261     CLog::Log(LOGDEBUG, "Testing path: [{:03}] {}", i, CURL::GetRedacted(pItem1->GetPath()));
 262
 263     int j = i + 1;
 264     do
 265     {
 266       CFileItemPtr pItem2 = items.Get(j);
 267       if (pItem2->GetLabel() != pItem1->GetLabel())
 268         break;
 269
 270       // ignore any filefolders which may coincidently have
 271       // the same label as a true folder
 272       if (!pItem2->IsFileFolder())
 273       {
 274         stack.push_back(j);
 275         CLog::Log(LOGDEBUG, "  Adding path: [{:03}] {}", j, CURL::GetRedacted(pItem2->GetPath()));
 276       }
 277       j++;
 278     }
 279     while (j < items.Size());
 280
 281     // do we have anything to combine?
 282     if (stack.size() > 1)
 283     {
 284       // we have a multipath so remove the items and add the new item
 285       std::string newPath = ConstructMultiPath(items, stack);
 286       for (unsigned int k = stack.size() - 1; k > 0; --k)
 287         items.Remove(stack[k]);
 288       pItem1->SetPath(newPath);
 289       CLog::Log(LOGDEBUG, "  New path: {}", CURL::GetRedacted(pItem1->GetPath()));
 290     }
 291
 292     i++;
 293   }
 294
 295   auto end = std::chrono::steady_clock::now();
 296   auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
 297
 298   CLog::Log(LOGDEBUG, "CMultiPathDirectory::MergeItems, items = {},  took {} ms", items.Size(),
 299             duration.count());
 300 }
 301
 302 bool CMultiPathDirectory::SupportsWriteFileOperations(const std::string &strPath)
 303 {
 304   std::vector<std::string> paths;
 305   GetPaths(strPath, paths);
 306   for (unsigned int i = 0; i < paths.size(); ++i)
 307     if (CUtil::SupportsWriteFileOperations(paths[i]))
 308       return true;
 309   return false;
 310 }