xbmc/video/VideoInfoScanner.cpp

   1 /*
   2  *  Copyright (C) 2005-2018 Team Kodi
   3  *  This file is part of Kodi - https://kodi.tv
   4  *
   5  *  SPDX-License-Identifier: GPL-2.0-or-later
   6  *  See LICENSES/README.md for more information.
   7  */
   8
   9 #include "VideoInfoScanner.h"
  10
  11 #include "FileItem.h"
  12 #include "FileItemList.h"
  13 #include "GUIInfoManager.h"
  14 #include "GUIUserMessages.h"
  15 #include "ServiceBroker.h"
  16 #include "TextureCache.h"
  17 #include "URL.h"
  18 #include "Util.h"
  19 #include "VideoInfoDownloader.h"
  20 #include "cores/VideoPlayer/DVDFileInfo.h"
  21 #include "dialogs/GUIDialogExtendedProgressBar.h"
  22 #include "dialogs/GUIDialogProgress.h"
  23 #include "events/EventLog.h"
  24 #include "events/MediaLibraryEvent.h"
  25 #include "filesystem/Directory.h"
  26 #include "filesystem/File.h"
  27 #include "filesystem/MultiPathDirectory.h"
  28 #include "filesystem/PluginDirectory.h"
  29 #include "guilib/GUIComponent.h"
  30 #include "guilib/GUIWindowManager.h"
  31 #include "guilib/LocalizeStrings.h"
  32 #include "imagefiles/ImageFileURL.h"
  33 #include "interfaces/AnnouncementManager.h"
  34 #include "messaging/helpers/DialogHelper.h"
  35 #include "messaging/helpers/DialogOKHelper.h"
  36 #include "playlists/PlayListFileItemClassify.h"
  37 #include "settings/AdvancedSettings.h"
  38 #include "settings/Settings.h"
  39 #include "settings/SettingsComponent.h"
  40 #include "tags/VideoInfoTagLoaderFactory.h"
  41 #include "utils/ArtUtils.h"
  42 #include "utils/Digest.h"
  43 #include "utils/FileExtensionProvider.h"
  44 #include "utils/RegExp.h"
  45 #include "utils/StringUtils.h"
  46 #include "utils/URIUtils.h"
  47 #include "utils/Variant.h"
  48 #include "utils/log.h"
  49 #include "video/VideoFileItemClassify.h"
  50 #include "video/VideoManagerTypes.h"
  51 #include "video/VideoThumbLoader.h"
  52 #include "video/dialogs/GUIDialogVideoManagerExtras.h"
  53 #include "video/dialogs/GUIDialogVideoManagerVersions.h"
  54
  55 #include <algorithm>
  56 #include <memory>
  57 #include <utility>
  58
  59 using namespace XFILE;
  60 using namespace ADDON;
  61 using namespace KODI::MESSAGING;
  62 using namespace KODI::VIDEO;
  63
  64 using KODI::MESSAGING::HELPERS::DialogResponse;
  65 using KODI::UTILITY::CDigest;
  66
  67 namespace KODI::VIDEO
  68 {
  69
  70   CVideoInfoScanner::CVideoInfoScanner()
  71   {
  72     m_bStop = false;
  73     m_scanAll = false;
  74
  75     const auto settings = CServiceBroker::GetSettingsComponent()->GetSettings();
  76
  77     m_ignoreVideoVersions = settings->GetBool(CSettings::SETTING_VIDEOLIBRARY_IGNOREVIDEOVERSIONS);
  78     m_ignoreVideoExtras = settings->GetBool(CSettings::SETTING_VIDEOLIBRARY_IGNOREVIDEOEXTRAS);
  79   }
  80
  81   CVideoInfoScanner::~CVideoInfoScanner()
  82   = default;
  83
  84   void CVideoInfoScanner::Process()
  85   {
  86     m_bStop = false;
  87
  88     try
  89     {
  90       const auto settings = CServiceBroker::GetSettingsComponent()->GetSettings();
  91
  92       if (m_showDialog && !settings->GetBool(CSettings::SETTING_VIDEOLIBRARY_BACKGROUNDUPDATE))
  93       {
  94         CGUIDialogExtendedProgressBar* dialog =
  95           CServiceBroker::GetGUI()->GetWindowManager().GetWindow<CGUIDialogExtendedProgressBar>(WINDOW_DIALOG_EXT_PROGRESS);
  96         if (dialog)
  97            m_handle = dialog->GetHandle(g_localizeStrings.Get(314));
  98       }
  99
 100       // check if we only need to perform a cleaning
 101       if (m_bClean && m_pathsToScan.empty())
 102       {
 103         std::set<int> paths;
 104         m_database.CleanDatabase(m_handle, paths, false);
 105
 106         if (m_handle)
 107           m_handle->MarkFinished();
 108         m_handle = NULL;
 109
 110         m_bRunning = false;
 111
 112         return;
 113       }
 114
 115       auto start = std::chrono::steady_clock::now();
 116
 117       m_database.Open();
 118
 119       m_bCanInterrupt = true;
 120
 121       CLog::Log(LOGINFO, "VideoInfoScanner: Starting scan ..");
 122       CServiceBroker::GetAnnouncementManager()->Announce(ANNOUNCEMENT::VideoLibrary,
 123                                                          "OnScanStarted");
 124
 125       // Database operations should not be canceled
 126       // using Interrupt() while scanning as it could
 127       // result in unexpected behaviour.
 128       m_bCanInterrupt = false;
 129
 130       bool bCancelled = false;
 131       while (!bCancelled && !m_pathsToScan.empty())
 132       {
 133         /*
 134          * A copy of the directory path is used because the path supplied is
 135          * immediately removed from the m_pathsToScan set in DoScan(). If the
 136          * reference points to the entry in the set a null reference error
 137          * occurs.
 138          */
 139         std::string directory = *m_pathsToScan.begin();
 140         if (m_bStop)
 141         {
 142           bCancelled = true;
 143         }
 144         else if (!CDirectory::Exists(directory))
 145         {
 146           /*
 147            * Note that this will skip clean (if m_bClean is enabled) if the directory really
 148            * doesn't exist rather than a NAS being switched off.  A manual clean from settings
 149            * will still pick up and remove it though.
 150            */
 151           CLog::Log(LOGWARNING, "{} directory '{}' does not exist - skipping scan{}.", __FUNCTION__,
 152                     CURL::GetRedacted(directory), m_bClean ? " and clean" : "");
 153           m_pathsToScan.erase(m_pathsToScan.begin());
 154         }
 155         else if (!DoScan(directory))
 156           bCancelled = true;
 157       }
 158
 159       if (!bCancelled)
 160       {
 161         if (m_bClean)
 162           m_database.CleanDatabase(m_handle, m_pathsToClean, false);
 163         else
 164         {
 165           if (m_handle)
 166             m_handle->SetTitle(g_localizeStrings.Get(331));
 167           m_database.Compress(false);
 168         }
 169       }
 170
 171       CServiceBroker::GetGUI()->GetInfoManager().GetInfoProviders().GetLibraryInfoProvider().ResetLibraryBools();
 172       m_database.Close();
 173
 174       auto end = std::chrono::steady_clock::now();
 175       auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
 176
 177       CLog::Log(LOGINFO, "VideoInfoScanner: Finished scan. Scanning for video info took {} ms",
 178                 duration.count());
 179     }
 180     catch (...)
 181     {
 182       CLog::Log(LOGERROR, "VideoInfoScanner: Exception while scanning.");
 183     }
 184
 185     m_bRunning = false;
 186     CServiceBroker::GetAnnouncementManager()->Announce(ANNOUNCEMENT::VideoLibrary,
 187                                                        "OnScanFinished");
 188
 189     if (m_handle)
 190       m_handle->MarkFinished();
 191     m_handle = NULL;
 192   }
 193
 194   void CVideoInfoScanner::Start(const std::string& strDirectory, bool scanAll)
 195   {
 196     m_strStartDir = strDirectory;
 197     m_scanAll = scanAll;
 198     m_pathsToScan.clear();
 199     m_pathsToClean.clear();
 200
 201     m_database.Open();
 202     if (strDirectory.empty())
 203     { // scan all paths in the database.  We do this by scanning all paths in the db, and crossing them off the list as
 204       // we go.
 205       m_database.GetPaths(m_pathsToScan);
 206     }
 207     else
 208     { // scan all the paths of this subtree that is in the database
 209       std::vector<std::string> rootDirs;
 210       if (URIUtils::IsMultiPath(strDirectory))
 211         CMultiPathDirectory::GetPaths(strDirectory, rootDirs);
 212       else
 213         rootDirs.push_back(strDirectory);
 214
 215       for (std::vector<std::string>::const_iterator it = rootDirs.begin(); it < rootDirs.end(); ++it)
 216       {
 217         m_pathsToScan.insert(*it);
 218         std::vector<std::pair<int, std::string>> subpaths;
 219         m_database.GetSubPaths(*it, subpaths);
 220         for (std::vector<std::pair<int, std::string>>::iterator it = subpaths.begin(); it < subpaths.end(); ++it)
 221           m_pathsToScan.insert(it->second);
 222       }
 223     }
 224     m_database.Close();
 225     m_bClean = CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_bVideoLibraryCleanOnUpdate;
 226
 227     m_bRunning = true;
 228     Process();
 229   }
 230
 231   void CVideoInfoScanner::Stop()
 232   {
 233     if (m_bCanInterrupt)
 234       m_database.Interrupt();
 235
 236     m_bStop = true;
 237   }
 238
 239   static void OnDirectoryScanned(const std::string& strDirectory)
 240   {
 241     CGUIMessage msg(GUI_MSG_DIRECTORY_SCANNED, 0, 0, 0);
 242     msg.SetStringParam(strDirectory);
 243     CServiceBroker::GetGUI()->GetWindowManager().SendThreadMessage(msg);
 244   }
 245
 246   bool CVideoInfoScanner::DoScan(const std::string& strDirectory)
 247   {
 248     if (m_handle)
 249     {
 250       m_handle->SetText(g_localizeStrings.Get(20415));
 251     }
 252
 253     /*
 254      * Remove this path from the list we're processing. This must be done prior to
 255      * the check for file or folder exclusion to prevent an infinite while loop
 256      * in Process().
 257      */
 258     std::set<std::string>::iterator it = m_pathsToScan.find(strDirectory);
 259     if (it != m_pathsToScan.end())
 260       m_pathsToScan.erase(it);
 261
 262     // load subfolder
 263     CFileItemList items;
 264     bool foundDirectly = false;
 265     bool bSkip = false;
 266
 267     SScanSettings settings;
 268     ScraperPtr info = m_database.GetScraperForPath(strDirectory, settings, foundDirectly);
 269     CONTENT_TYPE content = info ? info->Content() : CONTENT_NONE;
 270
 271     // exclude folders that match our exclude regexps
 272     const std::vector<std::string> &regexps = content == CONTENT_TVSHOWS ? CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_tvshowExcludeFromScanRegExps
 273                                                          : CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_moviesExcludeFromScanRegExps;
 274
 275     if (CUtil::ExcludeFileOrFolder(strDirectory, regexps))
 276       return true;
 277
 278     if (HasNoMedia(strDirectory))
 279       return true;
 280
 281     bool ignoreFolder = !m_scanAll && settings.noupdate;
 282     if (content == CONTENT_NONE || ignoreFolder)
 283       return true;
 284
 285     if (URIUtils::IsPlugin(strDirectory) && !CPluginDirectory::IsMediaLibraryScanningAllowed(TranslateContent(content), strDirectory))
 286     {
 287       CLog::Log(
 288           LOGINFO,
 289           "VideoInfoScanner: Plugin '{}' does not support media library scanning for '{}' content",
 290           CURL::GetRedacted(strDirectory), TranslateContent(content));
 291       return true;
 292     }
 293
 294     std::string hash, dbHash;
 295     if (content == CONTENT_MOVIES ||content == CONTENT_MUSICVIDEOS)
 296     {
 297       if (m_handle)
 298       {
 299         int str = content == CONTENT_MOVIES ? 20317:20318;
 300         m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(str), info->Name()));
 301       }
 302
 303       std::string fastHash;
 304       if (CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_bVideoLibraryUseFastHash && !URIUtils::IsPlugin(strDirectory))
 305         fastHash = GetFastHash(strDirectory, regexps);
 306
 307       if (m_database.GetPathHash(strDirectory, dbHash) && !fastHash.empty() && StringUtils::EqualsNoCase(fastHash, dbHash))
 308       { // fast hashes match - no need to process anything
 309         hash = fastHash;
 310       }
 311       else
 312       { // need to fetch the folder
 313         CDirectory::GetDirectory(strDirectory, items, CServiceBroker::GetFileExtensionProvider().GetVideoExtensions(),
 314                                  DIR_FLAG_DEFAULTS);
 315         // do not consider inner folders with .nomedia
 316         items.erase(std::remove_if(items.begin(), items.end(),
 317                                    [this](const CFileItemPtr& item) {
 318                                      return item->m_bIsFolder && HasNoMedia(item->GetPath());
 319                                    }),
 320                     items.end());
 321         items.Stack();
 322
 323         // check whether to re-use previously computed fast hash
 324         if (!CanFastHash(items, regexps) || fastHash.empty())
 325           GetPathHash(items, hash);
 326         else
 327           hash = fastHash;
 328       }
 329
 330       if (StringUtils::EqualsNoCase(hash, dbHash))
 331       { // hash matches - skipping
 332         CLog::Log(LOGDEBUG, "VideoInfoScanner: Skipping dir '{}' due to no change{}",
 333                   CURL::GetRedacted(strDirectory), !fastHash.empty() ? " (fasthash)" : "");
 334         bSkip = true;
 335       }
 336       else if (hash.empty())
 337       { // directory empty or non-existent - add to clean list and skip
 338         CLog::Log(LOGDEBUG,
 339                   "VideoInfoScanner: Skipping dir '{}' as it's empty or doesn't exist - adding to "
 340                   "clean list",
 341                   CURL::GetRedacted(strDirectory));
 342         if (m_bClean)
 343           m_pathsToClean.insert(m_database.GetPathId(strDirectory));
 344         bSkip = true;
 345       }
 346       else if (dbHash.empty())
 347       { // new folder - scan
 348         CLog::Log(LOGDEBUG, "VideoInfoScanner: Scanning dir '{}' as not in the database",
 349                   CURL::GetRedacted(strDirectory));
 350       }
 351       else
 352       { // hash changed - rescan
 353         CLog::Log(LOGDEBUG, "VideoInfoScanner: Rescanning dir '{}' due to change ({} != {})",
 354                   CURL::GetRedacted(strDirectory), dbHash, hash);
 355       }
 356     }
 357     else if (content == CONTENT_TVSHOWS)
 358     {
 359       if (m_handle)
 360         m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(20319), info->Name()));
 361
 362       if (foundDirectly && !settings.parent_name_root)
 363       {
 364         CDirectory::GetDirectory(strDirectory, items, CServiceBroker::GetFileExtensionProvider().GetVideoExtensions(),
 365                                  DIR_FLAG_DEFAULTS);
 366         items.SetPath(strDirectory);
 367         GetPathHash(items, hash);
 368         bSkip = true;
 369         if (!m_database.GetPathHash(strDirectory, dbHash) || !StringUtils::EqualsNoCase(dbHash, hash))
 370           bSkip = false;
 371         else
 372           items.Clear();
 373       }
 374       else
 375       {
 376         CFileItemPtr item(new CFileItem(URIUtils::GetFileName(strDirectory)));
 377         item->SetPath(strDirectory);
 378         item->m_bIsFolder = true;
 379         items.Add(item);
 380         items.SetPath(URIUtils::GetParentPath(item->GetPath()));
 381       }
 382     }
 383     bool foundSomething = false;
 384     if (!bSkip)
 385     {
 386       foundSomething = RetrieveVideoInfo(items, settings.parent_name_root, content);
 387       if (foundSomething)
 388       {
 389         if (!m_bStop && (content == CONTENT_MOVIES || content == CONTENT_MUSICVIDEOS))
 390         {
 391           m_database.SetPathHash(strDirectory, hash);
 392           if (m_bClean)
 393             m_pathsToClean.insert(m_database.GetPathId(strDirectory));
 394           CLog::Log(LOGDEBUG, "VideoInfoScanner: Finished adding information from dir {}",
 395                     CURL::GetRedacted(strDirectory));
 396         }
 397       }
 398       else
 399       {
 400         if (m_bClean)
 401           m_pathsToClean.insert(m_database.GetPathId(strDirectory));
 402         CLog::Log(LOGDEBUG, "VideoInfoScanner: No (new) information was found in dir {}",
 403                   CURL::GetRedacted(strDirectory));
 404       }
 405     }
 406     else if (!StringUtils::EqualsNoCase(hash, dbHash) && (content == CONTENT_MOVIES || content == CONTENT_MUSICVIDEOS))
 407     { // update the hash either way - we may have changed the hash to a fast version
 408       m_database.SetPathHash(strDirectory, hash);
 409     }
 410
 411     if (m_handle)
 412       OnDirectoryScanned(strDirectory);
 413
 414     for (int i = 0; i < items.Size(); ++i)
 415     {
 416       CFileItemPtr pItem = items[i];
 417
 418       if (m_bStop)
 419         break;
 420
 421       // add video extras to library
 422       if (foundSomething && !m_ignoreVideoExtras && IsVideoExtrasFolder(*pItem))
 423       {
 424         if (AddVideoExtras(items, content, pItem->GetPath()))
 425         {
 426           CLog::Log(LOGDEBUG, "VideoInfoScanner: Finished adding video extras from dir {}",
 427                     CURL::GetRedacted(pItem->GetPath()));
 428         }
 429
 430         // no further processing required
 431         continue;
 432       }
 433
 434       // if we have a directory item (non-playlist) we then recurse into that folder
 435       // do not recurse for tv shows - we have already looked recursively for episodes
 436       if (pItem->m_bIsFolder && !pItem->IsParentFolder() && !PLAYLIST::IsPlayList(*pItem) &&
 437           settings.recurse > 0 && content != CONTENT_TVSHOWS)
 438       {
 439         if (!DoScan(pItem->GetPath()))
 440         {
 441           m_bStop = true;
 442         }
 443       }
 444     }
 445     return !m_bStop;
 446   }
 447
 448   bool CVideoInfoScanner::RetrieveVideoInfo(CFileItemList& items, bool bDirNames, CONTENT_TYPE content, bool useLocal, CScraperUrl* pURL, bool fetchEpisodes, CGUIDialogProgress* pDlgProgress)
 449   {
 450     if (pDlgProgress)
 451     {
 452       if (items.Size() > 1 || (items[0]->m_bIsFolder && fetchEpisodes))
 453       {
 454         pDlgProgress->ShowProgressBar(true);
 455         pDlgProgress->SetPercentage(0);
 456       }
 457       else
 458         pDlgProgress->ShowProgressBar(false);
 459
 460       pDlgProgress->Progress();
 461     }
 462
 463     m_database.Open();
 464
 465     bool FoundSomeInfo = false;
 466     std::vector<int> seenPaths;
 467     for (int i = 0; i < items.Size(); ++i)
 468     {
 469       CFileItemPtr pItem = items[i];
 470
 471       // we do this since we may have a override per dir
 472       ScraperPtr info2 = m_database.GetScraperForPath(pItem->m_bIsFolder ? pItem->GetPath() : items.GetPath());
 473       if (!info2) // skip
 474         continue;
 475
 476       // Discard all .nomedia folders
 477       if (pItem->m_bIsFolder && HasNoMedia(pItem->GetPath()))
 478         continue;
 479
 480       // Discard all exclude files defined by regExExclude
 481       if (CUtil::ExcludeFileOrFolder(pItem->GetPath(), (content == CONTENT_TVSHOWS) ? CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_tvshowExcludeFromScanRegExps
 482                                                                     : CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_moviesExcludeFromScanRegExps))
 483         continue;
 484
 485       if (info2->Content() == CONTENT_MOVIES || info2->Content() == CONTENT_MUSICVIDEOS)
 486       {
 487         if (m_handle)
 488           m_handle->SetPercentage(i*100.f/items.Size());
 489       }
 490
 491       // clear our scraper cache
 492       info2->ClearCache();
 493
 494       INFO_RET ret = INFO_CANCELLED;
 495       if (info2->Content() == CONTENT_TVSHOWS)
 496         ret = RetrieveInfoForTvShow(pItem.get(), bDirNames, info2, useLocal, pURL, fetchEpisodes, pDlgProgress);
 497       else if (info2->Content() == CONTENT_MOVIES)
 498         ret = RetrieveInfoForMovie(pItem.get(), bDirNames, info2, useLocal, pURL, pDlgProgress);
 499       else if (info2->Content() == CONTENT_MUSICVIDEOS)
 500         ret = RetrieveInfoForMusicVideo(pItem.get(), bDirNames, info2, useLocal, pURL, pDlgProgress);
 501       else
 502       {
 503         CLog::Log(LOGERROR, "VideoInfoScanner: Unknown content type {} ({})", info2->Content(),
 504                   CURL::GetRedacted(pItem->GetPath()));
 505         FoundSomeInfo = false;
 506         break;
 507       }
 508       if (ret == INFO_CANCELLED || ret == INFO_ERROR)
 509       {
 510         CLog::Log(LOGWARNING,
 511                   "VideoInfoScanner: Error {} occurred while retrieving"
 512                   "information for {}.",
 513                   ret, CURL::GetRedacted(pItem->GetPath()));
 514         FoundSomeInfo = false;
 515         break;
 516       }
 517       if (ret == INFO_ADDED || ret == INFO_HAVE_ALREADY)
 518         FoundSomeInfo = true;
 519       else if (ret == INFO_NOT_FOUND)
 520       {
 521         CLog::Log(LOGWARNING,
 522                   "No information found for item '{}', it won't be added to the library.",
 523                   CURL::GetRedacted(pItem->GetPath()));
 524
 525         MediaType mediaType = MediaTypeMovie;
 526         if (info2->Content() == CONTENT_TVSHOWS)
 527           mediaType = MediaTypeTvShow;
 528         else if (info2->Content() == CONTENT_MUSICVIDEOS)
 529           mediaType = MediaTypeMusicVideo;
 530
 531         auto eventLog = CServiceBroker::GetEventLog();
 532         if (eventLog)
 533         {
 534           const std::string itemlogpath = (info2->Content() == CONTENT_TVSHOWS)
 535                                               ? CURL::GetRedacted(pItem->GetPath())
 536                                               : URIUtils::GetFileName(pItem->GetPath());
 537
 538           eventLog->Add(EventPtr(new CMediaLibraryEvent(
 539               mediaType, pItem->GetPath(), 24145,
 540               StringUtils::Format(g_localizeStrings.Get(24147), mediaType, itemlogpath),
 541               EventLevel::Warning)));
 542         }
 543       }
 544
 545       pURL = NULL;
 546
 547       // Keep track of directories we've seen
 548       if (m_bClean && pItem->m_bIsFolder)
 549         seenPaths.push_back(m_database.GetPathId(pItem->GetPath()));
 550     }
 551
 552     if (content == CONTENT_TVSHOWS && ! seenPaths.empty())
 553     {
 554       std::vector<std::pair<int, std::string>> libPaths;
 555       m_database.GetSubPaths(items.GetPath(), libPaths);
 556       for (std::vector<std::pair<int, std::string> >::iterator i = libPaths.begin(); i < libPaths.end(); ++i)
 557       {
 558         if (find(seenPaths.begin(), seenPaths.end(), i->first) == seenPaths.end())
 559           m_pathsToClean.insert(i->first);
 560       }
 561     }
 562     if(pDlgProgress)
 563       pDlgProgress->ShowProgressBar(false);
 564
 565     m_database.Close();
 566     return FoundSomeInfo;
 567   }
 568
 569   CInfoScanner::INFO_RET
 570   CVideoInfoScanner::RetrieveInfoForTvShow(CFileItem *pItem,
 571                                            bool bDirNames,
 572                                            ScraperPtr &info2,
 573                                            bool useLocal,
 574                                            CScraperUrl* pURL,
 575                                            bool fetchEpisodes,
 576                                            CGUIDialogProgress* pDlgProgress)
 577   {
 578     const bool isSeason =
 579         pItem->HasVideoInfoTag() && pItem->GetVideoInfoTag()->m_type == MediaTypeSeason;
 580
 581     int idTvShow = -1;
 582     int idSeason = -1;
 583     std::string strPath = pItem->GetPath();
 584     if (pItem->m_bIsFolder)
 585     {
 586       idTvShow = m_database.GetTvShowId(strPath);
 587       if (isSeason && idTvShow > -1)
 588         idSeason = m_database.GetSeasonId(idTvShow, pItem->GetVideoInfoTag()->m_iSeason);
 589     }
 590     else if (pItem->IsPlugin() && pItem->HasVideoInfoTag() && pItem->GetVideoInfoTag()->m_iIdShow >= 0)
 591     {
 592       // for plugin source we cannot get idTvShow from episode path with URIUtils::GetDirectory() in all cases
 593       // so use m_iIdShow from video info tag if possible
 594       idTvShow = pItem->GetVideoInfoTag()->m_iIdShow;
 595       CVideoInfoTag showInfo;
 596       if (m_database.GetTvShowInfo(std::string(), showInfo, idTvShow, nullptr, 0))
 597         strPath = showInfo.GetPath();
 598     }
 599     else
 600     {
 601       strPath = URIUtils::GetDirectory(strPath);
 602       idTvShow = m_database.GetTvShowId(strPath);
 603       if (isSeason && idTvShow > -1)
 604         idSeason = m_database.GetSeasonId(idTvShow, pItem->GetVideoInfoTag()->m_iSeason);
 605     }
 606     if (idTvShow > -1 && (!isSeason || idSeason > -1) && (fetchEpisodes || !pItem->m_bIsFolder))
 607     {
 608       INFO_RET ret = RetrieveInfoForEpisodes(pItem, idTvShow, info2, useLocal, pDlgProgress);
 609       if (ret == INFO_ADDED)
 610         m_database.SetPathHash(strPath, pItem->GetProperty("hash").asString());
 611       return ret;
 612     }
 613
 614     if (ProgressCancelled(pDlgProgress, pItem->m_bIsFolder ? 20353 : 20361,
 615                           pItem->m_bIsFolder ? pItem->GetVideoInfoTag()->m_strShowTitle
 616                                              : pItem->GetVideoInfoTag()->m_strTitle))
 617       return INFO_CANCELLED;
 618
 619     if (m_handle)
 620       m_handle->SetText(pItem->GetMovieName(bDirNames));
 621
 622     CInfoScanner::INFO_TYPE result=CInfoScanner::NO_NFO;
 623     CScraperUrl scrUrl;
 624     // handle .nfo files
 625     std::unique_ptr<IVideoInfoTagLoader> loader;
 626     if (useLocal)
 627     {
 628       loader.reset(CVideoInfoTagLoaderFactory::CreateLoader(*pItem, info2, bDirNames));
 629       if (loader)
 630       {
 631         pItem->GetVideoInfoTag()->Reset();
 632         result = loader->Load(*pItem->GetVideoInfoTag(), false);
 633       }
 634     }
 635
 636     if (result == CInfoScanner::FULL_NFO)
 637     {
 638
 639       long lResult = AddVideo(pItem, info2->Content(), bDirNames, useLocal);
 640       if (lResult < 0)
 641         return INFO_ERROR;
 642       if (fetchEpisodes)
 643       {
 644         INFO_RET ret = RetrieveInfoForEpisodes(pItem, lResult, info2, useLocal, pDlgProgress);
 645         if (ret == INFO_ADDED)
 646           m_database.SetPathHash(pItem->GetPath(), pItem->GetProperty("hash").asString());
 647         return ret;
 648       }
 649       return INFO_ADDED;
 650     }
 651     if (result == CInfoScanner::URL_NFO || result == CInfoScanner::COMBINED_NFO)
 652     {
 653       scrUrl = loader->ScraperUrl();
 654       pURL = &scrUrl;
 655     }
 656
 657     CScraperUrl url;
 658     int retVal = 0;
 659     std::string movieTitle = pItem->GetMovieName(bDirNames);
 660     int movieYear = -1; // hint that movie title was not found
 661     if (result == CInfoScanner::TITLE_NFO)
 662     {
 663       CVideoInfoTag* tag = pItem->GetVideoInfoTag();
 664       movieTitle = tag->GetTitle();
 665       movieYear = tag->GetYear(); // movieYear is expected to be >= 0
 666     }
 667
 668     std::string identifierType;
 669     std::string identifier;
 670     long lResult = -1;
 671     if (info2->IsPython() && CUtil::GetFilenameIdentifier(movieTitle, identifierType, identifier))
 672     {
 673       const std::unordered_map<std::string, std::string> uniqueIDs{{identifierType, identifier}};
 674       if (GetDetails(pItem, uniqueIDs, url, info2,
 675                      (result == CInfoScanner::COMBINED_NFO || result == CInfoScanner::OVERRIDE_NFO)
 676                          ? loader.get()
 677                          : nullptr,
 678                      pDlgProgress))
 679       {
 680         if ((lResult = AddVideo(pItem, info2->Content(), false, useLocal)) < 0)
 681           return INFO_ERROR;
 682
 683         if (fetchEpisodes)
 684         {
 685           INFO_RET ret = RetrieveInfoForEpisodes(pItem, lResult, info2, useLocal, pDlgProgress);
 686           if (ret == INFO_ADDED)
 687           {
 688             m_database.SetPathHash(pItem->GetPath(), pItem->GetProperty("hash").asString());
 689             return INFO_ADDED;
 690           }
 691         }
 692         return INFO_ADDED;
 693       }
 694     }
 695
 696     if (pURL && pURL->HasUrls())
 697       url = *pURL;
 698     else if ((retVal = FindVideo(movieTitle, movieYear, info2, url, pDlgProgress)) <= 0)
 699       return retVal < 0 ? INFO_CANCELLED : INFO_NOT_FOUND;
 700
 701     CLog::Log(LOGDEBUG, "VideoInfoScanner: Fetching url '{}' using {} scraper (content: '{}')",
 702               url.GetFirstThumbUrl(), info2->Name(), TranslateContent(info2->Content()));
 703     const std::unordered_map<std::string, std::string> uniqueIDs{{identifierType, identifier}};
 704
 705     if (GetDetails(pItem, {}, url, info2,
 706                    (result == CInfoScanner::COMBINED_NFO || result == CInfoScanner::OVERRIDE_NFO)
 707                        ? loader.get()
 708                        : nullptr,
 709                    pDlgProgress))
 710     {
 711       if ((lResult = AddVideo(pItem, info2->Content(), false, useLocal)) < 0)
 712         return INFO_ERROR;
 713     }
 714     if (fetchEpisodes)
 715     {
 716       INFO_RET ret = RetrieveInfoForEpisodes(pItem, lResult, info2, useLocal, pDlgProgress);
 717       if (ret == INFO_ADDED)
 718         m_database.SetPathHash(pItem->GetPath(), pItem->GetProperty("hash").asString());
 719     }
 720     return INFO_ADDED;
 721   }
 722
 723   CInfoScanner::INFO_RET
 724   CVideoInfoScanner::RetrieveInfoForMovie(CFileItem *pItem,
 725                                           bool bDirNames,
 726                                           ScraperPtr &info2,
 727                                           bool useLocal,
 728                                           CScraperUrl* pURL,
 729                                           CGUIDialogProgress* pDlgProgress)
 730   {
 731     if (pItem->m_bIsFolder || !IsVideo(*pItem) || pItem->IsNFO() ||
 732         (PLAYLIST::IsPlayList(*pItem) && !URIUtils::HasExtension(pItem->GetPath(), ".strm")))
 733       return INFO_NOT_NEEDED;
 734
 735     if (ProgressCancelled(pDlgProgress, 198, pItem->GetLabel()))
 736       return INFO_CANCELLED;
 737
 738     if (m_database.HasMovieInfo(pItem->GetDynPath()))
 739       return INFO_HAVE_ALREADY;
 740
 741     if (m_handle)
 742       m_handle->SetText(pItem->GetMovieName(bDirNames));
 743
 744     CInfoScanner::INFO_TYPE result = CInfoScanner::NO_NFO;
 745     CScraperUrl scrUrl;
 746     // handle .nfo files
 747     std::unique_ptr<IVideoInfoTagLoader> loader;
 748     if (useLocal)
 749     {
 750       loader.reset(CVideoInfoTagLoaderFactory::CreateLoader(*pItem, info2, bDirNames));
 751       if (loader)
 752       {
 753         pItem->GetVideoInfoTag()->Reset();
 754         result = loader->Load(*pItem->GetVideoInfoTag(), false);
 755       }
 756     }
 757     if (result == CInfoScanner::FULL_NFO)
 758     {
 759       const int dbId = AddVideo(pItem, info2->Content(), bDirNames, true);
 760       if (dbId < 0)
 761         return INFO_ERROR;
 762       if (!m_ignoreVideoVersions && ProcessVideoVersion(VideoDbContentType::MOVIES, dbId))
 763         return INFO_HAVE_ALREADY;
 764       return INFO_ADDED;
 765     }
 766     if (result == CInfoScanner::URL_NFO || result == CInfoScanner::COMBINED_NFO)
 767     {
 768       scrUrl = loader->ScraperUrl();
 769       pURL = &scrUrl;
 770     }
 771
 772     CScraperUrl url;
 773     int retVal = 0;
 774     std::string movieTitle = pItem->GetMovieName(bDirNames);
 775     int movieYear = -1; // hint that movie title was not found
 776     if (result == CInfoScanner::TITLE_NFO)
 777     {
 778       CVideoInfoTag* tag = pItem->GetVideoInfoTag();
 779       movieTitle = tag->GetTitle();
 780       movieYear = tag->GetYear(); // movieYear is expected to be >= 0
 781     }
 782
 783     std::string identifierType;
 784     std::string identifier;
 785     if (info2->IsPython() && CUtil::GetFilenameIdentifier(movieTitle, identifierType, identifier))
 786     {
 787       const std::unordered_map<std::string, std::string> uniqueIDs{{identifierType, identifier}};
 788       if (GetDetails(pItem, uniqueIDs, url, info2,
 789                      (result == CInfoScanner::COMBINED_NFO || result == CInfoScanner::OVERRIDE_NFO)
 790                          ? loader.get()
 791                          : nullptr,
 792                      pDlgProgress))
 793       {
 794         const int dbId = AddVideo(pItem, info2->Content(), bDirNames, useLocal);
 795         if (dbId < 0)
 796           return INFO_ERROR;
 797         if (!m_ignoreVideoVersions && ProcessVideoVersion(VideoDbContentType::MOVIES, dbId))
 798           return INFO_HAVE_ALREADY;
 799         return INFO_ADDED;
 800       }
 801     }
 802
 803     if (pURL && pURL->HasUrls())
 804       url = *pURL;
 805     else if ((retVal = FindVideo(movieTitle, movieYear, info2, url, pDlgProgress)) <= 0)
 806       return retVal < 0 ? INFO_CANCELLED : INFO_NOT_FOUND;
 807
 808     CLog::Log(LOGDEBUG, "VideoInfoScanner: Fetching url '{}' using {} scraper (content: '{}')",
 809               url.GetFirstThumbUrl(), info2->Name(), TranslateContent(info2->Content()));
 810
 811     if (GetDetails(pItem, {}, url, info2,
 812                    (result == CInfoScanner::COMBINED_NFO || result == CInfoScanner::OVERRIDE_NFO)
 813                        ? loader.get()
 814                        : nullptr,
 815                    pDlgProgress))
 816     {
 817       const int dbId = AddVideo(pItem, info2->Content(), bDirNames, useLocal);
 818       if (dbId < 0)
 819         return INFO_ERROR;
 820       if (!m_ignoreVideoVersions && ProcessVideoVersion(VideoDbContentType::MOVIES, dbId))
 821         return INFO_HAVE_ALREADY;
 822       return INFO_ADDED;
 823     }
 824     //! @todo This is not strictly correct as we could fail to download information here or error, or be cancelled
 825     return INFO_NOT_FOUND;
 826   }
 827
 828   CInfoScanner::INFO_RET
 829   CVideoInfoScanner::RetrieveInfoForMusicVideo(CFileItem *pItem,
 830                                                bool bDirNames,
 831                                                ScraperPtr &info2,
 832                                                bool useLocal,
 833                                                CScraperUrl* pURL,
 834                                                CGUIDialogProgress* pDlgProgress)
 835   {
 836     if (pItem->m_bIsFolder || !IsVideo(*pItem) || pItem->IsNFO() ||
 837         (PLAYLIST::IsPlayList(*pItem) && !URIUtils::HasExtension(pItem->GetPath(), ".strm")))
 838       return INFO_NOT_NEEDED;
 839
 840     if (ProgressCancelled(pDlgProgress, 20394, pItem->GetLabel()))
 841       return INFO_CANCELLED;
 842
 843     if (m_database.HasMusicVideoInfo(pItem->GetPath()))
 844       return INFO_HAVE_ALREADY;
 845
 846     if (m_handle)
 847       m_handle->SetText(pItem->GetMovieName(bDirNames));
 848
 849     CInfoScanner::INFO_TYPE result = CInfoScanner::NO_NFO;
 850     CScraperUrl scrUrl;
 851     // handle .nfo files
 852     std::unique_ptr<IVideoInfoTagLoader> loader;
 853     if (useLocal)
 854     {
 855       loader.reset(CVideoInfoTagLoaderFactory::CreateLoader(*pItem, info2, bDirNames));
 856       if (loader)
 857       {
 858         pItem->GetVideoInfoTag()->Reset();
 859         result = loader->Load(*pItem->GetVideoInfoTag(), false);
 860       }
 861     }
 862     if (result == CInfoScanner::FULL_NFO)
 863     {
 864       if (AddVideo(pItem, info2->Content(), bDirNames, true) < 0)
 865         return INFO_ERROR;
 866       return INFO_ADDED;
 867     }
 868     if (result == CInfoScanner::URL_NFO || result == CInfoScanner::COMBINED_NFO)
 869     {
 870       scrUrl = loader->ScraperUrl();
 871       pURL = &scrUrl;
 872     }
 873
 874     CScraperUrl url;
 875     int retVal = 0;
 876     std::string movieTitle = pItem->GetMovieName(bDirNames);
 877     int movieYear = -1; // hint that movie title was not found
 878     if (result == CInfoScanner::TITLE_NFO)
 879     {
 880       CVideoInfoTag* tag = pItem->GetVideoInfoTag();
 881       movieTitle = tag->GetTitle();
 882       movieYear = tag->GetYear(); // movieYear is expected to be >= 0
 883     }
 884
 885     std::string identifierType;
 886     std::string identifier;
 887     if (info2->IsPython() && CUtil::GetFilenameIdentifier(movieTitle, identifierType, identifier))
 888     {
 889       const std::unordered_map<std::string, std::string> uniqueIDs{{identifierType, identifier}};
 890       if (GetDetails(pItem, uniqueIDs, url, info2,
 891                      (result == CInfoScanner::COMBINED_NFO || result == CInfoScanner::OVERRIDE_NFO)
 892                          ? loader.get()
 893                          : nullptr,
 894                      pDlgProgress))
 895       {
 896         if (AddVideo(pItem, info2->Content(), bDirNames, useLocal) < 0)
 897           return INFO_ERROR;
 898         return INFO_ADDED;
 899       }
 900     }
 901
 902     if (pURL && pURL->HasUrls())
 903       url = *pURL;
 904     else if ((retVal = FindVideo(movieTitle, movieYear, info2, url, pDlgProgress)) <= 0)
 905       return retVal < 0 ? INFO_CANCELLED : INFO_NOT_FOUND;
 906
 907     CLog::Log(LOGDEBUG, "VideoInfoScanner: Fetching url '{}' using {} scraper (content: '{}')",
 908               url.GetFirstThumbUrl(), info2->Name(), TranslateContent(info2->Content()));
 909
 910     if (GetDetails(pItem, {}, url, info2,
 911                    (result == CInfoScanner::COMBINED_NFO || result == CInfoScanner::OVERRIDE_NFO)
 912                        ? loader.get()
 913                        : nullptr,
 914                    pDlgProgress))
 915     {
 916       if (AddVideo(pItem, info2->Content(), bDirNames, useLocal) < 0)
 917         return INFO_ERROR;
 918       return INFO_ADDED;
 919     }
 920     //! @todo This is not strictly correct as we could fail to download information here or error, or be cancelled
 921     return INFO_NOT_FOUND;
 922   }
 923
 924   CInfoScanner::INFO_RET
 925   CVideoInfoScanner::RetrieveInfoForEpisodes(CFileItem *item,
 926                                              long showID,
 927                                              const ADDON::ScraperPtr &scraper,
 928                                              bool useLocal,
 929                                              CGUIDialogProgress *progress)
 930   {
 931     // enumerate episodes
 932     EPISODELIST files;
 933     if (!EnumerateSeriesFolder(item, files))
 934       return INFO_HAVE_ALREADY;
 935     if (files.empty()) // no update or no files
 936       return INFO_NOT_NEEDED;
 937
 938     if (m_bStop || (progress && progress->IsCanceled()))
 939       return INFO_CANCELLED;
 940
 941     CVideoInfoTag showInfo;
 942     m_database.GetTvShowInfo("", showInfo, showID);
 943     INFO_RET ret = OnProcessSeriesFolder(files, scraper, useLocal, showInfo, progress);
 944
 945     if (ret == INFO_ADDED)
 946     {
 947       std::map<int, std::map<std::string, std::string>> seasonArt;
 948       m_database.GetTvShowSeasonArt(showID, seasonArt);
 949
 950       bool updateSeasonArt = false;
 951       for (std::map<int, std::map<std::string, std::string>>::const_iterator i = seasonArt.begin(); i != seasonArt.end(); ++i)
 952       {
 953         if (i->second.empty())
 954         {
 955           updateSeasonArt = true;
 956           break;
 957         }
 958       }
 959
 960       if (updateSeasonArt)
 961       {
 962         if (!item->IsPlugin() || scraper->ID() != "metadata.local")
 963         {
 964           CVideoInfoDownloader loader(scraper);
 965           loader.GetArtwork(showInfo);
 966         }
 967         GetSeasonThumbs(showInfo, seasonArt, CVideoThumbLoader::GetArtTypes(MediaTypeSeason), useLocal && !item->IsPlugin());
 968         for (std::map<int, std::map<std::string, std::string> >::const_iterator i = seasonArt.begin(); i != seasonArt.end(); ++i)
 969         {
 970           int seasonID = m_database.AddSeason(showID, i->first);
 971           m_database.SetArtForItem(seasonID, MediaTypeSeason, i->second);
 972         }
 973       }
 974     }
 975     return ret;
 976   }
 977
 978   bool CVideoInfoScanner::EnumerateSeriesFolder(CFileItem* item, EPISODELIST& episodeList)
 979   {
 980     CFileItemList items;
 981     const std::vector<std::string> &regexps = CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_tvshowExcludeFromScanRegExps;
 982
 983     bool bSkip = false;
 984
 985     if (item->m_bIsFolder)
 986     {
 987       /*
 988        * Note: DoScan() will not remove this path as it's not recursing for tvshows.
 989        * Remove this path from the list we're processing in order to avoid hitting
 990        * it twice in the main loop.
 991        */
 992       std::set<std::string>::iterator it = m_pathsToScan.find(item->GetPath());
 993       if (it != m_pathsToScan.end())
 994         m_pathsToScan.erase(it);
 995
 996       if (HasNoMedia(item->GetPath()))
 997         return true;
 998
 999       std::string hash, dbHash;
1000       bool allowEmptyHash = false;
1001       if (item->IsPlugin())
1002       {
1003         // if plugin has already calculated a hash for directory contents - use it
1004         // in this case we don't need to get directory listing from plugin for hash checking
1005         if (item->HasProperty("hash"))
1006         {
1007           hash = item->GetProperty("hash").asString();
1008           allowEmptyHash = true;
1009         }
1010       }
1011       else if (CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_bVideoLibraryUseFastHash)
1012         hash = GetRecursiveFastHash(item->GetPath(), regexps);
1013
1014       if (m_database.GetPathHash(item->GetPath(), dbHash) && (allowEmptyHash || !hash.empty()) && StringUtils::EqualsNoCase(dbHash, hash))
1015       {
1016         // fast hashes match - no need to process anything
1017         bSkip = true;
1018       }
1019
1020       // fast hash cannot be computed or we need to rescan. fetch the listing.
1021       if (!bSkip)
1022       {
1023         int flags = DIR_FLAG_DEFAULTS;
1024         if (!hash.empty())
1025           flags |= DIR_FLAG_NO_FILE_INFO;
1026
1027         // Listing that ignores files inside and below folders containing .nomedia files.
1028         CDirectory::EnumerateDirectory(
1029             item->GetPath(), [&items](const std::shared_ptr<CFileItem>& item) { items.Add(item); },
1030             [this](const std::shared_ptr<CFileItem>& folder)
1031             { return !HasNoMedia(folder->GetPath()); },
1032             true, CServiceBroker::GetFileExtensionProvider().GetVideoExtensions(), flags);
1033
1034         // fast hash failed - compute slow one
1035         if (hash.empty())
1036         {
1037           GetPathHash(items, hash);
1038           if (StringUtils::EqualsNoCase(dbHash, hash))
1039           {
1040             // slow hashes match - no need to process anything
1041             bSkip = true;
1042           }
1043         }
1044       }
1045
1046       if (bSkip)
1047       {
1048         CLog::Log(LOGDEBUG, "VideoInfoScanner: Skipping dir '{}' due to no change",
1049                   CURL::GetRedacted(item->GetPath()));
1050         // update our dialog with our progress
1051         if (m_handle)
1052           OnDirectoryScanned(item->GetPath());
1053         return false;
1054       }
1055
1056       if (dbHash.empty())
1057         CLog::Log(LOGDEBUG, "VideoInfoScanner: Scanning dir '{}' as not in the database",
1058                   CURL::GetRedacted(item->GetPath()));
1059       else
1060         CLog::Log(LOGDEBUG, "VideoInfoScanner: Rescanning dir '{}' due to change ({} != {})",
1061                   CURL::GetRedacted(item->GetPath()), dbHash, hash);
1062
1063       if (m_bClean)
1064       {
1065         m_pathsToClean.insert(m_database.GetPathId(item->GetPath()));
1066         m_database.GetPathsForTvShow(m_database.GetTvShowId(item->GetPath()), m_pathsToClean);
1067       }
1068       item->SetProperty("hash", hash);
1069     }
1070     else
1071     {
1072       CFileItemPtr newItem(new CFileItem(*item));
1073       items.Add(newItem);
1074     }
1075
1076     /*
1077     stack down any dvd folders
1078     need to sort using the full path since this is a collapsed recursive listing of all subdirs
1079     video_ts.ifo files should sort at the top of a dvd folder in ascending order
1080
1081     /foo/bar/video_ts.ifo
1082     /foo/bar/vts_x_y.ifo
1083     /foo/bar/vts_x_y.vob
1084     */
1085
1086     // since we're doing this now anyway, should other items be stacked?
1087     items.Sort(SortByPath, SortOrderAscending);
1088
1089     // If found VIDEO_TS.IFO or INDEX.BDMV then we are dealing with Blu-ray or DVD files on disc
1090     // somewhere in the directory tree. Assume that all other files/folders in the same folder
1091     // with VIDEO_TS or BDMV can be ignored.
1092     // THere can be a BACKUP/INDEX.BDMV which needs to be ignored (and broke the old while loop here)
1093
1094     // Get folders to remove
1095     std::vector<std::string> foldersToRemove;
1096     for (const auto& item : items)
1097     {
1098       const std::string file = StringUtils::ToUpper(item->GetPath());
1099       if (file.find("VIDEO_TS.IFO") != std::string::npos)
1100         foldersToRemove.emplace_back(StringUtils::ToUpper(URIUtils::GetDirectory(file)));
1101       if (file.find("INDEX.BDMV") != std::string::npos &&
1102           file.find("BACKUP/INDEX.BDMV") == std::string::npos)
1103         foldersToRemove.emplace_back(
1104             StringUtils::ToUpper(URIUtils::GetParentPath(URIUtils::GetDirectory(file))));
1105     }
1106
1107     // Remove folders
1108     items.erase(
1109         std::remove_if(items.begin(), items.end(),
1110                        [&](const CFileItemPtr& i)
1111                        {
1112                          const std::string fileAndPath(StringUtils::ToUpper(i->GetPath()));
1113                          std::string file;
1114                          std::string path;
1115                          URIUtils::Split(fileAndPath, path, file);
1116                          return (std::count_if(foldersToRemove.begin(), foldersToRemove.end(),
1117                                                [&](const std::string& removePath)
1118                                                { return path.rfind(removePath, 0) == 0; }) > 0) &&
1119                                 file != "VIDEO_TS.IFO" &&
1120                                 (file != "INDEX.BDMV" ||
1121                                  fileAndPath.find("BACKUP/INDEX.BDMV") != std::string::npos);
1122                        }),
1123         items.end());
1124
1125     // enumerate
1126     for (int i=0;i<items.Size();++i)
1127     {
1128       if (items[i]->m_bIsFolder)
1129         continue;
1130       std::string strPath = URIUtils::GetDirectory(items[i]->GetPath());
1131       URIUtils::RemoveSlashAtEnd(strPath); // want no slash for the test that follows
1132
1133       if (StringUtils::EqualsNoCase(URIUtils::GetFileName(strPath), "sample"))
1134         continue;
1135
1136       // Discard all exclude files defined by regExExcludes
1137       if (CUtil::ExcludeFileOrFolder(items[i]->GetPath(), regexps))
1138         continue;
1139
1140       /*
1141        * Check if the media source has already set the season and episode or original air date in
1142        * the VideoInfoTag. If it has, do not try to parse any of them from the file path to avoid
1143        * any false positive matches.
1144        */
1145       if (ProcessItemByVideoInfoTag(items[i].get(), episodeList))
1146         continue;
1147
1148       if (!EnumerateEpisodeItem(items[i].get(), episodeList))
1149         CLog::Log(LOGDEBUG, "VideoInfoScanner: Could not enumerate file {}", CURL::GetRedacted(items[i]->GetPath()));
1150     }
1151     return true;
1152   }
1153
1154   bool CVideoInfoScanner::ProcessItemByVideoInfoTag(const CFileItem *item, EPISODELIST &episodeList)
1155   {
1156     if (!item->HasVideoInfoTag())
1157       return false;
1158
1159     const CVideoInfoTag* tag = item->GetVideoInfoTag();
1160     bool isValid = false;
1161     /*
1162      * First check the season and episode number. This takes precedence over the original air
1163      * date and episode title. Must be a valid season and episode number combination.
1164      */
1165     if (tag->m_iSeason > -1 && tag->m_iEpisode > 0)
1166       isValid = true;
1167
1168     // episode 0 with non-zero season is valid! (e.g. prequel episode)
1169     if (item->IsPlugin() && tag->m_iSeason > 0 && tag->m_iEpisode >= 0)
1170       isValid = true;
1171
1172     if (isValid)
1173     {
1174       EPISODE episode;
1175       episode.strPath = item->GetPath();
1176       episode.iSeason = tag->m_iSeason;
1177       episode.iEpisode = tag->m_iEpisode;
1178       episode.isFolder = false;
1179       // save full item for plugin source
1180       if (item->IsPlugin())
1181         episode.item = std::make_shared<CFileItem>(*item);
1182       episodeList.push_back(episode);
1183       CLog::Log(LOGDEBUG, "{} - found match for: {}. Season {}, Episode {}", __FUNCTION__,
1184                 CURL::GetRedacted(episode.strPath), episode.iSeason, episode.iEpisode);
1185       return true;
1186     }
1187
1188     /*
1189      * Next preference is the first aired date. If it exists use that for matching the TV Show
1190      * information. Also set the title in case there are multiple matches for the first aired date.
1191      */
1192     if (tag->m_firstAired.IsValid())
1193     {
1194       EPISODE episode;
1195       episode.strPath = item->GetPath();
1196       episode.strTitle = tag->m_strTitle;
1197       episode.isFolder = false;
1198       /*
1199        * Set season and episode to -1 to indicate to use the aired date.
1200        */
1201       episode.iSeason = -1;
1202       episode.iEpisode = -1;
1203       /*
1204        * The first aired date string must be parseable.
1205        */
1206       episode.cDate = item->GetVideoInfoTag()->m_firstAired;
1207       episodeList.push_back(episode);
1208       CLog::Log(LOGDEBUG, "{} - found match for: '{}', firstAired: '{}' = '{}', title: '{}'",
1209                 __FUNCTION__, CURL::GetRedacted(episode.strPath),
1210                 tag->m_firstAired.GetAsDBDateTime(), episode.cDate.GetAsLocalizedDate(),
1211                 episode.strTitle);
1212       return true;
1213     }
1214
1215     /*
1216      * Next preference is the episode title. If it exists use that for matching the TV Show
1217      * information.
1218      */
1219     if (!tag->m_strTitle.empty())
1220     {
1221       EPISODE episode;
1222       episode.strPath = item->GetPath();
1223       episode.strTitle = tag->m_strTitle;
1224       episode.isFolder = false;
1225       /*
1226        * Set season and episode to -1 to indicate to use the title.
1227        */
1228       episode.iSeason = -1;
1229       episode.iEpisode = -1;
1230       episodeList.push_back(episode);
1231       CLog::Log(LOGDEBUG, "{} - found match for: '{}', title: '{}'", __FUNCTION__,
1232                 CURL::GetRedacted(episode.strPath), episode.strTitle);
1233       return true;
1234     }
1235
1236     /*
1237      * There is no further episode information available if both the season and episode number have
1238      * been set to 0. Return the match as true so no further matching is attempted, but don't add it
1239      * to the episode list.
1240      */
1241     if (tag->m_iSeason == 0 && tag->m_iEpisode == 0)
1242     {
1243       CLog::Log(LOGDEBUG,
1244                 "{} - found exclusion match for: {}. Both Season and Episode are 0. Item will be "
1245                 "ignored for scanning.",
1246                 __FUNCTION__, CURL::GetRedacted(item->GetPath()));
1247       return true;
1248     }
1249
1250     return false;
1251   }
1252
1253   bool CVideoInfoScanner::EnumerateEpisodeItem(const CFileItem *item, EPISODELIST& episodeList)
1254   {
1255     SETTINGS_TVSHOWLIST expression = CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_tvshowEnumRegExps;
1256
1257     std::string strLabel;
1258
1259     // remove path to main file if it's a bd or dvd folder to regex the right (folder) name
1260     if (item->IsOpticalMediaFile())
1261     {
1262       strLabel = item->GetLocalMetadataPath();
1263       URIUtils::RemoveSlashAtEnd(strLabel);
1264     }
1265     else
1266       strLabel = item->GetPath();
1267
1268     // URLDecode in case an episode is on a http/https/dav/davs:// source and URL-encoded like foo%201x01%20bar.avi
1269     strLabel = CURL::Decode(CURL::GetRedacted(strLabel));
1270
1271     for (unsigned int i=0;i<expression.size();++i)
1272     {
1273       CRegExp reg(true, CRegExp::autoUtf8);
1274       if (!reg.RegComp(expression[i].regexp))
1275         continue;
1276
1277       int regexppos, regexp2pos;
1278       //CLog::Log(LOGDEBUG,"running expression {} on {}",expression[i].regexp,strLabel);
1279       if ((regexppos = reg.RegFind(strLabel.c_str())) < 0)
1280         continue;
1281
1282       EPISODE episode;
1283       episode.strPath = item->GetPath();
1284       episode.iSeason = -1;
1285       episode.iEpisode = -1;
1286       episode.cDate.SetValid(false);
1287       episode.isFolder = false;
1288
1289       bool byDate = expression[i].byDate ? true : false;
1290       bool byTitle = expression[i].byTitle;
1291       int defaultSeason = expression[i].defaultSeason;
1292
1293       if (byDate)
1294       {
1295         if (!GetAirDateFromRegExp(reg, episode))
1296           continue;
1297
1298         CLog::Log(LOGDEBUG, "VideoInfoScanner: Found date based match {} ({}) [{}]",
1299                   CURL::GetRedacted(episode.strPath), episode.cDate.GetAsLocalizedDate(),
1300                   expression[i].regexp);
1301       }
1302       else if (byTitle)
1303       {
1304         if (!GetEpisodeTitleFromRegExp(reg, episode))
1305           continue;
1306
1307         CLog::Log(LOGDEBUG, "VideoInfoScanner: Found title based match {} ({}) [{}]",
1308                   CURL::GetRedacted(episode.strPath), episode.strTitle, expression[i].regexp);
1309       }
1310       else
1311       {
1312         if (!GetEpisodeAndSeasonFromRegExp(reg, episode, defaultSeason))
1313           continue;
1314
1315         CLog::Log(LOGDEBUG, "VideoInfoScanner: Found episode match {} (s{}e{}) [{}]",
1316                   CURL::GetRedacted(episode.strPath), episode.iSeason, episode.iEpisode,
1317                   expression[i].regexp);
1318       }
1319
1320       // Grab the remainder from first regexp run
1321       // as second run might modify or empty it.
1322       std::string remainder(reg.GetMatch(3));
1323
1324       /*
1325        * Check if the files base path is a dedicated folder that contains
1326        * only this single episode. If season and episode match with the
1327        * actual media file, we set episode.isFolder to true.
1328        */
1329       std::string strBasePath = item->GetBaseMoviePath(true);
1330       URIUtils::RemoveSlashAtEnd(strBasePath);
1331       strBasePath = URIUtils::GetFileName(strBasePath);
1332
1333       if (reg.RegFind(strBasePath.c_str()) > -1)
1334       {
1335         EPISODE parent;
1336         if (byDate)
1337         {
1338           GetAirDateFromRegExp(reg, parent);
1339           if (episode.cDate == parent.cDate)
1340             episode.isFolder = true;
1341         }
1342         else
1343         {
1344           GetEpisodeAndSeasonFromRegExp(reg, parent, defaultSeason);
1345           if (episode.iSeason == parent.iSeason && episode.iEpisode == parent.iEpisode)
1346             episode.isFolder = true;
1347         }
1348       }
1349
1350       // add what we found by now
1351       episodeList.push_back(episode);
1352
1353       CRegExp reg2(true, CRegExp::autoUtf8);
1354       // check the remainder of the string for any further episodes.
1355       if (!byDate && reg2.RegComp(CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_tvshowMultiPartEnumRegExp))
1356       {
1357         int offset = 0;
1358
1359         // we want "long circuit" OR below so that both offsets are evaluated
1360         while (static_cast<int>((regexp2pos = reg2.RegFind(remainder.c_str() + offset)) > -1) |
1361                static_cast<int>((regexppos = reg.RegFind(remainder.c_str() + offset)) > -1))
1362         {
1363           if (((regexppos <= regexp2pos) && regexppos != -1) ||
1364              (regexppos >= 0 && regexp2pos == -1))
1365           {
1366             GetEpisodeAndSeasonFromRegExp(reg, episode, defaultSeason);
1367
1368             CLog::Log(LOGDEBUG, "VideoInfoScanner: Adding new season {}, multipart episode {} [{}]",
1369                       episode.iSeason, episode.iEpisode,
1370                       CServiceBroker::GetSettingsComponent()
1371                           ->GetAdvancedSettings()
1372                           ->m_tvshowMultiPartEnumRegExp);
1373
1374             episodeList.push_back(episode);
1375             remainder = reg.GetMatch(3);
1376             offset = 0;
1377           }
1378           else if (((regexp2pos < regexppos) && regexp2pos != -1) ||
1379                    (regexp2pos >= 0 && regexppos == -1))
1380           {
1381             episode.iEpisode = atoi(reg2.GetMatch(1).c_str());
1382             CLog::Log(LOGDEBUG, "VideoInfoScanner: Adding multipart episode {} [{}]",
1383                       episode.iEpisode,
1384                       CServiceBroker::GetSettingsComponent()
1385                           ->GetAdvancedSettings()
1386                           ->m_tvshowMultiPartEnumRegExp);
1387             episodeList.push_back(episode);
1388             offset += regexp2pos + reg2.GetFindLen();
1389           }
1390         }
1391       }
1392       return true;
1393     }
1394     return false;
1395   }
1396
1397   bool CVideoInfoScanner::GetEpisodeAndSeasonFromRegExp(CRegExp &reg, EPISODE &episodeInfo, int defaultSeason)
1398   {
1399     std::string season(reg.GetMatch(1));
1400     std::string episode(reg.GetMatch(2));
1401
1402     if (!season.empty() || !episode.empty())
1403     {
1404       char* endptr = NULL;
1405       if (season.empty() && !episode.empty())
1406       { // no season specified -> assume defaultSeason
1407         episodeInfo.iSeason = defaultSeason;
1408         if ((episodeInfo.iEpisode = CUtil::TranslateRomanNumeral(episode.c_str())) == -1)
1409           episodeInfo.iEpisode = strtol(episode.c_str(), &endptr, 10);
1410       }
1411       else if (!season.empty() && episode.empty())
1412       { // no episode specification -> assume defaultSeason
1413         episodeInfo.iSeason = defaultSeason;
1414         if ((episodeInfo.iEpisode = CUtil::TranslateRomanNumeral(season.c_str())) == -1)
1415           episodeInfo.iEpisode = atoi(season.c_str());
1416       }
1417       else
1418       { // season and episode specified
1419         episodeInfo.iSeason = atoi(season.c_str());
1420         episodeInfo.iEpisode = strtol(episode.c_str(), &endptr, 10);
1421       }
1422       if (endptr)
1423       {
1424         if (isalpha(*endptr))
1425           episodeInfo.iSubepisode = *endptr - (islower(*endptr) ? 'a' : 'A') + 1;
1426         else if (*endptr == '.')
1427           episodeInfo.iSubepisode = atoi(endptr+1);
1428       }
1429       return true;
1430     }
1431     return false;
1432   }
1433
1434   bool CVideoInfoScanner::GetAirDateFromRegExp(CRegExp &reg, EPISODE &episodeInfo)
1435   {
1436     std::string param1(reg.GetMatch(1));
1437     std::string param2(reg.GetMatch(2));
1438     std::string param3(reg.GetMatch(3));
1439
1440     if (!param1.empty() && !param2.empty() && !param3.empty())
1441     {
1442       // regular expression by date
1443       int len1 = param1.size();
1444       int len2 = param2.size();
1445       int len3 = param3.size();
1446
1447       if (len1==4 && len2==2 && len3==2)
1448       {
1449         // yyyy mm dd format
1450         episodeInfo.cDate.SetDate(atoi(param1.c_str()), atoi(param2.c_str()), atoi(param3.c_str()));
1451       }
1452       else if (len1==2 && len2==2 && len3==4)
1453       {
1454         // mm dd yyyy format
1455         episodeInfo.cDate.SetDate(atoi(param3.c_str()), atoi(param1.c_str()), atoi(param2.c_str()));
1456       }
1457     }
1458     return episodeInfo.cDate.IsValid();
1459   }
1460
1461   bool CVideoInfoScanner::GetEpisodeTitleFromRegExp(CRegExp& reg, EPISODE& episodeInfo)
1462   {
1463     std::string param1(reg.GetMatch(1));
1464
1465     if (!param1.empty())
1466     {
1467       episodeInfo.strTitle = param1;
1468       return true;
1469     }
1470     return false;
1471   }
1472
1473   long CVideoInfoScanner::AddVideo(CFileItem *pItem, const CONTENT_TYPE &content, bool videoFolder /* = false */, bool useLocal /* = true */, const CVideoInfoTag *showInfo /* = NULL */, bool libraryImport /* = false */)
1474   {
1475     // ensure our database is open (this can get called via other classes)
1476     if (!m_database.Open())
1477       return -1;
1478
1479     if (!libraryImport)
1480       GetArtwork(pItem, content, videoFolder, useLocal && !pItem->IsPlugin(), showInfo ? showInfo->m_strPath : "");
1481
1482     // ensure the art map isn't completely empty by specifying an empty thumb
1483     std::map<std::string, std::string> art = pItem->GetArt();
1484     if (art.empty())
1485       art["thumb"] = "";
1486
1487     CVideoInfoTag &movieDetails = *pItem->GetVideoInfoTag();
1488     if (movieDetails.m_basePath.empty())
1489       movieDetails.m_basePath = pItem->GetBaseMoviePath(videoFolder);
1490     movieDetails.m_parentPathID = m_database.AddPath(URIUtils::GetParentPath(movieDetails.m_basePath));
1491
1492     movieDetails.m_strFileNameAndPath = pItem->GetPath();
1493
1494     if (pItem->m_bIsFolder)
1495       movieDetails.m_strPath = pItem->GetPath();
1496
1497     std::string strTitle(movieDetails.m_strTitle);
1498
1499     if (showInfo && content == CONTENT_TVSHOWS)
1500     {
1501       strTitle = StringUtils::Format("{} - {}x{} - {}", showInfo->m_strTitle,
1502                                      movieDetails.m_iSeason, movieDetails.m_iEpisode, strTitle);
1503     }
1504
1505     /* As HasStreamDetails() returns true for TV shows (because the scraper calls SetVideoInfoTag()
1506      * directly to set the duration) a better test is just to see if we have any common flag info
1507      * missing.  If we have already read an nfo file then this data should be populated, otherwise
1508      * get it from the video file */
1509
1510     if (CServiceBroker::GetSettingsComponent()->GetSettings()->GetBool(
1511             CSettings::SETTING_MYVIDEOS_EXTRACTFLAGS))
1512     {
1513       const auto& strmdetails = movieDetails.m_streamDetails;
1514       if (strmdetails.GetVideoCodec(1).empty() || strmdetails.GetVideoHeight(1) == 0 ||
1515           strmdetails.GetVideoWidth(1) == 0 || strmdetails.GetVideoDuration(1) == 0)
1516
1517       {
1518         CDVDFileInfo::GetFileStreamDetails(pItem);
1519         CLog::Log(LOGDEBUG, "VideoInfoScanner: Extracted filestream details from video file {}",
1520                   CURL::GetRedacted(pItem->GetPath()));
1521       }
1522     }
1523
1524     CLog::Log(LOGDEBUG, "VideoInfoScanner: Adding new item to {}:{}", TranslateContent(content), CURL::GetRedacted(pItem->GetPath()));
1525     long lResult = -1;
1526
1527     if (content == CONTENT_MOVIES)
1528     {
1529       // find local trailer first
1530       std::string strTrailer = pItem->FindTrailer();
1531       if (!strTrailer.empty())
1532         movieDetails.m_strTrailer = strTrailer;
1533
1534       // Deal with 'Disc n' subdirectories
1535       const std::string discNum{
1536           CUtil::GetDiscNumberFromPath(URIUtils::GetParentPath(movieDetails.m_strFileNameAndPath))};
1537       if (!discNum.empty())
1538       {
1539         if (movieDetails.m_set.title.empty())
1540         {
1541           const std::string setName{m_database.GetSetByNameLike(movieDetails.m_strTitle)};
1542           if (!setName.empty())
1543           {
1544             // Add movie to existing set
1545             movieDetails.SetSet(setName);
1546           }
1547           else
1548           {
1549             // Create set, then add movie to the set
1550             const int idSet{m_database.AddSet(movieDetails.m_strTitle)};
1551             m_database.SetArtForItem(idSet, MediaTypeVideoCollection, art);
1552             movieDetails.SetSet(movieDetails.m_strTitle);
1553           }
1554         }
1555
1556         // Add '(Disc n)' to title (in local language)
1557         movieDetails.m_strTitle =
1558             StringUtils::Format(g_localizeStrings.Get(29995), movieDetails.m_strTitle, discNum);
1559       }
1560
1561       lResult = m_database.SetDetailsForMovie(movieDetails, art);
1562       movieDetails.m_iDbId = lResult;
1563       movieDetails.m_type = MediaTypeMovie;
1564
1565       // setup links to shows if the linked shows are in the db
1566       for (unsigned int i=0; i < movieDetails.m_showLink.size(); ++i)
1567       {
1568         CFileItemList items;
1569         m_database.GetTvShowsByName(movieDetails.m_showLink[i], items);
1570         if (items.Size())
1571           m_database.LinkMovieToTvshow(lResult, items[0]->GetVideoInfoTag()->m_iDbId, false);
1572         else
1573           CLog::Log(LOGDEBUG, "VideoInfoScanner: Failed to link movie {} to show {}",
1574                     movieDetails.m_strTitle, movieDetails.m_showLink[i]);
1575       }
1576     }
1577     else if (content == CONTENT_TVSHOWS)
1578     {
1579       if (pItem->m_bIsFolder)
1580       {
1581         /*
1582          multipaths are not stored in the database, so in the case we have one,
1583          we split the paths, and compute the parent paths in each case.
1584          */
1585         std::vector<std::string> multipath;
1586         if (!URIUtils::IsMultiPath(pItem->GetPath()) || !CMultiPathDirectory::GetPaths(pItem->GetPath(), multipath))
1587           multipath.push_back(pItem->GetPath());
1588         std::vector<std::pair<std::string, std::string> > paths;
1589         for (std::vector<std::string>::const_iterator i = multipath.begin(); i != multipath.end(); ++i)
1590           paths.emplace_back(*i, URIUtils::GetParentPath(*i));
1591
1592         std::map<int, std::map<std::string, std::string> > seasonArt;
1593
1594         if (!libraryImport)
1595           GetSeasonThumbs(movieDetails, seasonArt, CVideoThumbLoader::GetArtTypes(MediaTypeSeason), useLocal && !pItem->IsPlugin());
1596
1597         lResult = m_database.SetDetailsForTvShow(paths, movieDetails, art, seasonArt);
1598         movieDetails.m_iDbId = lResult;
1599         movieDetails.m_type = MediaTypeTvShow;
1600       }
1601       else
1602       {
1603         // we add episode then set details, as otherwise set details will delete the
1604         // episode then add, which breaks multi-episode files.
1605         int idShow = showInfo ? showInfo->m_iDbId : -1;
1606         int idEpisode = m_database.AddNewEpisode(idShow, movieDetails);
1607         lResult = m_database.SetDetailsForEpisode(movieDetails, art, idShow, idEpisode);
1608         movieDetails.m_iDbId = lResult;
1609         movieDetails.m_type = MediaTypeEpisode;
1610         movieDetails.m_strShowTitle = showInfo ? showInfo->m_strTitle : "";
1611         if (movieDetails.m_EpBookmark.timeInSeconds > 0)
1612         {
1613           movieDetails.m_strFileNameAndPath = pItem->GetPath();
1614           movieDetails.m_EpBookmark.seasonNumber = movieDetails.m_iSeason;
1615           movieDetails.m_EpBookmark.episodeNumber = movieDetails.m_iEpisode;
1616           m_database.AddBookMarkForEpisode(movieDetails, movieDetails.m_EpBookmark);
1617         }
1618       }
1619     }
1620     else if (content == CONTENT_MUSICVIDEOS)
1621     {
1622       lResult = m_database.SetDetailsForMusicVideo(movieDetails, art);
1623       movieDetails.m_iDbId = lResult;
1624       movieDetails.m_type = MediaTypeMusicVideo;
1625     }
1626
1627     if (!pItem->m_bIsFolder)
1628     {
1629       const auto advancedSettings = CServiceBroker::GetSettingsComponent()->GetAdvancedSettings();
1630       if ((libraryImport || advancedSettings->m_bVideoLibraryImportWatchedState) &&
1631           (movieDetails.IsPlayCountSet() || movieDetails.m_lastPlayed.IsValid()))
1632         m_database.SetPlayCount(*pItem, movieDetails.GetPlayCount(), movieDetails.m_lastPlayed);
1633
1634       if ((libraryImport || advancedSettings->m_bVideoLibraryImportResumePoint) &&
1635           movieDetails.GetResumePoint().IsSet())
1636         m_database.AddBookMarkToFile(pItem->GetPath(), movieDetails.GetResumePoint(), CBookmark::RESUME);
1637     }
1638
1639     m_database.Close();
1640
1641     CFileItemPtr itemCopy = std::make_shared<CFileItem>(*pItem);
1642     CVariant data;
1643     data["added"] = true;
1644     if (m_bRunning)
1645       data["transaction"] = true;
1646     CServiceBroker::GetAnnouncementManager()->Announce(ANNOUNCEMENT::VideoLibrary, "OnUpdate",
1647                                                        itemCopy, data);
1648     return lResult;
1649   }
1650
1651   std::string ContentToMediaType(CONTENT_TYPE content, bool folder)
1652   {
1653     switch (content)
1654     {
1655       case CONTENT_MOVIES:
1656         return MediaTypeMovie;
1657       case CONTENT_MUSICVIDEOS:
1658         return MediaTypeMusicVideo;
1659       case CONTENT_TVSHOWS:
1660         return folder ? MediaTypeTvShow : MediaTypeEpisode;
1661       default:
1662         return "";
1663     }
1664   }
1665
1666   VideoDbContentType ContentToVideoDbType(CONTENT_TYPE content)
1667   {
1668     switch (content)
1669     {
1670       case CONTENT_MOVIES:
1671         return VideoDbContentType::MOVIES;
1672       case CONTENT_MUSICVIDEOS:
1673         return VideoDbContentType::MUSICVIDEOS;
1674       case CONTENT_TVSHOWS:
1675         return VideoDbContentType::EPISODES;
1676       default:
1677         return VideoDbContentType::UNKNOWN;
1678     }
1679   }
1680
1681   std::string CVideoInfoScanner::GetArtTypeFromSize(unsigned int width, unsigned int height)
1682   {
1683     std::string type = "thumb";
1684     if (width*5 < height*4)
1685       type = "poster";
1686     else if (width*1 > height*4)
1687       type = "banner";
1688     return type;
1689   }
1690
1691   std::string CVideoInfoScanner::GetMovieSetInfoFolder(const std::string& setTitle)
1692   {
1693     if (setTitle.empty())
1694       return "";
1695     std::string path = CServiceBroker::GetSettingsComponent()->GetSettings()->GetString(
1696         CSettings::SETTING_VIDEOLIBRARY_MOVIESETSFOLDER);
1697     if (path.empty())
1698       return "";
1699     path = URIUtils::AddFileToFolder(path, CUtil::MakeLegalFileName(setTitle, LEGAL_WIN32_COMPAT));
1700     URIUtils::AddSlashAtEnd(path);
1701     CLog::Log(LOGDEBUG,
1702         "VideoInfoScanner: Looking for local artwork for movie set '{}' in folder '{}'",
1703         setTitle,
1704         CURL::GetRedacted(path));
1705     return CDirectory::Exists(path) ? path : "";
1706   }
1707
1708   void CVideoInfoScanner::AddLocalItemArtwork(CGUIListItem::ArtMap& itemArt,
1709     const std::vector<std::string>& wantedArtTypes, const std::string& itemPath,
1710     bool addAll, bool exactName)
1711   {
1712     std::string path = URIUtils::GetDirectory(itemPath);
1713     if (path.empty())
1714       return;
1715
1716     CFileItemList availableArtFiles;
1717     CDirectory::GetDirectory(path, availableArtFiles,
1718         CServiceBroker::GetFileExtensionProvider().GetPictureExtensions(),
1719         DIR_FLAG_NO_FILE_DIRS | DIR_FLAG_READ_CACHE | DIR_FLAG_NO_FILE_INFO);
1720
1721     std::string baseFilename = URIUtils::GetFileName(itemPath);
1722     if (!baseFilename.empty())
1723     {
1724       URIUtils::RemoveExtension(baseFilename);
1725       baseFilename.append("-");
1726     }
1727
1728     for (const auto& artFile : availableArtFiles)
1729     {
1730       std::string candidate = URIUtils::GetFileName(artFile->GetPath());
1731
1732       bool matchesFilename =
1733         !baseFilename.empty() && StringUtils::StartsWith(candidate, baseFilename);
1734       if (!baseFilename.empty() && !matchesFilename)
1735         continue;
1736
1737       if (matchesFilename)
1738         candidate.erase(0, baseFilename.length());
1739       URIUtils::RemoveExtension(candidate);
1740       StringUtils::ToLower(candidate);
1741
1742       // move 'folder' to thumb / poster / banner based on aspect ratio
1743       // if such artwork doesn't already exist
1744       if (!matchesFilename && StringUtils::EqualsNoCase(candidate, "folder") &&
1745         !CVideoThumbLoader::IsArtTypeInWhitelist("folder", wantedArtTypes, exactName))
1746       {
1747         // cache the image to determine sizing
1748         CTextureDetails details;
1749         if (CServiceBroker::GetTextureCache()->CacheImage(artFile->GetPath(), details))
1750         {
1751           candidate = GetArtTypeFromSize(details.width, details.height);
1752           if (itemArt.find(candidate) != itemArt.end())
1753             continue;
1754         }
1755       }
1756
1757       if ((addAll && CVideoThumbLoader::IsValidArtType(candidate)) ||
1758         CVideoThumbLoader::IsArtTypeInWhitelist(candidate, wantedArtTypes, exactName))
1759       {
1760         itemArt[candidate] = artFile->GetPath();
1761       }
1762     }
1763   }
1764
1765   void CVideoInfoScanner::GetArtwork(CFileItem *pItem, const CONTENT_TYPE &content, bool bApplyToDir, bool useLocal, const std::string &actorArtPath)
1766   {
1767     int artLevel = CServiceBroker::GetSettingsComponent()->GetSettings()->GetInt(
1768         CSettings::SETTING_VIDEOLIBRARY_ARTWORK_LEVEL);
1769     if (artLevel == CSettings::VIDEOLIBRARY_ARTWORK_LEVEL_NONE)
1770       return;
1771
1772     CVideoInfoTag &movieDetails = *pItem->GetVideoInfoTag();
1773     movieDetails.m_fanart.Unpack();
1774     movieDetails.m_strPictureURL.Parse();
1775
1776     CGUIListItem::ArtMap art = pItem->GetArt();
1777
1778     // get and cache thumb images
1779     std::string mediaType = ContentToMediaType(content, pItem->m_bIsFolder);
1780     std::vector<std::string> artTypes = CVideoThumbLoader::GetArtTypes(mediaType);
1781     bool moviePartOfSet = content == CONTENT_MOVIES && !movieDetails.m_set.title.empty();
1782     std::vector<std::string> movieSetArtTypes;
1783     if (moviePartOfSet)
1784     {
1785       movieSetArtTypes = CVideoThumbLoader::GetArtTypes(MediaTypeVideoCollection);
1786       for (const std::string& artType : movieSetArtTypes)
1787         artTypes.push_back("set." + artType);
1788     }
1789     bool addAll = artLevel == CSettings::VIDEOLIBRARY_ARTWORK_LEVEL_ALL;
1790     bool exactName = artLevel == CSettings::VIDEOLIBRARY_ARTWORK_LEVEL_BASIC;
1791     // find local art
1792     if (useLocal)
1793     {
1794       if (!pItem->SkipLocalArt())
1795       {
1796         bool useFolder = false;
1797         if (bApplyToDir && (content == CONTENT_MOVIES || content == CONTENT_MUSICVIDEOS))
1798         {
1799           std::string filename = ART::GetLocalArtBaseFilename(*pItem, useFolder);
1800           std::string directory = URIUtils::GetDirectory(filename);
1801           if (filename != directory)
1802             AddLocalItemArtwork(art, artTypes, directory, addAll, exactName);
1803         }
1804
1805         // Reset useFolder to false as GetLocalArtBaseFilename may modify it in
1806         // the previous call.
1807         useFolder = false;
1808
1809         AddLocalItemArtwork(art, artTypes, ART::GetLocalArtBaseFilename(*pItem, useFolder), addAll,
1810                             exactName);
1811       }
1812
1813       if (moviePartOfSet)
1814       {
1815         std::string movieSetInfoPath = GetMovieSetInfoFolder(movieDetails.m_set.title);
1816         if (!movieSetInfoPath.empty())
1817         {
1818           CGUIListItem::ArtMap movieSetArt;
1819           AddLocalItemArtwork(movieSetArt, movieSetArtTypes, movieSetInfoPath, addAll, exactName);
1820           for (const auto& artItem : movieSetArt)
1821           {
1822             art["set." + artItem.first] = artItem.second;
1823           }
1824         }
1825       }
1826     }
1827
1828     // find embedded art
1829     if (pItem->HasVideoInfoTag() && !pItem->GetVideoInfoTag()->m_coverArt.empty())
1830     {
1831       for (auto& it : pItem->GetVideoInfoTag()->m_coverArt)
1832       {
1833         if ((addAll || CVideoThumbLoader::IsArtTypeInWhitelist(it.m_type, artTypes, exactName)) &&
1834           art.find(it.m_type) == art.end())
1835         {
1836           std::string thumb = IMAGE_FILES::URLFromFile(pItem->GetPath(), "video_" + it.m_type);
1837           art.insert(std::make_pair(it.m_type, thumb));
1838         }
1839       }
1840     }
1841
1842     // add online fanart (treated separately due to it being stored in m_fanart)
1843     if ((addAll || CVideoThumbLoader::IsArtTypeInWhitelist("fanart", artTypes, exactName)) &&
1844       art.find("fanart") == art.end())
1845     {
1846       std::string fanart = pItem->GetVideoInfoTag()->m_fanart.GetImageURL();
1847       if (!fanart.empty())
1848         art.insert(std::make_pair("fanart", fanart));
1849     }
1850
1851     // add online art
1852     for (const auto& url : pItem->GetVideoInfoTag()->m_strPictureURL.GetUrls())
1853     {
1854       if (url.m_type != CScraperUrl::UrlType::General)
1855         continue;
1856       std::string aspect = url.m_aspect;
1857       if (aspect.empty())
1858         // Backward compatibility with Kodi 11 Eden NFO files
1859         aspect = mediaType == MediaTypeEpisode ? "thumb" : "poster";
1860
1861       if ((addAll || CVideoThumbLoader::IsArtTypeInWhitelist(aspect, artTypes, exactName)) &&
1862         art.find(aspect) == art.end())
1863       {
1864         std::string image = GetImage(url, pItem->GetPath());
1865         if (!image.empty())
1866           art.insert(std::make_pair(aspect, image));
1867       }
1868     }
1869
1870     if (art.find("thumb") == art.end() &&
1871         CServiceBroker::GetSettingsComponent()->GetSettings()->GetBool(
1872             CSettings::SETTING_MYVIDEOS_EXTRACTTHUMB) &&
1873         CDVDFileInfo::CanExtract(*pItem))
1874     {
1875       art["thumb"] = CVideoThumbLoader::GetEmbeddedThumbURL(*pItem);
1876     }
1877
1878     for (const auto& artType : artTypes)
1879     {
1880       if (art.find(artType) != art.end())
1881         CServiceBroker::GetTextureCache()->BackgroundCacheImage(art[artType]);
1882     }
1883
1884     pItem->SetArt(art);
1885
1886     // parent folder to apply the thumb to and to search for local actor thumbs
1887     std::string parentDir = URIUtils::GetBasePath(pItem->GetPath());
1888     if (CServiceBroker::GetSettingsComponent()->GetSettings()->GetBool(CSettings::SETTING_VIDEOLIBRARY_ACTORTHUMBS))
1889       FetchActorThumbs(movieDetails.m_cast, actorArtPath.empty() ? parentDir : actorArtPath);
1890     if (bApplyToDir)
1891       ApplyThumbToFolder(parentDir, art["thumb"]);
1892   }
1893
1894   std::string CVideoInfoScanner::GetImage(const CScraperUrl::SUrlEntry &image, const std::string& itemPath)
1895   {
1896     std::string thumb = CScraperUrl::GetThumbUrl(image);
1897     if (!thumb.empty() && thumb.find('/') == std::string::npos &&
1898         thumb.find('\\') == std::string::npos)
1899     {
1900       std::string strPath = URIUtils::GetDirectory(itemPath);
1901       thumb = URIUtils::AddFileToFolder(strPath, thumb);
1902     }
1903     return thumb;
1904   }
1905
1906   CInfoScanner::INFO_RET
1907   CVideoInfoScanner::OnProcessSeriesFolder(EPISODELIST& files,
1908                                            const ADDON::ScraperPtr &scraper,
1909                                            bool useLocal,
1910                                            const CVideoInfoTag& showInfo,
1911                                            CGUIDialogProgress* pDlgProgress /* = NULL */)
1912   {
1913     if (pDlgProgress)
1914     {
1915       pDlgProgress->SetLine(1, CVariant{20361}); // Loading episode details
1916       pDlgProgress->SetPercentage(0);
1917       pDlgProgress->ShowProgressBar(true);
1918       pDlgProgress->Progress();
1919     }
1920
1921     EPISODELIST episodes;
1922     bool hasEpisodeGuide = false;
1923
1924     int iMax = files.size();
1925     int iCurr = 1;
1926     for (EPISODELIST::iterator file = files.begin(); file != files.end(); ++file)
1927     {
1928       if (pDlgProgress)
1929       {
1930         pDlgProgress->SetLine(1, CVariant{20361}); // Loading episode details
1931         pDlgProgress->SetLine(2, StringUtils::Format("{} {}", g_localizeStrings.Get(20373),
1932                                                      file->iSeason)); // Season x
1933         pDlgProgress->SetLine(3, StringUtils::Format("{} {}", g_localizeStrings.Get(20359),
1934                                                      file->iEpisode)); // Episode y
1935         pDlgProgress->SetPercentage((int)((float)(iCurr++)/iMax*100));
1936         pDlgProgress->Progress();
1937       }
1938       if (m_handle)
1939         m_handle->SetPercentage(100.f*iCurr++/iMax);
1940
1941       if ((pDlgProgress && pDlgProgress->IsCanceled()) || m_bStop)
1942         return INFO_CANCELLED;
1943
1944       if (m_database.GetEpisodeId(file->strPath, file->iEpisode, file->iSeason) > -1)
1945       {
1946         if (m_handle)
1947           m_handle->SetText(g_localizeStrings.Get(20415));
1948         continue;
1949       }
1950
1951       CFileItem item;
1952       if (file->item)
1953         item = *file->item;
1954       else
1955       {
1956         item.SetPath(file->strPath);
1957         item.GetVideoInfoTag()->m_iEpisode = file->iEpisode;
1958       }
1959
1960       // handle .nfo files
1961       CInfoScanner::INFO_TYPE result=CInfoScanner::NO_NFO;
1962       CScraperUrl scrUrl;
1963       const ScraperPtr& info(scraper);
1964       std::unique_ptr<IVideoInfoTagLoader> loader;
1965       if (useLocal)
1966       {
1967         loader.reset(CVideoInfoTagLoaderFactory::CreateLoader(item, info, false));
1968         if (loader)
1969         {
1970           // no reset here on purpose
1971           result = loader->Load(*item.GetVideoInfoTag(), false);
1972         }
1973       }
1974       if (result == CInfoScanner::FULL_NFO)
1975       {
1976         // override with episode and season number from file if available
1977         if (file->iEpisode > -1)
1978         {
1979           item.GetVideoInfoTag()->m_iEpisode = file->iEpisode;
1980           item.GetVideoInfoTag()->m_iSeason = file->iSeason;
1981         }
1982         if (AddVideo(&item, CONTENT_TVSHOWS, file->isFolder, true, &showInfo) < 0)
1983           return INFO_ERROR;
1984         continue;
1985       }
1986
1987       if (!hasEpisodeGuide)
1988       {
1989         // fetch episode guide
1990         if (!showInfo.m_strEpisodeGuide.empty() && scraper->ID() != "metadata.local")
1991         {
1992           CScraperUrl url;
1993           url.ParseAndAppendUrlsFromEpisodeGuide(showInfo.m_strEpisodeGuide);
1994
1995           if (pDlgProgress)
1996           {
1997             pDlgProgress->SetLine(1, CVariant{20354}); // Fetching episode guide
1998             pDlgProgress->Progress();
1999           }
2000
2001           CVideoInfoDownloader imdb(scraper);
2002           if (!imdb.GetEpisodeList(url, episodes))
2003             return INFO_NOT_FOUND;
2004
2005           hasEpisodeGuide = true;
2006         }
2007       }
2008
2009       if (episodes.empty())
2010       {
2011         CLog::Log(LOGERROR,
2012                   "VideoInfoScanner: Asked to lookup episode {}"
2013                   " online, but we have either no episode guide or"
2014                   " we are using the local scraper. Check your tvshow.nfo and make"
2015                   " sure the <episodeguide> tag is in place and/or use an online"
2016                   " scraper.",
2017                   CURL::GetRedacted(file->strPath));
2018         continue;
2019       }
2020
2021       EPISODE key(file->iSeason, file->iEpisode, file->iSubepisode);
2022       EPISODE backupkey(file->iSeason, file->iEpisode, 0);
2023       bool bFound = false;
2024       EPISODELIST::iterator guide = episodes.begin();
2025       EPISODELIST matches;
2026
2027       for (; guide != episodes.end(); ++guide )
2028       {
2029         if ((file->iEpisode!=-1) && (file->iSeason!=-1))
2030         {
2031           if (key==*guide)
2032           {
2033             bFound = true;
2034             break;
2035           }
2036           else if ((file->iSubepisode!=0) && (backupkey==*guide))
2037           {
2038             matches.push_back(*guide);
2039             continue;
2040           }
2041         }
2042         if (file->cDate.IsValid() && guide->cDate.IsValid() && file->cDate==guide->cDate)
2043         {
2044           matches.push_back(*guide);
2045           continue;
2046         }
2047         if (!guide->cScraperUrl.GetTitle().empty() &&
2048             StringUtils::EqualsNoCase(guide->cScraperUrl.GetTitle(), file->strTitle))
2049         {
2050           bFound = true;
2051           break;
2052         }
2053         if (!guide->strTitle.empty() && StringUtils::EqualsNoCase(guide->strTitle, file->strTitle))
2054         {
2055           bFound = true;
2056           break;
2057         }
2058       }
2059
2060       if (!bFound)
2061       {
2062         /*
2063          * If there is only one match or there are matches but no title to compare with to help
2064          * identify the best match, then pick the first match as the best possible candidate.
2065          *
2066          * Otherwise, use the title to further refine the best match.
2067          */
2068         if (matches.size() == 1 || (file->strTitle.empty() && matches.size() > 1))
2069         {
2070           guide = matches.begin();
2071           bFound = true;
2072         }
2073         else if (!file->strTitle.empty())
2074         {
2075           CLog::Log(LOGDEBUG, "VideoInfoScanner: analyzing parsed title '{}'", file->strTitle);
2076           double minscore = 0; // Default minimum score is 0 to find whatever is the best match.
2077
2078           EPISODELIST *candidates;
2079           if (matches.empty()) // No matches found using earlier criteria. Use fuzzy match on titles across all episodes.
2080           {
2081             minscore = 0.8; // 80% should ensure a good match.
2082             candidates = &episodes;
2083           }
2084           else // Multiple matches found. Use fuzzy match on the title with already matched episodes to pick the best.
2085             candidates = &matches;
2086
2087           std::vector<std::string> titles;
2088           for (guide = candidates->begin(); guide != candidates->end(); ++guide)
2089           {
2090             auto title = guide->cScraperUrl.GetTitle();
2091             if (title.empty())
2092             {
2093               title = guide->strTitle;
2094             }
2095             StringUtils::ToLower(title);
2096             guide->cScraperUrl.SetTitle(title);
2097             titles.push_back(title);
2098           }
2099
2100           double matchscore;
2101           std::string loweredTitle(file->strTitle);
2102           StringUtils::ToLower(loweredTitle);
2103           int index = StringUtils::FindBestMatch(loweredTitle, titles, matchscore);
2104           if (index >= 0 && matchscore >= minscore)
2105           {
2106             guide = candidates->begin() + index;
2107             bFound = true;
2108             CLog::Log(LOGDEBUG,
2109                       "{} fuzzy title match for show: '{}', title: '{}', match: '{}', score: {:f} "
2110                       ">= {:f}",
2111                       __FUNCTION__, showInfo.m_strTitle, file->strTitle, titles[index], matchscore,
2112                       minscore);
2113           }
2114         }
2115       }
2116
2117       if (bFound)
2118       {
2119         CVideoInfoDownloader imdb(scraper);
2120         CFileItem item;
2121         item.SetPath(file->strPath);
2122         if (!imdb.GetEpisodeDetails(guide->cScraperUrl, *item.GetVideoInfoTag(), pDlgProgress))
2123           return INFO_NOT_FOUND; //! @todo should we just skip to the next episode?
2124
2125         // Only set season/epnum from filename when it is not already set by a scraper
2126         if (item.GetVideoInfoTag()->m_iSeason == -1)
2127           item.GetVideoInfoTag()->m_iSeason = guide->iSeason;
2128         if (item.GetVideoInfoTag()->m_iEpisode == -1)
2129           item.GetVideoInfoTag()->m_iEpisode = guide->iEpisode;
2130
2131         if (AddVideo(&item, CONTENT_TVSHOWS, file->isFolder, useLocal, &showInfo) < 0)
2132           return INFO_ERROR;
2133       }
2134       else
2135       {
2136         CLog::Log(
2137             LOGDEBUG,
2138             "{} - no match for show: '{}', season: {}, episode: {}.{}, airdate: '{}', title: '{}'",
2139             __FUNCTION__, showInfo.m_strTitle, file->iSeason, file->iEpisode, file->iSubepisode,
2140             file->cDate.GetAsLocalizedDate(), file->strTitle);
2141       }
2142     }
2143     return INFO_ADDED;
2144   }
2145
2146   bool CVideoInfoScanner::GetDetails(CFileItem* pItem,
2147                                      const std::unordered_map<std::string, std::string>& uniqueIDs,
2148                                      CScraperUrl& url,
2149                                      const ScraperPtr& scraper,
2150                                      IVideoInfoTagLoader* loader,
2151                                      CGUIDialogProgress* pDialog /* = NULL */)
2152   {
2153     CVideoInfoTag movieDetails;
2154
2155     if (m_handle && !url.GetTitle().empty())
2156       m_handle->SetText(url.GetTitle());
2157
2158     CVideoInfoDownloader imdb(scraper);
2159     bool ret = imdb.GetDetails(uniqueIDs, url, movieDetails, pDialog);
2160
2161     if (ret)
2162     {
2163       if (loader)
2164         loader->Load(movieDetails, true);
2165
2166       if (m_handle && url.GetTitle().empty())
2167         m_handle->SetText(movieDetails.m_strTitle);
2168
2169       if (pDialog)
2170       {
2171         if (!pDialog->HasText())
2172           pDialog->SetLine(0, CVariant{movieDetails.m_strTitle});
2173         pDialog->Progress();
2174       }
2175
2176       *pItem->GetVideoInfoTag() = movieDetails;
2177       return true;
2178     }
2179     return false; // no info found, or cancelled
2180   }
2181
2182   void CVideoInfoScanner::ApplyThumbToFolder(const std::string &folder, const std::string &imdbThumb)
2183   {
2184     // copy icon to folder also;
2185     if (!imdbThumb.empty())
2186     {
2187       CFileItem folderItem(folder, true);
2188       CThumbLoader loader;
2189       loader.SetCachedImage(folderItem, "thumb", imdbThumb);
2190     }
2191   }
2192
2193   int CVideoInfoScanner::GetPathHash(const CFileItemList &items, std::string &hash)
2194   {
2195     // Create a hash based on the filenames, filesize and filedate.  Also count the number of files
2196     if (0 == items.Size()) return 0;
2197     CDigest digest{CDigest::Type::MD5};
2198     int count = 0;
2199     for (int i = 0; i < items.Size(); ++i)
2200     {
2201       const CFileItemPtr pItem = items[i];
2202       digest.Update(pItem->GetPath());
2203       if (pItem->IsPlugin())
2204       {
2205         // allow plugin to calculate hash itself using strings rather than binary data for size and date
2206         // according to ListItem.setInfo() documentation date format should be "d.m.Y"
2207         if (pItem->m_dwSize)
2208           digest.Update(std::to_string(pItem->m_dwSize));
2209         if (pItem->m_dateTime.IsValid())
2210           digest.Update(StringUtils::Format("{:02}.{:02}.{:04}", pItem->m_dateTime.GetDay(),
2211                                             pItem->m_dateTime.GetMonth(),
2212                                             pItem->m_dateTime.GetYear()));
2213       }
2214       else
2215       {
2216         digest.Update(&pItem->m_dwSize, sizeof(pItem->m_dwSize));
2217         KODI::TIME::FileTime time = pItem->m_dateTime;
2218         digest.Update(&time, sizeof(KODI::TIME::FileTime));
2219       }
2220       if (IsVideo(*pItem) && !PLAYLIST::IsPlayList(*pItem) && !pItem->IsNFO())
2221         count++;
2222     }
2223     hash = digest.Finalize();
2224     return count;
2225   }
2226
2227   bool CVideoInfoScanner::CanFastHash(const CFileItemList &items, const std::vector<std::string> &excludes) const
2228   {
2229     if (!CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_bVideoLibraryUseFastHash || items.IsPlugin())
2230       return false;
2231
2232     for (int i = 0; i < items.Size(); ++i)
2233     {
2234       if (items[i]->m_bIsFolder && !CUtil::ExcludeFileOrFolder(items[i]->GetPath(), excludes))
2235         return false;
2236     }
2237     return true;
2238   }
2239
2240   std::string CVideoInfoScanner::GetFastHash(const std::string &directory,
2241       const std::vector<std::string> &excludes) const
2242   {
2243     CDigest digest{CDigest::Type::MD5};
2244
2245     if (excludes.size())
2246       digest.Update(StringUtils::Join(excludes, "|"));
2247
2248     struct __stat64 buffer;
2249     if (XFILE::CFile::Stat(directory, &buffer) == 0)
2250     {
2251       int64_t time = buffer.st_mtime;
2252       if (!time)
2253         time = buffer.st_ctime;
2254       if (time)
2255       {
2256         digest.Update((unsigned char *)&time, sizeof(time));
2257         return digest.Finalize();
2258       }
2259     }
2260     return "";
2261   }
2262
2263   std::string CVideoInfoScanner::GetRecursiveFastHash(const std::string &directory,
2264       const std::vector<std::string> &excludes) const
2265   {
2266     CFileItemList items;
2267     items.Add(std::make_shared<CFileItem>(directory, true));
2268     CUtil::GetRecursiveDirsListing(directory, items, DIR_FLAG_NO_FILE_DIRS | DIR_FLAG_NO_FILE_INFO);
2269
2270     CDigest digest{CDigest::Type::MD5};
2271
2272     if (excludes.size())
2273       digest.Update(StringUtils::Join(excludes, "|"));
2274
2275     int64_t time = 0;
2276     for (int i=0; i < items.Size(); ++i)
2277     {
2278       int64_t stat_time = 0;
2279       struct __stat64 buffer;
2280       if (XFILE::CFile::Stat(items[i]->GetPath(), &buffer) == 0)
2281       {
2282         //! @todo some filesystems may return the mtime/ctime inline, in which case this is
2283         //! unnecessarily expensive. Consider supporting Stat() in our directory cache?
2284         stat_time = buffer.st_mtime ? buffer.st_mtime : buffer.st_ctime;
2285         time += stat_time;
2286       }
2287
2288       if (!stat_time)
2289         return "";
2290     }
2291
2292     if (time)
2293     {
2294       digest.Update((unsigned char *)&time, sizeof(time));
2295       return digest.Finalize();
2296     }
2297     return "";
2298   }
2299
2300   void CVideoInfoScanner::GetSeasonThumbs(const CVideoInfoTag &show,
2301       std::map<int, std::map<std::string, std::string>> &seasonArt, const std::vector<std::string> &artTypes, bool useLocal)
2302   {
2303     int artLevel = CServiceBroker::GetSettingsComponent()->GetSettings()->
2304       GetInt(CSettings::SETTING_VIDEOLIBRARY_ARTWORK_LEVEL);
2305     bool addAll = artLevel == CSettings::VIDEOLIBRARY_ARTWORK_LEVEL_ALL;
2306     bool exactName = artLevel == CSettings::VIDEOLIBRARY_ARTWORK_LEVEL_BASIC;
2307     if (useLocal)
2308     {
2309       // find the maximum number of seasons we have local thumbs for
2310       int maxSeasons = 0;
2311       CFileItemList items;
2312       std::string extensions = CServiceBroker::GetFileExtensionProvider().GetPictureExtensions();
2313       if (!show.m_strPath.empty())
2314       {
2315         CDirectory::GetDirectory(show.m_strPath, items, extensions,
2316                                  DIR_FLAG_NO_FILE_DIRS | DIR_FLAG_READ_CACHE |
2317                                      DIR_FLAG_NO_FILE_INFO);
2318       }
2319       extensions.erase(std::remove(extensions.begin(), extensions.end(), '.'), extensions.end());
2320       CRegExp reg;
2321       if (items.Size() && reg.RegComp("season([0-9]+)(-[a-z0-9]+)?\\.(" + extensions + ")"))
2322       {
2323         for (const auto& item : items)
2324         {
2325           std::string name = URIUtils::GetFileName(item->GetPath());
2326           if (reg.RegFind(name) > -1)
2327           {
2328             int season = atoi(reg.GetMatch(1).c_str());
2329             if (season > maxSeasons)
2330               maxSeasons = season;
2331           }
2332         }
2333       }
2334       for (int season = -1; season <= maxSeasons; season++)
2335       {
2336         // skip if we already have some art
2337         std::map<int, std::map<std::string, std::string>>::const_iterator it = seasonArt.find(season);
2338         if (it != seasonArt.end() && !it->second.empty())
2339           continue;
2340
2341         std::map<std::string, std::string> art;
2342         std::string basePath;
2343         if (season == -1)
2344           basePath = "season-all";
2345         else if (season == 0)
2346           basePath = "season-specials";
2347         else
2348           basePath = StringUtils::Format("season{:02}", season);
2349
2350         AddLocalItemArtwork(art, artTypes,
2351           URIUtils::AddFileToFolder(show.m_strPath, basePath),
2352           addAll, exactName);
2353
2354         seasonArt[season] = art;
2355       }
2356     }
2357     // add online art
2358     for (const auto& url : show.m_strPictureURL.GetUrls())
2359     {
2360       if (url.m_type != CScraperUrl::UrlType::Season)
2361         continue;
2362       std::string aspect = url.m_aspect;
2363       if (aspect.empty())
2364         aspect = "thumb";
2365       std::map<std::string, std::string>& art = seasonArt[url.m_season];
2366       if ((addAll || CVideoThumbLoader::IsArtTypeInWhitelist(aspect, artTypes, exactName)) &&
2367         art.find(aspect) == art.end())
2368       {
2369         std::string image = CScraperUrl::GetThumbUrl(url);
2370         if (!image.empty())
2371           art.insert(std::make_pair(aspect, image));
2372       }
2373     }
2374   }
2375
2376   void CVideoInfoScanner::FetchActorThumbs(std::vector<SActorInfo>& actors, const std::string& strPath)
2377   {
2378     CFileItemList items;
2379     // don't try to fetch anything local with plugin source
2380     if (!URIUtils::IsPlugin(strPath))
2381     {
2382       std::string actorsDir = URIUtils::AddFileToFolder(strPath, ".actors");
2383       if (CDirectory::Exists(actorsDir))
2384         CDirectory::GetDirectory(actorsDir, items, ".png|.jpg|.tbn", DIR_FLAG_NO_FILE_DIRS |
2385                                  DIR_FLAG_NO_FILE_INFO);
2386     }
2387     for (std::vector<SActorInfo>::iterator i = actors.begin(); i != actors.end(); ++i)
2388     {
2389       if (i->thumb.empty())
2390       {
2391         std::string thumbFile = i->strName;
2392         StringUtils::Replace(thumbFile, ' ', '_');
2393         for (int j = 0; j < items.Size(); j++)
2394         {
2395           std::string compare = URIUtils::GetFileName(items[j]->GetPath());
2396           URIUtils::RemoveExtension(compare);
2397           if (!items[j]->m_bIsFolder && compare == thumbFile)
2398           {
2399             i->thumb = items[j]->GetPath();
2400             break;
2401           }
2402         }
2403         if (i->thumb.empty() && !i->thumbUrl.GetFirstUrlByType().m_url.empty())
2404           i->thumb = CScraperUrl::GetThumbUrl(i->thumbUrl.GetFirstUrlByType());
2405         if (!i->thumb.empty())
2406           CServiceBroker::GetTextureCache()->BackgroundCacheImage(i->thumb);
2407       }
2408     }
2409   }
2410
2411   bool CVideoInfoScanner::DownloadFailed(CGUIDialogProgress* pDialog)
2412   {
2413     if (CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_bVideoScannerIgnoreErrors)
2414       return true;
2415
2416     if (pDialog)
2417     {
2418       HELPERS::ShowOKDialogText(CVariant{20448}, CVariant{20449});
2419       return false;
2420     }
2421     return HELPERS::ShowYesNoDialogText(CVariant{20448}, CVariant{20450}) ==
2422            DialogResponse::CHOICE_YES;
2423   }
2424
2425   bool CVideoInfoScanner::ProgressCancelled(CGUIDialogProgress* progress, int heading, const std::string &line1)
2426   {
2427     if (progress)
2428     {
2429       progress->SetHeading(CVariant{heading});
2430       progress->SetLine(0, CVariant{line1});
2431       progress->Progress();
2432       return progress->IsCanceled();
2433     }
2434     return m_bStop;
2435   }
2436
2437   int CVideoInfoScanner::FindVideo(const std::string &title, int year, const ScraperPtr &scraper, CScraperUrl &url, CGUIDialogProgress *progress)
2438   {
2439     MOVIELIST movielist;
2440     CVideoInfoDownloader imdb(scraper);
2441     int returncode = imdb.FindMovie(title, year, movielist, progress);
2442     if (returncode < 0 || (returncode == 0 && (m_bStop || !DownloadFailed(progress))))
2443     { // scraper reported an error, or we had an error and user wants to cancel the scan
2444       m_bStop = true;
2445       return -1; // cancelled
2446     }
2447     if (returncode > 0 && movielist.size())
2448     {
2449       url = movielist[0];
2450       return 1;  // found a movie
2451     }
2452     return 0;    // didn't find anything
2453   }
2454
2455   bool CVideoInfoScanner::AddVideoExtras(CFileItemList& items,
2456                                          const CONTENT_TYPE& content,
2457                                          const std::string& path)
2458   {
2459     int dbId = -1;
2460
2461     // get the library item which was added previously with the specified conent type
2462     for (const auto& item : items)
2463     {
2464       if (content == CONTENT_MOVIES)
2465       {
2466         dbId = m_database.GetMovieId(item->GetPath());
2467         if (dbId != -1)
2468         {
2469           break;
2470         }
2471       }
2472     }
2473
2474     if (dbId == -1)
2475     {
2476       CLog::Log(LOGERROR, "VideoInfoScanner: Failed to find the library item for video extras {}",
2477                 CURL::GetRedacted(path));
2478       return false;
2479     }
2480
2481     // Add video extras to library
2482     CDirectory::EnumerateDirectory(
2483         path,
2484         [this, content, dbId, path](const std::shared_ptr<CFileItem>& item)
2485         {
2486           if (CServiceBroker::GetSettingsComponent()->GetSettings()->GetBool(
2487                   CSettings::SETTING_MYVIDEOS_EXTRACTFLAGS))
2488           {
2489             CDVDFileInfo::GetFileStreamDetails(item.get());
2490             CLog::Log(LOGDEBUG, "VideoInfoScanner: Extracted filestream details from video file {}",
2491                       CURL::GetRedacted(item->GetPath()));
2492           }
2493
2494           const std::string typeVideoVersion =
2495               CGUIDialogVideoManagerExtras::GenerateVideoExtra(path, item->GetPath());
2496
2497           const int idVideoVersion = m_database.AddVideoVersionType(
2498               typeVideoVersion, VideoAssetTypeOwner::AUTO, VideoAssetType::EXTRA);
2499
2500           m_database.AddVideoAsset(ContentToVideoDbType(content), dbId, idVideoVersion,
2501                                    VideoAssetType::EXTRA, *item.get());
2502
2503           CLog::Log(LOGDEBUG, "VideoInfoScanner: Added video extras {}",
2504                     CURL::GetRedacted(item->GetPath()));
2505         },
2506         [](auto) { return true; }, true,
2507         CServiceBroker::GetFileExtensionProvider().GetVideoExtensions(), DIR_FLAG_DEFAULTS);
2508
2509     return true;
2510   }
2511
2512   bool CVideoInfoScanner::ProcessVideoVersion(VideoDbContentType itemType, int dbId)
2513   {
2514     return CGUIDialogVideoManagerVersions::ProcessVideoVersion(itemType, dbId);
2515   }
2516 }