xbmc/video/VideoInfoScanner.cpp

   1 /*
   2  *  Copyright (C) 2005-2018 Team Kodi
   3  *  This file is part of Kodi - https://kodi.tv
   4  *
   5  *  SPDX-License-Identifier: GPL-2.0-or-later
   6  *  See LICENSES/README.md for more information.
   7  */
   8
   9 #include "VideoInfoScanner.h"
  10
  11 #include "FileItem.h"
  12 #include "GUIInfoManager.h"
  13 #include "GUIUserMessages.h"
  14 #include "NfoFile.h"
  15 #include "ServiceBroker.h"
  16 #include "TextureCache.h"
  17 #include "URL.h"
  18 #include "Util.h"
  19 #include "VideoInfoDownloader.h"
  20 #include "cores/VideoPlayer/DVDFileInfo.h"
  21 #include "dialogs/GUIDialogExtendedProgressBar.h"
  22 #include "dialogs/GUIDialogProgress.h"
  23 #include "events/EventLog.h"
  24 #include "events/MediaLibraryEvent.h"
  25 #include "filesystem/Directory.h"
  26 #include "filesystem/DirectoryCache.h"
  27 #include "filesystem/File.h"
  28 #include "filesystem/MultiPathDirectory.h"
  29 #include "filesystem/PluginDirectory.h"
  30 #include "guilib/GUIComponent.h"
  31 #include "guilib/GUIWindowManager.h"
  32 #include "guilib/LocalizeStrings.h"
  33 #include "interfaces/AnnouncementManager.h"
  34 #include "messaging/helpers/DialogHelper.h"
  35 #include "messaging/helpers/DialogOKHelper.h"
  36 #include "settings/AdvancedSettings.h"
  37 #include "settings/Settings.h"
  38 #include "settings/SettingsComponent.h"
  39 #include "tags/VideoInfoTagLoaderFactory.h"
  40 #include "utils/Digest.h"
  41 #include "utils/FileExtensionProvider.h"
  42 #include "utils/RegExp.h"
  43 #include "utils/StringUtils.h"
  44 #include "utils/URIUtils.h"
  45 #include "utils/Variant.h"
  46 #include "utils/log.h"
  47 #include "video/VideoThumbLoader.h"
  48
  49 #include <algorithm>
  50 #include <utility>
  51
  52 using namespace XFILE;
  53 using namespace ADDON;
  54 using namespace KODI::MESSAGING;
  55
  56 using KODI::MESSAGING::HELPERS::DialogResponse;
  57 using KODI::UTILITY::CDigest;
  58
  59 namespace VIDEO
  60 {
  61
  62   CVideoInfoScanner::CVideoInfoScanner()
  63   {
  64     m_bStop = false;
  65     m_scanAll = false;
  66   }
  67
  68   CVideoInfoScanner::~CVideoInfoScanner()
  69   = default;
  70
  71   void CVideoInfoScanner::Process()
  72   {
  73     m_bStop = false;
  74
  75     try
  76     {
  77       if (m_showDialog && !CServiceBroker::GetSettingsComponent()->GetSettings()->GetBool(CSettings::SETTING_VIDEOLIBRARY_BACKGROUNDUPDATE))
  78       {
  79         CGUIDialogExtendedProgressBar* dialog =
  80           CServiceBroker::GetGUI()->GetWindowManager().GetWindow<CGUIDialogExtendedProgressBar>(WINDOW_DIALOG_EXT_PROGRESS);
  81         if (dialog)
  82            m_handle = dialog->GetHandle(g_localizeStrings.Get(314));
  83       }
  84
  85       // check if we only need to perform a cleaning
  86       if (m_bClean && m_pathsToScan.empty())
  87       {
  88         std::set<int> paths;
  89         m_database.CleanDatabase(m_handle, paths, false);
  90
  91         if (m_handle)
  92           m_handle->MarkFinished();
  93         m_handle = NULL;
  94
  95         m_bRunning = false;
  96
  97         return;
  98       }
  99
 100       auto start = std::chrono::steady_clock::now();
 101
 102       m_database.Open();
 103
 104       m_bCanInterrupt = true;
 105
 106       CLog::Log(LOGINFO, "VideoInfoScanner: Starting scan ..");
 107       CServiceBroker::GetAnnouncementManager()->Announce(ANNOUNCEMENT::VideoLibrary,
 108                                                          "OnScanStarted");
 109
 110       // Database operations should not be canceled
 111       // using Interrupt() while scanning as it could
 112       // result in unexpected behaviour.
 113       m_bCanInterrupt = false;
 114
 115       bool bCancelled = false;
 116       while (!bCancelled && !m_pathsToScan.empty())
 117       {
 118         /*
 119          * A copy of the directory path is used because the path supplied is
 120          * immediately removed from the m_pathsToScan set in DoScan(). If the
 121          * reference points to the entry in the set a null reference error
 122          * occurs.
 123          */
 124         std::string directory = *m_pathsToScan.begin();
 125         if (m_bStop)
 126         {
 127           bCancelled = true;
 128         }
 129         else if (!CDirectory::Exists(directory))
 130         {
 131           /*
 132            * Note that this will skip clean (if m_bClean is enabled) if the directory really
 133            * doesn't exist rather than a NAS being switched off.  A manual clean from settings
 134            * will still pick up and remove it though.
 135            */
 136           CLog::Log(LOGWARNING, "{} directory '{}' does not exist - skipping scan{}.", __FUNCTION__,
 137                     CURL::GetRedacted(directory), m_bClean ? " and clean" : "");
 138           m_pathsToScan.erase(m_pathsToScan.begin());
 139         }
 140         else if (!DoScan(directory))
 141           bCancelled = true;
 142       }
 143
 144       if (!bCancelled)
 145       {
 146         if (m_bClean)
 147           m_database.CleanDatabase(m_handle, m_pathsToClean, false);
 148         else
 149         {
 150           if (m_handle)
 151             m_handle->SetTitle(g_localizeStrings.Get(331));
 152           m_database.Compress(false);
 153         }
 154       }
 155
 156       CServiceBroker::GetGUI()->GetInfoManager().GetInfoProviders().GetLibraryInfoProvider().ResetLibraryBools();
 157       m_database.Close();
 158
 159       auto end = std::chrono::steady_clock::now();
 160       auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
 161
 162       CLog::Log(LOGINFO, "VideoInfoScanner: Finished scan. Scanning for video info took {} ms",
 163                 duration.count());
 164     }
 165     catch (...)
 166     {
 167       CLog::Log(LOGERROR, "VideoInfoScanner: Exception while scanning.");
 168     }
 169
 170     m_bRunning = false;
 171     CServiceBroker::GetAnnouncementManager()->Announce(ANNOUNCEMENT::VideoLibrary,
 172                                                        "OnScanFinished");
 173
 174     if (m_handle)
 175       m_handle->MarkFinished();
 176     m_handle = NULL;
 177   }
 178
 179   void CVideoInfoScanner::Start(const std::string& strDirectory, bool scanAll)
 180   {
 181     m_strStartDir = strDirectory;
 182     m_scanAll = scanAll;
 183     m_pathsToScan.clear();
 184     m_pathsToClean.clear();
 185
 186     m_database.Open();
 187     if (strDirectory.empty())
 188     { // scan all paths in the database.  We do this by scanning all paths in the db, and crossing them off the list as
 189       // we go.
 190       m_database.GetPaths(m_pathsToScan);
 191     }
 192     else
 193     { // scan all the paths of this subtree that is in the database
 194       std::vector<std::string> rootDirs;
 195       if (URIUtils::IsMultiPath(strDirectory))
 196         CMultiPathDirectory::GetPaths(strDirectory, rootDirs);
 197       else
 198         rootDirs.push_back(strDirectory);
 199
 200       for (std::vector<std::string>::const_iterator it = rootDirs.begin(); it < rootDirs.end(); ++it)
 201       {
 202         m_pathsToScan.insert(*it);
 203         std::vector<std::pair<int, std::string>> subpaths;
 204         m_database.GetSubPaths(*it, subpaths);
 205         for (std::vector<std::pair<int, std::string>>::iterator it = subpaths.begin(); it < subpaths.end(); ++it)
 206           m_pathsToScan.insert(it->second);
 207       }
 208     }
 209     m_database.Close();
 210     m_bClean = CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_bVideoLibraryCleanOnUpdate;
 211
 212     m_bRunning = true;
 213     Process();
 214   }
 215
 216   void CVideoInfoScanner::Stop()
 217   {
 218     if (m_bCanInterrupt)
 219       m_database.Interrupt();
 220
 221     m_bStop = true;
 222   }
 223
 224   static void OnDirectoryScanned(const std::string& strDirectory)
 225   {
 226     CGUIMessage msg(GUI_MSG_DIRECTORY_SCANNED, 0, 0, 0);
 227     msg.SetStringParam(strDirectory);
 228     CServiceBroker::GetGUI()->GetWindowManager().SendThreadMessage(msg);
 229   }
 230
 231   bool CVideoInfoScanner::DoScan(const std::string& strDirectory)
 232   {
 233     if (m_handle)
 234     {
 235       m_handle->SetText(g_localizeStrings.Get(20415));
 236     }
 237
 238     /*
 239      * Remove this path from the list we're processing. This must be done prior to
 240      * the check for file or folder exclusion to prevent an infinite while loop
 241      * in Process().
 242      */
 243     std::set<std::string>::iterator it = m_pathsToScan.find(strDirectory);
 244     if (it != m_pathsToScan.end())
 245       m_pathsToScan.erase(it);
 246
 247     // load subfolder
 248     CFileItemList items;
 249     bool foundDirectly = false;
 250     bool bSkip = false;
 251
 252     SScanSettings settings;
 253     ScraperPtr info = m_database.GetScraperForPath(strDirectory, settings, foundDirectly);
 254     CONTENT_TYPE content = info ? info->Content() : CONTENT_NONE;
 255
 256     // exclude folders that match our exclude regexps
 257     const std::vector<std::string> &regexps = content == CONTENT_TVSHOWS ? CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_tvshowExcludeFromScanRegExps
 258                                                          : CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_moviesExcludeFromScanRegExps;
 259
 260     if (CUtil::ExcludeFileOrFolder(strDirectory, regexps))
 261       return true;
 262
 263     if (HasNoMedia(strDirectory))
 264       return true;
 265
 266     bool ignoreFolder = !m_scanAll && settings.noupdate;
 267     if (content == CONTENT_NONE || ignoreFolder)
 268       return true;
 269
 270     if (URIUtils::IsPlugin(strDirectory) && !CPluginDirectory::IsMediaLibraryScanningAllowed(TranslateContent(content), strDirectory))
 271     {
 272       CLog::Log(
 273           LOGINFO,
 274           "VideoInfoScanner: Plugin '{}' does not support media library scanning for '{}' content",
 275           CURL::GetRedacted(strDirectory), TranslateContent(content));
 276       return true;
 277     }
 278
 279     std::string hash, dbHash;
 280     if (content == CONTENT_MOVIES ||content == CONTENT_MUSICVIDEOS)
 281     {
 282       if (m_handle)
 283       {
 284         int str = content == CONTENT_MOVIES ? 20317:20318;
 285         m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(str), info->Name()));
 286       }
 287
 288       std::string fastHash;
 289       if (CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_bVideoLibraryUseFastHash && !URIUtils::IsPlugin(strDirectory))
 290         fastHash = GetFastHash(strDirectory, regexps);
 291
 292       if (m_database.GetPathHash(strDirectory, dbHash) && !fastHash.empty() && StringUtils::EqualsNoCase(fastHash, dbHash))
 293       { // fast hashes match - no need to process anything
 294         hash = fastHash;
 295       }
 296       else
 297       { // need to fetch the folder
 298         CDirectory::GetDirectory(strDirectory, items, CServiceBroker::GetFileExtensionProvider().GetVideoExtensions(),
 299                                  DIR_FLAG_DEFAULTS);
 300         // do not consider inner folders with .nomedia
 301         items.erase(std::remove_if(items.begin(), items.end(),
 302                                    [this](const CFileItemPtr& item) {
 303                                      return item->m_bIsFolder && HasNoMedia(item->GetPath());
 304                                    }),
 305                     items.end());
 306         items.Stack();
 307
 308         // check whether to re-use previously computed fast hash
 309         if (!CanFastHash(items, regexps) || fastHash.empty())
 310           GetPathHash(items, hash);
 311         else
 312           hash = fastHash;
 313       }
 314
 315       if (StringUtils::EqualsNoCase(hash, dbHash))
 316       { // hash matches - skipping
 317         CLog::Log(LOGDEBUG, "VideoInfoScanner: Skipping dir '{}' due to no change{}",
 318                   CURL::GetRedacted(strDirectory), !fastHash.empty() ? " (fasthash)" : "");
 319         bSkip = true;
 320       }
 321       else if (hash.empty())
 322       { // directory empty or non-existent - add to clean list and skip
 323         CLog::Log(LOGDEBUG,
 324                   "VideoInfoScanner: Skipping dir '{}' as it's empty or doesn't exist - adding to "
 325                   "clean list",
 326                   CURL::GetRedacted(strDirectory));
 327         if (m_bClean)
 328           m_pathsToClean.insert(m_database.GetPathId(strDirectory));
 329         bSkip = true;
 330       }
 331       else if (dbHash.empty())
 332       { // new folder - scan
 333         CLog::Log(LOGDEBUG, "VideoInfoScanner: Scanning dir '{}' as not in the database",
 334                   CURL::GetRedacted(strDirectory));
 335       }
 336       else
 337       { // hash changed - rescan
 338         CLog::Log(LOGDEBUG, "VideoInfoScanner: Rescanning dir '{}' due to change ({} != {})",
 339                   CURL::GetRedacted(strDirectory), dbHash, hash);
 340       }
 341     }
 342     else if (content == CONTENT_TVSHOWS)
 343     {
 344       if (m_handle)
 345         m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(20319), info->Name()));
 346
 347       if (foundDirectly && !settings.parent_name_root)
 348       {
 349         CDirectory::GetDirectory(strDirectory, items, CServiceBroker::GetFileExtensionProvider().GetVideoExtensions(),
 350                                  DIR_FLAG_DEFAULTS);
 351         items.SetPath(strDirectory);
 352         GetPathHash(items, hash);
 353         bSkip = true;
 354         if (!m_database.GetPathHash(strDirectory, dbHash) || !StringUtils::EqualsNoCase(dbHash, hash))
 355           bSkip = false;
 356         else
 357           items.Clear();
 358       }
 359       else
 360       {
 361         CFileItemPtr item(new CFileItem(URIUtils::GetFileName(strDirectory)));
 362         item->SetPath(strDirectory);
 363         item->m_bIsFolder = true;
 364         items.Add(item);
 365         items.SetPath(URIUtils::GetParentPath(item->GetPath()));
 366       }
 367     }
 368
 369     if (!bSkip)
 370     {
 371       if (RetrieveVideoInfo(items, settings.parent_name_root, content))
 372       {
 373         if (!m_bStop && (content == CONTENT_MOVIES || content == CONTENT_MUSICVIDEOS))
 374         {
 375           m_database.SetPathHash(strDirectory, hash);
 376           if (m_bClean)
 377             m_pathsToClean.insert(m_database.GetPathId(strDirectory));
 378           CLog::Log(LOGDEBUG, "VideoInfoScanner: Finished adding information from dir {}",
 379                     CURL::GetRedacted(strDirectory));
 380         }
 381       }
 382       else
 383       {
 384         if (m_bClean)
 385           m_pathsToClean.insert(m_database.GetPathId(strDirectory));
 386         CLog::Log(LOGDEBUG, "VideoInfoScanner: No (new) information was found in dir {}",
 387                   CURL::GetRedacted(strDirectory));
 388       }
 389     }
 390     else if (!StringUtils::EqualsNoCase(hash, dbHash) && (content == CONTENT_MOVIES || content == CONTENT_MUSICVIDEOS))
 391     { // update the hash either way - we may have changed the hash to a fast version
 392       m_database.SetPathHash(strDirectory, hash);
 393     }
 394
 395     if (m_handle)
 396       OnDirectoryScanned(strDirectory);
 397
 398     for (int i = 0; i < items.Size(); ++i)
 399     {
 400       CFileItemPtr pItem = items[i];
 401
 402       if (m_bStop)
 403         break;
 404
 405       // if we have a directory item (non-playlist) we then recurse into that folder
 406       // do not recurse for tv shows - we have already looked recursively for episodes
 407       if (pItem->m_bIsFolder && !pItem->IsParentFolder() && !pItem->IsPlayList() && settings.recurse > 0 && content != CONTENT_TVSHOWS)
 408       {
 409         if (!DoScan(pItem->GetPath()))
 410         {
 411           m_bStop = true;
 412         }
 413       }
 414     }
 415     return !m_bStop;
 416   }
 417
 418   bool CVideoInfoScanner::RetrieveVideoInfo(CFileItemList& items, bool bDirNames, CONTENT_TYPE content, bool useLocal, CScraperUrl* pURL, bool fetchEpisodes, CGUIDialogProgress* pDlgProgress)
 419   {
 420     if (pDlgProgress)
 421     {
 422       if (items.Size() > 1 || (items[0]->m_bIsFolder && fetchEpisodes))
 423       {
 424         pDlgProgress->ShowProgressBar(true);
 425         pDlgProgress->SetPercentage(0);
 426       }
 427       else
 428         pDlgProgress->ShowProgressBar(false);
 429
 430       pDlgProgress->Progress();
 431     }
 432
 433     m_database.Open();
 434
 435     bool FoundSomeInfo = false;
 436     std::vector<int> seenPaths;
 437     for (int i = 0; i < items.Size(); ++i)
 438     {
 439       CFileItemPtr pItem = items[i];
 440
 441       // we do this since we may have a override per dir
 442       ScraperPtr info2 = m_database.GetScraperForPath(pItem->m_bIsFolder ? pItem->GetPath() : items.GetPath());
 443       if (!info2) // skip
 444         continue;
 445
 446       // Discard all .nomedia folders
 447       if (pItem->m_bIsFolder && HasNoMedia(pItem->GetPath()))
 448         continue;
 449
 450       // Discard all exclude files defined by regExExclude
 451       if (CUtil::ExcludeFileOrFolder(pItem->GetPath(), (content == CONTENT_TVSHOWS) ? CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_tvshowExcludeFromScanRegExps
 452                                                                     : CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_moviesExcludeFromScanRegExps))
 453         continue;
 454
 455       if (info2->Content() == CONTENT_MOVIES || info2->Content() == CONTENT_MUSICVIDEOS)
 456       {
 457         if (m_handle)
 458           m_handle->SetPercentage(i*100.f/items.Size());
 459       }
 460
 461       // clear our scraper cache
 462       info2->ClearCache();
 463
 464       INFO_RET ret = INFO_CANCELLED;
 465       if (info2->Content() == CONTENT_TVSHOWS)
 466         ret = RetrieveInfoForTvShow(pItem.get(), bDirNames, info2, useLocal, pURL, fetchEpisodes, pDlgProgress);
 467       else if (info2->Content() == CONTENT_MOVIES)
 468         ret = RetrieveInfoForMovie(pItem.get(), bDirNames, info2, useLocal, pURL, pDlgProgress);
 469       else if (info2->Content() == CONTENT_MUSICVIDEOS)
 470         ret = RetrieveInfoForMusicVideo(pItem.get(), bDirNames, info2, useLocal, pURL, pDlgProgress);
 471       else
 472       {
 473         CLog::Log(LOGERROR, "VideoInfoScanner: Unknown content type {} ({})", info2->Content(),
 474                   CURL::GetRedacted(pItem->GetPath()));
 475         FoundSomeInfo = false;
 476         break;
 477       }
 478       if (ret == INFO_CANCELLED || ret == INFO_ERROR)
 479       {
 480         CLog::Log(LOGWARNING,
 481                   "VideoInfoScanner: Error {} occurred while retrieving"
 482                   "information for {}.",
 483                   ret, CURL::GetRedacted(pItem->GetPath()));
 484         FoundSomeInfo = false;
 485         break;
 486       }
 487       if (ret == INFO_ADDED || ret == INFO_HAVE_ALREADY)
 488         FoundSomeInfo = true;
 489       else if (ret == INFO_NOT_FOUND)
 490       {
 491         CLog::Log(LOGWARNING,
 492                   "No information found for item '{}', it won't be added to the library.",
 493                   CURL::GetRedacted(pItem->GetPath()));
 494
 495         MediaType mediaType = MediaTypeMovie;
 496         if (info2->Content() == CONTENT_TVSHOWS)
 497           mediaType = MediaTypeTvShow;
 498         else if (info2->Content() == CONTENT_MUSICVIDEOS)
 499           mediaType = MediaTypeMusicVideo;
 500
 501         auto eventLog = CServiceBroker::GetEventLog();
 502         if (eventLog)
 503           eventLog->Add(EventPtr(new CMediaLibraryEvent(
 504               mediaType, pItem->GetPath(), 24145,
 505               StringUtils::Format(g_localizeStrings.Get(24147), mediaType,
 506                                   URIUtils::GetFileName(pItem->GetPath())),
 507               pItem->GetArt("thumb"), CURL::GetRedacted(pItem->GetPath()), EventLevel::Warning)));
 508       }
 509
 510       pURL = NULL;
 511
 512       // Keep track of directories we've seen
 513       if (m_bClean && pItem->m_bIsFolder)
 514         seenPaths.push_back(m_database.GetPathId(pItem->GetPath()));
 515     }
 516
 517     if (content == CONTENT_TVSHOWS && ! seenPaths.empty())
 518     {
 519       std::vector<std::pair<int, std::string>> libPaths;
 520       m_database.GetSubPaths(items.GetPath(), libPaths);
 521       for (std::vector<std::pair<int, std::string> >::iterator i = libPaths.begin(); i < libPaths.end(); ++i)
 522       {
 523         if (find(seenPaths.begin(), seenPaths.end(), i->first) == seenPaths.end())
 524           m_pathsToClean.insert(i->first);
 525       }
 526     }
 527     if(pDlgProgress)
 528       pDlgProgress->ShowProgressBar(false);
 529
 530     m_database.Close();
 531     return FoundSomeInfo;
 532   }
 533
 534   CInfoScanner::INFO_RET
 535   CVideoInfoScanner::RetrieveInfoForTvShow(CFileItem *pItem,
 536                                            bool bDirNames,
 537                                            ScraperPtr &info2,
 538                                            bool useLocal,
 539                                            CScraperUrl* pURL,
 540                                            bool fetchEpisodes,
 541                                            CGUIDialogProgress* pDlgProgress)
 542   {
 543     long idTvShow = -1;
 544     std::string strPath = pItem->GetPath();
 545     if (pItem->m_bIsFolder)
 546       idTvShow = m_database.GetTvShowId(strPath);
 547     else if (pItem->IsPlugin() && pItem->HasVideoInfoTag() && pItem->GetVideoInfoTag()->m_iIdShow >= 0)
 548     {
 549       // for plugin source we cannot get idTvShow from episode path with URIUtils::GetDirectory() in all cases
 550       // so use m_iIdShow from video info tag if possible
 551       idTvShow = pItem->GetVideoInfoTag()->m_iIdShow;
 552       CVideoInfoTag showInfo;
 553       if (m_database.GetTvShowInfo(std::string(), showInfo, idTvShow, nullptr, 0))
 554         strPath = showInfo.GetPath();
 555     }
 556     else
 557     {
 558       strPath = URIUtils::GetDirectory(strPath);
 559       idTvShow = m_database.GetTvShowId(strPath);
 560     }
 561     if (idTvShow > -1 && (fetchEpisodes || !pItem->m_bIsFolder))
 562     {
 563       INFO_RET ret = RetrieveInfoForEpisodes(pItem, idTvShow, info2, useLocal, pDlgProgress);
 564       if (ret == INFO_ADDED)
 565         m_database.SetPathHash(strPath, pItem->GetProperty("hash").asString());
 566       return ret;
 567     }
 568
 569     if (ProgressCancelled(pDlgProgress, pItem->m_bIsFolder ? 20353 : 20361, pItem->GetLabel()))
 570       return INFO_CANCELLED;
 571
 572     if (m_handle)
 573       m_handle->SetText(pItem->GetMovieName(bDirNames));
 574
 575     CInfoScanner::INFO_TYPE result=CInfoScanner::NO_NFO;
 576     CScraperUrl scrUrl;
 577     // handle .nfo files
 578     std::unique_ptr<IVideoInfoTagLoader> loader;
 579     if (useLocal)
 580     {
 581       loader.reset(CVideoInfoTagLoaderFactory::CreateLoader(*pItem, info2, bDirNames));
 582       if (loader)
 583       {
 584         pItem->GetVideoInfoTag()->Reset();
 585         result = loader->Load(*pItem->GetVideoInfoTag(), false);
 586       }
 587     }
 588
 589     if (result == CInfoScanner::FULL_NFO)
 590     {
 591
 592       long lResult = AddVideo(pItem, info2->Content(), bDirNames, useLocal);
 593       if (lResult < 0)
 594         return INFO_ERROR;
 595       if (fetchEpisodes)
 596       {
 597         INFO_RET ret = RetrieveInfoForEpisodes(pItem, lResult, info2, useLocal, pDlgProgress);
 598         if (ret == INFO_ADDED)
 599           m_database.SetPathHash(pItem->GetPath(), pItem->GetProperty("hash").asString());
 600         return ret;
 601       }
 602       return INFO_ADDED;
 603     }
 604     if (result == CInfoScanner::URL_NFO || result == CInfoScanner::COMBINED_NFO)
 605     {
 606       scrUrl = loader->ScraperUrl();
 607       pURL = &scrUrl;
 608     }
 609
 610     CScraperUrl url;
 611     int retVal = 0;
 612     std::string movieTitle = pItem->GetMovieName(bDirNames);
 613     int movieYear = -1; // hint that movie title was not found
 614     if (result == CInfoScanner::TITLE_NFO)
 615     {
 616       CVideoInfoTag* tag = pItem->GetVideoInfoTag();
 617       movieTitle = tag->GetTitle();
 618       movieYear = tag->GetYear(); // movieYear is expected to be >= 0
 619     }
 620     if (pURL && pURL->HasUrls())
 621       url = *pURL;
 622     else if ((retVal = FindVideo(movieTitle, movieYear, info2, url, pDlgProgress)) <= 0)
 623       return retVal < 0 ? INFO_CANCELLED : INFO_NOT_FOUND;
 624
 625     CLog::Log(LOGDEBUG, "VideoInfoScanner: Fetching url '{}' using {} scraper (content: '{}')",
 626               url.GetFirstThumbUrl(), info2->Name(), TranslateContent(info2->Content()));
 627
 628     long lResult = -1;
 629     if (GetDetails(pItem, url, info2,
 630                    (result == CInfoScanner::COMBINED_NFO ||
 631                     result == CInfoScanner::OVERRIDE_NFO) ? loader.get() : nullptr,
 632                    pDlgProgress))
 633     {
 634       if ((lResult = AddVideo(pItem, info2->Content(), false, useLocal)) < 0)
 635         return INFO_ERROR;
 636     }
 637     if (fetchEpisodes)
 638     {
 639       INFO_RET ret = RetrieveInfoForEpisodes(pItem, lResult, info2, useLocal, pDlgProgress);
 640       if (ret == INFO_ADDED)
 641         m_database.SetPathHash(pItem->GetPath(), pItem->GetProperty("hash").asString());
 642     }
 643     return INFO_ADDED;
 644   }
 645
 646   CInfoScanner::INFO_RET
 647   CVideoInfoScanner::RetrieveInfoForMovie(CFileItem *pItem,
 648                                           bool bDirNames,
 649                                           ScraperPtr &info2,
 650                                           bool useLocal,
 651                                           CScraperUrl* pURL,
 652                                           CGUIDialogProgress* pDlgProgress)
 653   {
 654     if (pItem->m_bIsFolder || !pItem->IsVideo() || pItem->IsNFO() ||
 655        (pItem->IsPlayList() && !URIUtils::HasExtension(pItem->GetPath(), ".strm")))
 656       return INFO_NOT_NEEDED;
 657
 658     if (ProgressCancelled(pDlgProgress, 198, pItem->GetLabel()))
 659       return INFO_CANCELLED;
 660
 661     if (m_database.HasMovieInfo(pItem->GetPath()))
 662       return INFO_HAVE_ALREADY;
 663
 664     if (m_handle)
 665       m_handle->SetText(pItem->GetMovieName(bDirNames));
 666
 667     CInfoScanner::INFO_TYPE result = CInfoScanner::NO_NFO;
 668     CScraperUrl scrUrl;
 669     // handle .nfo files
 670     std::unique_ptr<IVideoInfoTagLoader> loader;
 671     if (useLocal)
 672     {
 673       loader.reset(CVideoInfoTagLoaderFactory::CreateLoader(*pItem, info2, bDirNames));
 674       if (loader)
 675       {
 676         pItem->GetVideoInfoTag()->Reset();
 677         result = loader->Load(*pItem->GetVideoInfoTag(), false);
 678       }
 679     }
 680     if (result == CInfoScanner::FULL_NFO)
 681     {
 682       if (AddVideo(pItem, info2->Content(), bDirNames, true) < 0)
 683         return INFO_ERROR;
 684       return INFO_ADDED;
 685     }
 686     if (result == CInfoScanner::URL_NFO || result == CInfoScanner::COMBINED_NFO)
 687     {
 688       scrUrl = loader->ScraperUrl();
 689       pURL = &scrUrl;
 690     }
 691
 692     CScraperUrl url;
 693     int retVal = 0;
 694     std::string movieTitle = pItem->GetMovieName(bDirNames);
 695     int movieYear = -1; // hint that movie title was not found
 696     if (result == CInfoScanner::TITLE_NFO)
 697     {
 698       CVideoInfoTag* tag = pItem->GetVideoInfoTag();
 699       movieTitle = tag->GetTitle();
 700       movieYear = tag->GetYear(); // movieYear is expected to be >= 0
 701     }
 702     if (pURL && pURL->HasUrls())
 703       url = *pURL;
 704     else if ((retVal = FindVideo(movieTitle, movieYear, info2, url, pDlgProgress)) <= 0)
 705       return retVal < 0 ? INFO_CANCELLED : INFO_NOT_FOUND;
 706
 707     CLog::Log(LOGDEBUG, "VideoInfoScanner: Fetching url '{}' using {} scraper (content: '{}')",
 708               url.GetFirstThumbUrl(), info2->Name(), TranslateContent(info2->Content()));
 709
 710     if (GetDetails(pItem, url, info2,
 711                    (result == CInfoScanner::COMBINED_NFO ||
 712                     result == CInfoScanner::OVERRIDE_NFO) ? loader.get() : nullptr,
 713                    pDlgProgress))
 714     {
 715       if (AddVideo(pItem, info2->Content(), bDirNames, useLocal) < 0)
 716         return INFO_ERROR;
 717       return INFO_ADDED;
 718     }
 719     //! @todo This is not strictly correct as we could fail to download information here or error, or be cancelled
 720     return INFO_NOT_FOUND;
 721   }
 722
 723   CInfoScanner::INFO_RET
 724   CVideoInfoScanner::RetrieveInfoForMusicVideo(CFileItem *pItem,
 725                                                bool bDirNames,
 726                                                ScraperPtr &info2,
 727                                                bool useLocal,
 728                                                CScraperUrl* pURL,
 729                                                CGUIDialogProgress* pDlgProgress)
 730   {
 731     if (pItem->m_bIsFolder || !pItem->IsVideo() || pItem->IsNFO() ||
 732        (pItem->IsPlayList() && !URIUtils::HasExtension(pItem->GetPath(), ".strm")))
 733       return INFO_NOT_NEEDED;
 734
 735     if (ProgressCancelled(pDlgProgress, 20394, pItem->GetLabel()))
 736       return INFO_CANCELLED;
 737
 738     if (m_database.HasMusicVideoInfo(pItem->GetPath()))
 739       return INFO_HAVE_ALREADY;
 740
 741     if (m_handle)
 742       m_handle->SetText(pItem->GetMovieName(bDirNames));
 743
 744     CInfoScanner::INFO_TYPE result = CInfoScanner::NO_NFO;
 745     CScraperUrl scrUrl;
 746     // handle .nfo files
 747     std::unique_ptr<IVideoInfoTagLoader> loader;
 748     if (useLocal)
 749     {
 750       loader.reset(CVideoInfoTagLoaderFactory::CreateLoader(*pItem, info2, bDirNames));
 751       if (loader)
 752       {
 753         pItem->GetVideoInfoTag()->Reset();
 754         result = loader->Load(*pItem->GetVideoInfoTag(), false);
 755       }
 756     }
 757     if (result == CInfoScanner::FULL_NFO)
 758     {
 759       if (AddVideo(pItem, info2->Content(), bDirNames, true) < 0)
 760         return INFO_ERROR;
 761       return INFO_ADDED;
 762     }
 763     if (result == CInfoScanner::URL_NFO || result == CInfoScanner::COMBINED_NFO)
 764     {
 765       scrUrl = loader->ScraperUrl();
 766       pURL = &scrUrl;
 767     }
 768
 769     CScraperUrl url;
 770     int retVal = 0;
 771     std::string movieTitle = pItem->GetMovieName(bDirNames);
 772     int movieYear = -1; // hint that movie title was not found
 773     if (result == CInfoScanner::TITLE_NFO)
 774     {
 775       CVideoInfoTag* tag = pItem->GetVideoInfoTag();
 776       movieTitle = tag->GetTitle();
 777       movieYear = tag->GetYear(); // movieYear is expected to be >= 0
 778     }
 779     if (pURL && pURL->HasUrls())
 780       url = *pURL;
 781     else if ((retVal = FindVideo(movieTitle, movieYear, info2, url, pDlgProgress)) <= 0)
 782       return retVal < 0 ? INFO_CANCELLED : INFO_NOT_FOUND;
 783
 784     CLog::Log(LOGDEBUG, "VideoInfoScanner: Fetching url '{}' using {} scraper (content: '{}')",
 785               url.GetFirstThumbUrl(), info2->Name(), TranslateContent(info2->Content()));
 786
 787     if (GetDetails(pItem, url, info2,
 788                    (result == CInfoScanner::COMBINED_NFO ||
 789                     result == CInfoScanner::OVERRIDE_NFO) ? loader.get() : nullptr,
 790                    pDlgProgress))
 791     {
 792       if (AddVideo(pItem, info2->Content(), bDirNames, useLocal) < 0)
 793         return INFO_ERROR;
 794       return INFO_ADDED;
 795     }
 796     //! @todo This is not strictly correct as we could fail to download information here or error, or be cancelled
 797     return INFO_NOT_FOUND;
 798   }
 799
 800   CInfoScanner::INFO_RET
 801   CVideoInfoScanner::RetrieveInfoForEpisodes(CFileItem *item,
 802                                              long showID,
 803                                              const ADDON::ScraperPtr &scraper,
 804                                              bool useLocal,
 805                                              CGUIDialogProgress *progress)
 806   {
 807     // enumerate episodes
 808     EPISODELIST files;
 809     if (!EnumerateSeriesFolder(item, files))
 810       return INFO_HAVE_ALREADY;
 811     if (files.empty()) // no update or no files
 812       return INFO_NOT_NEEDED;
 813
 814     if (m_bStop || (progress && progress->IsCanceled()))
 815       return INFO_CANCELLED;
 816
 817     CVideoInfoTag showInfo;
 818     m_database.GetTvShowInfo("", showInfo, showID);
 819     INFO_RET ret = OnProcessSeriesFolder(files, scraper, useLocal, showInfo, progress);
 820
 821     if (ret == INFO_ADDED)
 822     {
 823       std::map<int, std::map<std::string, std::string>> seasonArt;
 824       m_database.GetTvShowSeasonArt(showID, seasonArt);
 825
 826       bool updateSeasonArt = false;
 827       for (std::map<int, std::map<std::string, std::string>>::const_iterator i = seasonArt.begin(); i != seasonArt.end(); ++i)
 828       {
 829         if (i->second.empty())
 830         {
 831           updateSeasonArt = true;
 832           break;
 833         }
 834       }
 835
 836       if (updateSeasonArt)
 837       {
 838         if (!item->IsPlugin() || scraper->ID() != "metadata.local")
 839         {
 840           CVideoInfoDownloader loader(scraper);
 841           loader.GetArtwork(showInfo);
 842         }
 843         GetSeasonThumbs(showInfo, seasonArt, CVideoThumbLoader::GetArtTypes(MediaTypeSeason), useLocal && !item->IsPlugin());
 844         for (std::map<int, std::map<std::string, std::string> >::const_iterator i = seasonArt.begin(); i != seasonArt.end(); ++i)
 845         {
 846           int seasonID = m_database.AddSeason(showID, i->first);
 847           m_database.SetArtForItem(seasonID, MediaTypeSeason, i->second);
 848         }
 849       }
 850     }
 851     return ret;
 852   }
 853
 854   bool CVideoInfoScanner::EnumerateSeriesFolder(CFileItem* item, EPISODELIST& episodeList)
 855   {
 856     CFileItemList items;
 857     const std::vector<std::string> &regexps = CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_tvshowExcludeFromScanRegExps;
 858
 859     bool bSkip = false;
 860
 861     if (item->m_bIsFolder)
 862     {
 863       /*
 864        * Note: DoScan() will not remove this path as it's not recursing for tvshows.
 865        * Remove this path from the list we're processing in order to avoid hitting
 866        * it twice in the main loop.
 867        */
 868       std::set<std::string>::iterator it = m_pathsToScan.find(item->GetPath());
 869       if (it != m_pathsToScan.end())
 870         m_pathsToScan.erase(it);
 871
 872       std::string hash, dbHash;
 873       bool allowEmptyHash = false;
 874       if (item->IsPlugin())
 875       {
 876         // if plugin has already calculated a hash for directory contents - use it
 877         // in this case we don't need to get directory listing from plugin for hash checking
 878         if (item->HasProperty("hash"))
 879         {
 880           hash = item->GetProperty("hash").asString();
 881           allowEmptyHash = true;
 882         }
 883       }
 884       else if (CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_bVideoLibraryUseFastHash)
 885         hash = GetRecursiveFastHash(item->GetPath(), regexps);
 886
 887       if (m_database.GetPathHash(item->GetPath(), dbHash) && (allowEmptyHash || !hash.empty()) && StringUtils::EqualsNoCase(dbHash, hash))
 888       {
 889         // fast hashes match - no need to process anything
 890         bSkip = true;
 891       }
 892
 893       // fast hash cannot be computed or we need to rescan. fetch the listing.
 894       if (!bSkip)
 895       {
 896         int flags = DIR_FLAG_DEFAULTS;
 897         if (!hash.empty())
 898           flags |= DIR_FLAG_NO_FILE_INFO;
 899
 900         CUtil::GetRecursiveListing(item->GetPath(), items, CServiceBroker::GetFileExtensionProvider().GetVideoExtensions(), flags);
 901
 902         // fast hash failed - compute slow one
 903         if (hash.empty())
 904         {
 905           GetPathHash(items, hash);
 906           if (StringUtils::EqualsNoCase(dbHash, hash))
 907           {
 908             // slow hashes match - no need to process anything
 909             bSkip = true;
 910           }
 911         }
 912       }
 913
 914       if (bSkip)
 915       {
 916         CLog::Log(LOGDEBUG, "VideoInfoScanner: Skipping dir '{}' due to no change",
 917                   CURL::GetRedacted(item->GetPath()));
 918         // update our dialog with our progress
 919         if (m_handle)
 920           OnDirectoryScanned(item->GetPath());
 921         return false;
 922       }
 923
 924       if (dbHash.empty())
 925         CLog::Log(LOGDEBUG, "VideoInfoScanner: Scanning dir '{}' as not in the database",
 926                   CURL::GetRedacted(item->GetPath()));
 927       else
 928         CLog::Log(LOGDEBUG, "VideoInfoScanner: Rescanning dir '{}' due to change ({} != {})",
 929                   CURL::GetRedacted(item->GetPath()), dbHash, hash);
 930
 931       if (m_bClean)
 932       {
 933         m_pathsToClean.insert(m_database.GetPathId(item->GetPath()));
 934         m_database.GetPathsForTvShow(m_database.GetTvShowId(item->GetPath()), m_pathsToClean);
 935       }
 936       item->SetProperty("hash", hash);
 937     }
 938     else
 939     {
 940       CFileItemPtr newItem(new CFileItem(*item));
 941       items.Add(newItem);
 942     }
 943
 944     /*
 945     stack down any dvd folders
 946     need to sort using the full path since this is a collapsed recursive listing of all subdirs
 947     video_ts.ifo files should sort at the top of a dvd folder in ascending order
 948
 949     /foo/bar/video_ts.ifo
 950     /foo/bar/vts_x_y.ifo
 951     /foo/bar/vts_x_y.vob
 952     */
 953
 954     // since we're doing this now anyway, should other items be stacked?
 955     items.Sort(SortByPath, SortOrderAscending);
 956     int x = 0;
 957     while (x < items.Size())
 958     {
 959       if (items[x]->m_bIsFolder)
 960       {
 961         x++;
 962         continue;
 963       }
 964
 965       std::string strPathX, strFileX;
 966       URIUtils::Split(items[x]->GetPath(), strPathX, strFileX);
 967       //CLog::Log(LOGDEBUG,"{}:{}:{}", x, strPathX, strFileX);
 968
 969       const int y = x + 1;
 970       if (StringUtils::EqualsNoCase(strFileX, "VIDEO_TS.IFO"))
 971       {
 972         while (y < items.Size())
 973         {
 974           std::string strPathY, strFileY;
 975           URIUtils::Split(items[y]->GetPath(), strPathY, strFileY);
 976           //CLog::Log(LOGDEBUG," {}:{}:{}", y, strPathY, strFileY);
 977
 978           if (StringUtils::EqualsNoCase(strPathY, strPathX))
 979             /*
 980             remove everything sorted below the video_ts.ifo file in the same path.
 981             understandably this wont stack correctly if there are other files in the the dvd folder.
 982             this should be unlikely and thus is being ignored for now but we can monitor the
 983             where the path changes and potentially remove the items above the video_ts.ifo file.
 984             */
 985             items.Remove(y);
 986           else
 987             break;
 988         }
 989       }
 990       x++;
 991     }
 992
 993     // enumerate
 994     for (int i=0;i<items.Size();++i)
 995     {
 996       if (items[i]->m_bIsFolder)
 997         continue;
 998       std::string strPath = URIUtils::GetDirectory(items[i]->GetPath());
 999       URIUtils::RemoveSlashAtEnd(strPath); // want no slash for the test that follows
1000
1001       if (StringUtils::EqualsNoCase(URIUtils::GetFileName(strPath), "sample"))
1002         continue;
1003
1004       // Discard all exclude files defined by regExExcludes
1005       if (CUtil::ExcludeFileOrFolder(items[i]->GetPath(), regexps))
1006         continue;
1007
1008       /*
1009        * Check if the media source has already set the season and episode or original air date in
1010        * the VideoInfoTag. If it has, do not try to parse any of them from the file path to avoid
1011        * any false positive matches.
1012        */
1013       if (ProcessItemByVideoInfoTag(items[i].get(), episodeList))
1014         continue;
1015
1016       if (!EnumerateEpisodeItem(items[i].get(), episodeList))
1017         CLog::Log(LOGDEBUG, "VideoInfoScanner: Could not enumerate file {}", CURL::GetRedacted(items[i]->GetPath()));
1018     }
1019     return true;
1020   }
1021
1022   bool CVideoInfoScanner::ProcessItemByVideoInfoTag(const CFileItem *item, EPISODELIST &episodeList)
1023   {
1024     if (!item->HasVideoInfoTag())
1025       return false;
1026
1027     const CVideoInfoTag* tag = item->GetVideoInfoTag();
1028     bool isValid = false;
1029     /*
1030      * First check the season and episode number. This takes precedence over the original air
1031      * date and episode title. Must be a valid season and episode number combination.
1032      */
1033     if (tag->m_iSeason > -1 && tag->m_iEpisode > 0)
1034       isValid = true;
1035
1036     // episode 0 with non-zero season is valid! (e.g. prequel episode)
1037     if (item->IsPlugin() && tag->m_iSeason > 0 && tag->m_iEpisode >= 0)
1038       isValid = true;
1039
1040     if (isValid)
1041     {
1042       EPISODE episode;
1043       episode.strPath = item->GetPath();
1044       episode.iSeason = tag->m_iSeason;
1045       episode.iEpisode = tag->m_iEpisode;
1046       episode.isFolder = false;
1047       // save full item for plugin source
1048       if (item->IsPlugin())
1049         episode.item = std::make_shared<CFileItem>(*item);
1050       episodeList.push_back(episode);
1051       CLog::Log(LOGDEBUG, "{} - found match for: {}. Season {}, Episode {}", __FUNCTION__,
1052                 CURL::GetRedacted(episode.strPath), episode.iSeason, episode.iEpisode);
1053       return true;
1054     }
1055
1056     /*
1057      * Next preference is the first aired date. If it exists use that for matching the TV Show
1058      * information. Also set the title in case there are multiple matches for the first aired date.
1059      */
1060     if (tag->m_firstAired.IsValid())
1061     {
1062       EPISODE episode;
1063       episode.strPath = item->GetPath();
1064       episode.strTitle = tag->m_strTitle;
1065       episode.isFolder = false;
1066       /*
1067        * Set season and episode to -1 to indicate to use the aired date.
1068        */
1069       episode.iSeason = -1;
1070       episode.iEpisode = -1;
1071       /*
1072        * The first aired date string must be parseable.
1073        */
1074       episode.cDate = item->GetVideoInfoTag()->m_firstAired;
1075       episodeList.push_back(episode);
1076       CLog::Log(LOGDEBUG, "{} - found match for: '{}', firstAired: '{}' = '{}', title: '{}'",
1077                 __FUNCTION__, CURL::GetRedacted(episode.strPath),
1078                 tag->m_firstAired.GetAsDBDateTime(), episode.cDate.GetAsLocalizedDate(),
1079                 episode.strTitle);
1080       return true;
1081     }
1082
1083     /*
1084      * Next preference is the episode title. If it exists use that for matching the TV Show
1085      * information.
1086      */
1087     if (!tag->m_strTitle.empty())
1088     {
1089       EPISODE episode;
1090       episode.strPath = item->GetPath();
1091       episode.strTitle = tag->m_strTitle;
1092       episode.isFolder = false;
1093       /*
1094        * Set season and episode to -1 to indicate to use the title.
1095        */
1096       episode.iSeason = -1;
1097       episode.iEpisode = -1;
1098       episodeList.push_back(episode);
1099       CLog::Log(LOGDEBUG, "{} - found match for: '{}', title: '{}'", __FUNCTION__,
1100                 CURL::GetRedacted(episode.strPath), episode.strTitle);
1101       return true;
1102     }
1103
1104     /*
1105      * There is no further episode information available if both the season and episode number have
1106      * been set to 0. Return the match as true so no further matching is attempted, but don't add it
1107      * to the episode list.
1108      */
1109     if (tag->m_iSeason == 0 && tag->m_iEpisode == 0)
1110     {
1111       CLog::Log(LOGDEBUG,
1112                 "{} - found exclusion match for: {}. Both Season and Episode are 0. Item will be "
1113                 "ignored for scanning.",
1114                 __FUNCTION__, CURL::GetRedacted(item->GetPath()));
1115       return true;
1116     }
1117
1118     return false;
1119   }
1120
1121   bool CVideoInfoScanner::EnumerateEpisodeItem(const CFileItem *item, EPISODELIST& episodeList)
1122   {
1123     SETTINGS_TVSHOWLIST expression = CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_tvshowEnumRegExps;
1124
1125     std::string strLabel;
1126
1127     // remove path to main file if it's a bd or dvd folder to regex the right (folder) name
1128     if (item->IsOpticalMediaFile())
1129     {
1130       strLabel = item->GetLocalMetadataPath();
1131       URIUtils::RemoveSlashAtEnd(strLabel);
1132     }
1133     else
1134       strLabel = item->GetPath();
1135
1136     // URLDecode in case an episode is on a http/https/dav/davs:// source and URL-encoded like foo%201x01%20bar.avi
1137     strLabel = CURL::Decode(CURL::GetRedacted(strLabel));
1138
1139     for (unsigned int i=0;i<expression.size();++i)
1140     {
1141       CRegExp reg(true, CRegExp::autoUtf8);
1142       if (!reg.RegComp(expression[i].regexp))
1143         continue;
1144
1145       int regexppos, regexp2pos;
1146       //CLog::Log(LOGDEBUG,"running expression {} on {}",expression[i].regexp,strLabel);
1147       if ((regexppos = reg.RegFind(strLabel.c_str())) < 0)
1148         continue;
1149
1150       EPISODE episode;
1151       episode.strPath = item->GetPath();
1152       episode.iSeason = -1;
1153       episode.iEpisode = -1;
1154       episode.cDate.SetValid(false);
1155       episode.isFolder = false;
1156
1157       bool byDate = expression[i].byDate ? true : false;
1158       bool byTitle = expression[i].byTitle;
1159       int defaultSeason = expression[i].defaultSeason;
1160
1161       if (byDate)
1162       {
1163         if (!GetAirDateFromRegExp(reg, episode))
1164           continue;
1165
1166         CLog::Log(LOGDEBUG, "VideoInfoScanner: Found date based match {} ({}) [{}]",
1167                   CURL::GetRedacted(episode.strPath), episode.cDate.GetAsLocalizedDate(),
1168                   expression[i].regexp);
1169       }
1170       else if (byTitle)
1171       {
1172         if (!GetEpisodeTitleFromRegExp(reg, episode))
1173           continue;
1174
1175         CLog::Log(LOGDEBUG, "VideoInfoScanner: Found title based match {} ({}) [{}]",
1176                   CURL::GetRedacted(episode.strPath), episode.strTitle, expression[i].regexp);
1177       }
1178       else
1179       {
1180         if (!GetEpisodeAndSeasonFromRegExp(reg, episode, defaultSeason))
1181           continue;
1182
1183         CLog::Log(LOGDEBUG, "VideoInfoScanner: Found episode match {} (s{}e{}) [{}]",
1184                   CURL::GetRedacted(episode.strPath), episode.iSeason, episode.iEpisode,
1185                   expression[i].regexp);
1186       }
1187
1188       // Grab the remainder from first regexp run
1189       // as second run might modify or empty it.
1190       std::string remainder(reg.GetMatch(3));
1191
1192       /*
1193        * Check if the files base path is a dedicated folder that contains
1194        * only this single episode. If season and episode match with the
1195        * actual media file, we set episode.isFolder to true.
1196        */
1197       std::string strBasePath = item->GetBaseMoviePath(true);
1198       URIUtils::RemoveSlashAtEnd(strBasePath);
1199       strBasePath = URIUtils::GetFileName(strBasePath);
1200
1201       if (reg.RegFind(strBasePath.c_str()) > -1)
1202       {
1203         EPISODE parent;
1204         if (byDate)
1205         {
1206           GetAirDateFromRegExp(reg, parent);
1207           if (episode.cDate == parent.cDate)
1208             episode.isFolder = true;
1209         }
1210         else
1211         {
1212           GetEpisodeAndSeasonFromRegExp(reg, parent, defaultSeason);
1213           if (episode.iSeason == parent.iSeason && episode.iEpisode == parent.iEpisode)
1214             episode.isFolder = true;
1215         }
1216       }
1217
1218       // add what we found by now
1219       episodeList.push_back(episode);
1220
1221       CRegExp reg2(true, CRegExp::autoUtf8);
1222       // check the remainder of the string for any further episodes.
1223       if (!byDate && reg2.RegComp(CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_tvshowMultiPartEnumRegExp))
1224       {
1225         int offset = 0;
1226
1227         // we want "long circuit" OR below so that both offsets are evaluated
1228         while (static_cast<int>((regexp2pos = reg2.RegFind(remainder.c_str() + offset)) > -1) |
1229                static_cast<int>((regexppos = reg.RegFind(remainder.c_str() + offset)) > -1))
1230         {
1231           if (((regexppos <= regexp2pos) && regexppos != -1) ||
1232              (regexppos >= 0 && regexp2pos == -1))
1233           {
1234             GetEpisodeAndSeasonFromRegExp(reg, episode, defaultSeason);
1235
1236             CLog::Log(LOGDEBUG, "VideoInfoScanner: Adding new season {}, multipart episode {} [{}]",
1237                       episode.iSeason, episode.iEpisode,
1238                       CServiceBroker::GetSettingsComponent()
1239                           ->GetAdvancedSettings()
1240                           ->m_tvshowMultiPartEnumRegExp);
1241
1242             episodeList.push_back(episode);
1243             remainder = reg.GetMatch(3);
1244             offset = 0;
1245           }
1246           else if (((regexp2pos < regexppos) && regexp2pos != -1) ||
1247                    (regexp2pos >= 0 && regexppos == -1))
1248           {
1249             episode.iEpisode = atoi(reg2.GetMatch(1).c_str());
1250             CLog::Log(LOGDEBUG, "VideoInfoScanner: Adding multipart episode {} [{}]",
1251                       episode.iEpisode,
1252                       CServiceBroker::GetSettingsComponent()
1253                           ->GetAdvancedSettings()
1254                           ->m_tvshowMultiPartEnumRegExp);
1255             episodeList.push_back(episode);
1256             offset += regexp2pos + reg2.GetFindLen();
1257           }
1258         }
1259       }
1260       return true;
1261     }
1262     return false;
1263   }
1264
1265   bool CVideoInfoScanner::GetEpisodeAndSeasonFromRegExp(CRegExp &reg, EPISODE &episodeInfo, int defaultSeason)
1266   {
1267     std::string season(reg.GetMatch(1));
1268     std::string episode(reg.GetMatch(2));
1269
1270     if (!season.empty() || !episode.empty())
1271     {
1272       char* endptr = NULL;
1273       if (season.empty() && !episode.empty())
1274       { // no season specified -> assume defaultSeason
1275         episodeInfo.iSeason = defaultSeason;
1276         if ((episodeInfo.iEpisode = CUtil::TranslateRomanNumeral(episode.c_str())) == -1)
1277           episodeInfo.iEpisode = strtol(episode.c_str(), &endptr, 10);
1278       }
1279       else if (!season.empty() && episode.empty())
1280       { // no episode specification -> assume defaultSeason
1281         episodeInfo.iSeason = defaultSeason;
1282         if ((episodeInfo.iEpisode = CUtil::TranslateRomanNumeral(season.c_str())) == -1)
1283           episodeInfo.iEpisode = atoi(season.c_str());
1284       }
1285       else
1286       { // season and episode specified
1287         episodeInfo.iSeason = atoi(season.c_str());
1288         episodeInfo.iEpisode = strtol(episode.c_str(), &endptr, 10);
1289       }
1290       if (endptr)
1291       {
1292         if (isalpha(*endptr))
1293           episodeInfo.iSubepisode = *endptr - (islower(*endptr) ? 'a' : 'A') + 1;
1294         else if (*endptr == '.')
1295           episodeInfo.iSubepisode = atoi(endptr+1);
1296       }
1297       return true;
1298     }
1299     return false;
1300   }
1301
1302   bool CVideoInfoScanner::GetAirDateFromRegExp(CRegExp &reg, EPISODE &episodeInfo)
1303   {
1304     std::string param1(reg.GetMatch(1));
1305     std::string param2(reg.GetMatch(2));
1306     std::string param3(reg.GetMatch(3));
1307
1308     if (!param1.empty() && !param2.empty() && !param3.empty())
1309     {
1310       // regular expression by date
1311       int len1 = param1.size();
1312       int len2 = param2.size();
1313       int len3 = param3.size();
1314
1315       if (len1==4 && len2==2 && len3==2)
1316       {
1317         // yyyy mm dd format
1318         episodeInfo.cDate.SetDate(atoi(param1.c_str()), atoi(param2.c_str()), atoi(param3.c_str()));
1319       }
1320       else if (len1==2 && len2==2 && len3==4)
1321       {
1322         // mm dd yyyy format
1323         episodeInfo.cDate.SetDate(atoi(param3.c_str()), atoi(param1.c_str()), atoi(param2.c_str()));
1324       }
1325     }
1326     return episodeInfo.cDate.IsValid();
1327   }
1328
1329   bool CVideoInfoScanner::GetEpisodeTitleFromRegExp(CRegExp& reg, EPISODE& episodeInfo)
1330   {
1331     std::string param1(reg.GetMatch(1));
1332
1333     if (!param1.empty())
1334     {
1335       episodeInfo.strTitle = param1;
1336       return true;
1337     }
1338     return false;
1339   }
1340
1341   long CVideoInfoScanner::AddVideo(CFileItem *pItem, const CONTENT_TYPE &content, bool videoFolder /* = false */, bool useLocal /* = true */, const CVideoInfoTag *showInfo /* = NULL */, bool libraryImport /* = false */)
1342   {
1343     // ensure our database is open (this can get called via other classes)
1344     if (!m_database.Open())
1345       return -1;
1346
1347     if (!libraryImport)
1348       GetArtwork(pItem, content, videoFolder, useLocal && !pItem->IsPlugin(), showInfo ? showInfo->m_strPath : "");
1349
1350     // ensure the art map isn't completely empty by specifying an empty thumb
1351     std::map<std::string, std::string> art = pItem->GetArt();
1352     if (art.empty())
1353       art["thumb"] = "";
1354
1355     CVideoInfoTag &movieDetails = *pItem->GetVideoInfoTag();
1356     if (movieDetails.m_basePath.empty())
1357       movieDetails.m_basePath = pItem->GetBaseMoviePath(videoFolder);
1358     movieDetails.m_parentPathID = m_database.AddPath(URIUtils::GetParentPath(movieDetails.m_basePath));
1359
1360     movieDetails.m_strFileNameAndPath = pItem->GetPath();
1361
1362     if (pItem->m_bIsFolder)
1363       movieDetails.m_strPath = pItem->GetPath();
1364
1365     std::string strTitle(movieDetails.m_strTitle);
1366
1367     if (showInfo && content == CONTENT_TVSHOWS)
1368     {
1369       strTitle = StringUtils::Format("{} - {}x{} - {}", showInfo->m_strTitle,
1370                                      movieDetails.m_iSeason, movieDetails.m_iEpisode, strTitle);
1371     }
1372
1373     if (CServiceBroker::GetSettingsComponent()->GetSettings()->GetBool(
1374             CSettings::SETTING_MYVIDEOS_EXTRACTFLAGS) &&
1375         CDVDFileInfo::GetFileStreamDetails(pItem))
1376       CLog::Log(LOGDEBUG, "VideoInfoScanner: Extracted filestream details from video file {}",
1377                 CURL::GetRedacted(pItem->GetPath()));
1378
1379     CLog::Log(LOGDEBUG, "VideoInfoScanner: Adding new item to {}:{}", TranslateContent(content), CURL::GetRedacted(pItem->GetPath()));
1380     long lResult = -1;
1381
1382     if (content == CONTENT_MOVIES)
1383     {
1384       // find local trailer first
1385       std::string strTrailer = pItem->FindTrailer();
1386       if (!strTrailer.empty())
1387         movieDetails.m_strTrailer = strTrailer;
1388
1389       lResult = m_database.SetDetailsForMovie(movieDetails, art);
1390       movieDetails.m_iDbId = lResult;
1391       movieDetails.m_type = MediaTypeMovie;
1392
1393       // setup links to shows if the linked shows are in the db
1394       for (unsigned int i=0; i < movieDetails.m_showLink.size(); ++i)
1395       {
1396         CFileItemList items;
1397         m_database.GetTvShowsByName(movieDetails.m_showLink[i], items);
1398         if (items.Size())
1399           m_database.LinkMovieToTvshow(lResult, items[0]->GetVideoInfoTag()->m_iDbId, false);
1400         else
1401           CLog::Log(LOGDEBUG, "VideoInfoScanner: Failed to link movie {} to show {}",
1402                     movieDetails.m_strTitle, movieDetails.m_showLink[i]);
1403       }
1404     }
1405     else if (content == CONTENT_TVSHOWS)
1406     {
1407       if (pItem->m_bIsFolder)
1408       {
1409         /*
1410          multipaths are not stored in the database, so in the case we have one,
1411          we split the paths, and compute the parent paths in each case.
1412          */
1413         std::vector<std::string> multipath;
1414         if (!URIUtils::IsMultiPath(pItem->GetPath()) || !CMultiPathDirectory::GetPaths(pItem->GetPath(), multipath))
1415           multipath.push_back(pItem->GetPath());
1416         std::vector<std::pair<std::string, std::string> > paths;
1417         for (std::vector<std::string>::const_iterator i = multipath.begin(); i != multipath.end(); ++i)
1418           paths.emplace_back(*i, URIUtils::GetParentPath(*i));
1419
1420         std::map<int, std::map<std::string, std::string> > seasonArt;
1421
1422         if (!libraryImport)
1423           GetSeasonThumbs(movieDetails, seasonArt, CVideoThumbLoader::GetArtTypes(MediaTypeSeason), useLocal && !pItem->IsPlugin());
1424
1425         lResult = m_database.SetDetailsForTvShow(paths, movieDetails, art, seasonArt);
1426         movieDetails.m_iDbId = lResult;
1427         movieDetails.m_type = MediaTypeTvShow;
1428       }
1429       else
1430       {
1431         // we add episode then set details, as otherwise set details will delete the
1432         // episode then add, which breaks multi-episode files.
1433         int idShow = showInfo ? showInfo->m_iDbId : -1;
1434         int idEpisode = m_database.AddNewEpisode(idShow, movieDetails);
1435         lResult = m_database.SetDetailsForEpisode(movieDetails, art, idShow, idEpisode);
1436         movieDetails.m_iDbId = lResult;
1437         movieDetails.m_type = MediaTypeEpisode;
1438         movieDetails.m_strShowTitle = showInfo ? showInfo->m_strTitle : "";
1439         if (movieDetails.m_EpBookmark.timeInSeconds > 0)
1440         {
1441           movieDetails.m_strFileNameAndPath = pItem->GetPath();
1442           movieDetails.m_EpBookmark.seasonNumber = movieDetails.m_iSeason;
1443           movieDetails.m_EpBookmark.episodeNumber = movieDetails.m_iEpisode;
1444           m_database.AddBookMarkForEpisode(movieDetails, movieDetails.m_EpBookmark);
1445         }
1446       }
1447     }
1448     else if (content == CONTENT_MUSICVIDEOS)
1449     {
1450       lResult = m_database.SetDetailsForMusicVideo(movieDetails, art);
1451       movieDetails.m_iDbId = lResult;
1452       movieDetails.m_type = MediaTypeMusicVideo;
1453     }
1454
1455     if (!pItem->m_bIsFolder)
1456     {
1457       if (CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_bVideoLibraryImportWatchedState || libraryImport)
1458         m_database.SetPlayCount(*pItem, movieDetails.GetPlayCount(), movieDetails.m_lastPlayed);
1459
1460       if ((CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_bVideoLibraryImportResumePoint || libraryImport) &&
1461           movieDetails.GetResumePoint().IsSet())
1462         m_database.AddBookMarkToFile(pItem->GetPath(), movieDetails.GetResumePoint(), CBookmark::RESUME);
1463     }
1464
1465     m_database.Close();
1466
1467     CFileItemPtr itemCopy = CFileItemPtr(new CFileItem(*pItem));
1468     CVariant data;
1469     data["added"] = true;
1470     if (m_bRunning)
1471       data["transaction"] = true;
1472     CServiceBroker::GetAnnouncementManager()->Announce(ANNOUNCEMENT::VideoLibrary, "OnUpdate",
1473                                                        itemCopy, data);
1474     return lResult;
1475   }
1476
1477   std::string ContentToMediaType(CONTENT_TYPE content, bool folder)
1478   {
1479     switch (content)
1480     {
1481       case CONTENT_MOVIES:
1482         return MediaTypeMovie;
1483       case CONTENT_MUSICVIDEOS:
1484         return MediaTypeMusicVideo;
1485       case CONTENT_TVSHOWS:
1486         return folder ? MediaTypeTvShow : MediaTypeEpisode;
1487       default:
1488         return "";
1489     }
1490   }
1491
1492   std::string CVideoInfoScanner::GetArtTypeFromSize(unsigned int width, unsigned int height)
1493   {
1494     std::string type = "thumb";
1495     if (width*5 < height*4)
1496       type = "poster";
1497     else if (width*1 > height*4)
1498       type = "banner";
1499     return type;
1500   }
1501
1502   std::string CVideoInfoScanner::GetMovieSetInfoFolder(const std::string& setTitle)
1503   {
1504     if (setTitle.empty())
1505       return "";
1506     std::string path = CServiceBroker::GetSettingsComponent()->GetSettings()->GetString(
1507         CSettings::SETTING_VIDEOLIBRARY_MOVIESETSFOLDER);
1508     if (path.empty())
1509       return "";
1510     path = URIUtils::AddFileToFolder(path, CUtil::MakeLegalFileName(setTitle, LEGAL_WIN32_COMPAT));
1511     URIUtils::AddSlashAtEnd(path);
1512     CLog::Log(LOGDEBUG,
1513         "VideoInfoScanner: Looking for local artwork for movie set '{}' in folder '{}'",
1514         setTitle,
1515         CURL::GetRedacted(path));
1516     return CDirectory::Exists(path) ? path : "";
1517   }
1518
1519   void CVideoInfoScanner::AddLocalItemArtwork(CGUIListItem::ArtMap& itemArt,
1520     const std::vector<std::string>& wantedArtTypes, const std::string& itemPath,
1521     bool addAll, bool exactName)
1522   {
1523     std::string path = URIUtils::GetDirectory(itemPath);
1524     if (path.empty())
1525       return;
1526
1527     CFileItemList availableArtFiles;
1528     CDirectory::GetDirectory(path, availableArtFiles,
1529         CServiceBroker::GetFileExtensionProvider().GetPictureExtensions(),
1530         DIR_FLAG_NO_FILE_DIRS | DIR_FLAG_READ_CACHE | DIR_FLAG_NO_FILE_INFO);
1531
1532     std::string baseFilename = URIUtils::GetFileName(itemPath);
1533     if (!baseFilename.empty())
1534     {
1535       URIUtils::RemoveExtension(baseFilename);
1536       baseFilename.append("-");
1537     }
1538
1539     for (const auto& artFile : availableArtFiles)
1540     {
1541       std::string candidate = URIUtils::GetFileName(artFile->GetPath());
1542
1543       bool matchesFilename =
1544         !baseFilename.empty() && StringUtils::StartsWith(candidate, baseFilename);
1545       if (!baseFilename.empty() && !matchesFilename)
1546         continue;
1547
1548       if (matchesFilename)
1549         candidate.erase(0, baseFilename.length());
1550       URIUtils::RemoveExtension(candidate);
1551       StringUtils::ToLower(candidate);
1552
1553       // move 'folder' to thumb / poster / banner based on aspect ratio
1554       // if such artwork doesn't already exist
1555       if (!matchesFilename && StringUtils::EqualsNoCase(candidate, "folder") &&
1556         !CVideoThumbLoader::IsArtTypeInWhitelist("folder", wantedArtTypes, exactName))
1557       {
1558         // cache the image to determine sizing
1559         CTextureDetails details;
1560         if (CServiceBroker::GetTextureCache()->CacheImage(artFile->GetPath(), details))
1561         {
1562           candidate = GetArtTypeFromSize(details.width, details.height);
1563           if (itemArt.find(candidate) != itemArt.end())
1564             continue;
1565         }
1566       }
1567
1568       if ((addAll && CVideoThumbLoader::IsValidArtType(candidate)) ||
1569         CVideoThumbLoader::IsArtTypeInWhitelist(candidate, wantedArtTypes, exactName))
1570       {
1571         itemArt[candidate] = artFile->GetPath();
1572       }
1573     }
1574   }
1575
1576   void CVideoInfoScanner::GetArtwork(CFileItem *pItem, const CONTENT_TYPE &content, bool bApplyToDir, bool useLocal, const std::string &actorArtPath)
1577   {
1578     int artLevel = CServiceBroker::GetSettingsComponent()->GetSettings()->GetInt(
1579         CSettings::SETTING_VIDEOLIBRARY_ARTWORK_LEVEL);
1580     if (artLevel == CSettings::VIDEOLIBRARY_ARTWORK_LEVEL_NONE)
1581       return;
1582
1583     CVideoInfoTag &movieDetails = *pItem->GetVideoInfoTag();
1584     movieDetails.m_fanart.Unpack();
1585     movieDetails.m_strPictureURL.Parse();
1586
1587     CGUIListItem::ArtMap art = pItem->GetArt();
1588
1589     // get and cache thumb images
1590     std::string mediaType = ContentToMediaType(content, pItem->m_bIsFolder);
1591     std::vector<std::string> artTypes = CVideoThumbLoader::GetArtTypes(mediaType);
1592     bool moviePartOfSet = content == CONTENT_MOVIES && !movieDetails.m_set.title.empty();
1593     std::vector<std::string> movieSetArtTypes;
1594     if (moviePartOfSet)
1595     {
1596       movieSetArtTypes = CVideoThumbLoader::GetArtTypes(MediaTypeVideoCollection);
1597       for (const std::string& artType : movieSetArtTypes)
1598         artTypes.push_back("set." + artType);
1599     }
1600     bool addAll = artLevel == CSettings::VIDEOLIBRARY_ARTWORK_LEVEL_ALL;
1601     bool exactName = artLevel == CSettings::VIDEOLIBRARY_ARTWORK_LEVEL_BASIC;
1602     // find local art
1603     if (useLocal)
1604     {
1605       if (!pItem->SkipLocalArt())
1606       {
1607         if (bApplyToDir && (content == CONTENT_MOVIES || content == CONTENT_MUSICVIDEOS))
1608         {
1609           std::string filename = pItem->GetLocalArtBaseFilename();
1610           std::string directory = URIUtils::GetDirectory(filename);
1611           if (filename != directory)
1612             AddLocalItemArtwork(art, artTypes, directory, addAll, exactName);
1613         }
1614         AddLocalItemArtwork(art, artTypes, pItem->GetLocalArtBaseFilename(), addAll, exactName);
1615       }
1616
1617       if (moviePartOfSet)
1618       {
1619         std::string movieSetInfoPath = GetMovieSetInfoFolder(movieDetails.m_set.title);
1620         if (!movieSetInfoPath.empty())
1621         {
1622           CGUIListItem::ArtMap movieSetArt;
1623           AddLocalItemArtwork(movieSetArt, movieSetArtTypes, movieSetInfoPath, addAll, exactName);
1624           for (const auto& artItem : movieSetArt)
1625           {
1626             art["set." + artItem.first] = artItem.second;
1627           }
1628         }
1629       }
1630     }
1631
1632     // find embedded art
1633     if (pItem->HasVideoInfoTag() && !pItem->GetVideoInfoTag()->m_coverArt.empty())
1634     {
1635       for (auto& it : pItem->GetVideoInfoTag()->m_coverArt)
1636       {
1637         if ((addAll || CVideoThumbLoader::IsArtTypeInWhitelist(it.m_type, artTypes, exactName)) &&
1638           art.find(it.m_type) == art.end())
1639         {
1640           std::string thumb = CTextureUtils::GetWrappedImageURL(pItem->GetPath(),
1641                                                                 "video_" + it.m_type);
1642           art.insert(std::make_pair(it.m_type, thumb));
1643         }
1644       }
1645     }
1646
1647     // add online fanart (treated separately due to it being stored in m_fanart)
1648     if ((addAll || CVideoThumbLoader::IsArtTypeInWhitelist("fanart", artTypes, exactName)) &&
1649       art.find("fanart") == art.end())
1650     {
1651       std::string fanart = pItem->GetVideoInfoTag()->m_fanart.GetImageURL();
1652       if (!fanart.empty())
1653         art.insert(std::make_pair("fanart", fanart));
1654     }
1655
1656     // add online art
1657     for (const auto& url : pItem->GetVideoInfoTag()->m_strPictureURL.GetUrls())
1658     {
1659       if (url.m_type != CScraperUrl::UrlType::General)
1660         continue;
1661       std::string aspect = url.m_aspect;
1662       if (aspect.empty())
1663         // Backward compatibility with Kodi 11 Eden NFO files
1664         aspect = mediaType == MediaTypeEpisode ? "thumb" : "poster";
1665
1666       if ((addAll || CVideoThumbLoader::IsArtTypeInWhitelist(aspect, artTypes, exactName)) &&
1667         art.find(aspect) == art.end())
1668       {
1669         std::string image = GetImage(url, pItem->GetPath());
1670         if (!image.empty())
1671           art.insert(std::make_pair(aspect, image));
1672       }
1673     }
1674
1675     for (const auto& artType : artTypes)
1676     {
1677       if (art.find(artType) != art.end())
1678         CServiceBroker::GetTextureCache()->BackgroundCacheImage(art[artType]);
1679     }
1680
1681     pItem->SetArt(art);
1682
1683     // parent folder to apply the thumb to and to search for local actor thumbs
1684     std::string parentDir = URIUtils::GetBasePath(pItem->GetPath());
1685     if (CServiceBroker::GetSettingsComponent()->GetSettings()->GetBool(CSettings::SETTING_VIDEOLIBRARY_ACTORTHUMBS))
1686       FetchActorThumbs(movieDetails.m_cast, actorArtPath.empty() ? parentDir : actorArtPath);
1687     if (bApplyToDir)
1688       ApplyThumbToFolder(parentDir, art["thumb"]);
1689   }
1690
1691   std::string CVideoInfoScanner::GetImage(const CScraperUrl::SUrlEntry &image, const std::string& itemPath)
1692   {
1693     std::string thumb = CScraperUrl::GetThumbUrl(image);
1694     if (!thumb.empty() && thumb.find('/') == std::string::npos &&
1695         thumb.find('\\') == std::string::npos)
1696     {
1697       std::string strPath = URIUtils::GetDirectory(itemPath);
1698       thumb = URIUtils::AddFileToFolder(strPath, thumb);
1699     }
1700     return thumb;
1701   }
1702
1703   CInfoScanner::INFO_RET
1704   CVideoInfoScanner::OnProcessSeriesFolder(EPISODELIST& files,
1705                                            const ADDON::ScraperPtr &scraper,
1706                                            bool useLocal,
1707                                            const CVideoInfoTag& showInfo,
1708                                            CGUIDialogProgress* pDlgProgress /* = NULL */)
1709   {
1710     if (pDlgProgress)
1711     {
1712       pDlgProgress->SetLine(1, CVariant{showInfo.m_strTitle});
1713       pDlgProgress->SetLine(2, CVariant{20361});
1714       pDlgProgress->SetPercentage(0);
1715       pDlgProgress->ShowProgressBar(true);
1716       pDlgProgress->Progress();
1717     }
1718
1719     EPISODELIST episodes;
1720     bool hasEpisodeGuide = false;
1721
1722     int iMax = files.size();
1723     int iCurr = 1;
1724     for (EPISODELIST::iterator file = files.begin(); file != files.end(); ++file)
1725     {
1726       if (pDlgProgress)
1727       {
1728         pDlgProgress->SetLine(2, CVariant{20361});
1729         pDlgProgress->SetPercentage((int)((float)(iCurr++)/iMax*100));
1730         pDlgProgress->Progress();
1731       }
1732       if (m_handle)
1733         m_handle->SetPercentage(100.f*iCurr++/iMax);
1734
1735       if ((pDlgProgress && pDlgProgress->IsCanceled()) || m_bStop)
1736         return INFO_CANCELLED;
1737
1738       if (m_database.GetEpisodeId(file->strPath, file->iEpisode, file->iSeason) > -1)
1739       {
1740         if (m_handle)
1741           m_handle->SetText(g_localizeStrings.Get(20415));
1742         continue;
1743       }
1744
1745       CFileItem item;
1746       if (file->item)
1747         item = *file->item;
1748       else
1749       {
1750         item.SetPath(file->strPath);
1751         item.GetVideoInfoTag()->m_iEpisode = file->iEpisode;
1752       }
1753
1754       // handle .nfo files
1755       CInfoScanner::INFO_TYPE result=CInfoScanner::NO_NFO;
1756       CScraperUrl scrUrl;
1757       const ScraperPtr& info(scraper);
1758       std::unique_ptr<IVideoInfoTagLoader> loader;
1759       if (useLocal)
1760       {
1761         loader.reset(CVideoInfoTagLoaderFactory::CreateLoader(item, info, false));
1762         if (loader)
1763         {
1764           // no reset here on purpose
1765           result = loader->Load(*item.GetVideoInfoTag(), false);
1766         }
1767       }
1768       if (result == CInfoScanner::FULL_NFO)
1769       {
1770         // override with episode and season number from file if available
1771         if (file->iEpisode > -1)
1772         {
1773           item.GetVideoInfoTag()->m_iEpisode = file->iEpisode;
1774           item.GetVideoInfoTag()->m_iSeason = file->iSeason;
1775         }
1776         if (AddVideo(&item, CONTENT_TVSHOWS, file->isFolder, true, &showInfo) < 0)
1777           return INFO_ERROR;
1778         continue;
1779       }
1780
1781       if (!hasEpisodeGuide)
1782       {
1783         // fetch episode guide
1784         if (!showInfo.m_strEpisodeGuide.empty())
1785         {
1786           CScraperUrl url;
1787           url.ParseAndAppendUrlsFromEpisodeGuide(showInfo.m_strEpisodeGuide);
1788
1789           if (pDlgProgress)
1790           {
1791             pDlgProgress->SetLine(2, CVariant{20354});
1792             pDlgProgress->Progress();
1793           }
1794
1795           CVideoInfoDownloader imdb(scraper);
1796           if (!imdb.GetEpisodeList(url, episodes))
1797             return INFO_NOT_FOUND;
1798
1799           hasEpisodeGuide = true;
1800         }
1801       }
1802
1803       if (episodes.empty())
1804       {
1805         CLog::Log(LOGERROR,
1806                   "VideoInfoScanner: Asked to lookup episode {}"
1807                   " online, but we have no episode guide. Check your tvshow.nfo and make"
1808                   " sure the <episodeguide> tag is in place.",
1809                   CURL::GetRedacted(file->strPath));
1810         continue;
1811       }
1812
1813       EPISODE key(file->iSeason, file->iEpisode, file->iSubepisode);
1814       EPISODE backupkey(file->iSeason, file->iEpisode, 0);
1815       bool bFound = false;
1816       EPISODELIST::iterator guide = episodes.begin();
1817       EPISODELIST matches;
1818
1819       for (; guide != episodes.end(); ++guide )
1820       {
1821         if ((file->iEpisode!=-1) && (file->iSeason!=-1))
1822         {
1823           if (key==*guide)
1824           {
1825             bFound = true;
1826             break;
1827           }
1828           else if ((file->iSubepisode!=0) && (backupkey==*guide))
1829           {
1830             matches.push_back(*guide);
1831             continue;
1832           }
1833         }
1834         if (file->cDate.IsValid() && guide->cDate.IsValid() && file->cDate==guide->cDate)
1835         {
1836           matches.push_back(*guide);
1837           continue;
1838         }
1839         if (!guide->cScraperUrl.GetTitle().empty() &&
1840             StringUtils::EqualsNoCase(guide->cScraperUrl.GetTitle(), file->strTitle))
1841         {
1842           bFound = true;
1843           break;
1844         }
1845         if (!guide->strTitle.empty() && StringUtils::EqualsNoCase(guide->strTitle, file->strTitle))
1846         {
1847           bFound = true;
1848           break;
1849         }
1850       }
1851
1852       if (!bFound)
1853       {
1854         /*
1855          * If there is only one match or there are matches but no title to compare with to help
1856          * identify the best match, then pick the first match as the best possible candidate.
1857          *
1858          * Otherwise, use the title to further refine the best match.
1859          */
1860         if (matches.size() == 1 || (file->strTitle.empty() && matches.size() > 1))
1861         {
1862           guide = matches.begin();
1863           bFound = true;
1864         }
1865         else if (!file->strTitle.empty())
1866         {
1867           CLog::Log(LOGDEBUG, "VideoInfoScanner: analyzing parsed title '{}'", file->strTitle);
1868           double minscore = 0; // Default minimum score is 0 to find whatever is the best match.
1869
1870           EPISODELIST *candidates;
1871           if (matches.empty()) // No matches found using earlier criteria. Use fuzzy match on titles across all episodes.
1872           {
1873             minscore = 0.8; // 80% should ensure a good match.
1874             candidates = &episodes;
1875           }
1876           else // Multiple matches found. Use fuzzy match on the title with already matched episodes to pick the best.
1877             candidates = &matches;
1878
1879           std::vector<std::string> titles;
1880           for (guide = candidates->begin(); guide != candidates->end(); ++guide)
1881           {
1882             auto title = guide->cScraperUrl.GetTitle();
1883             if (title.empty())
1884             {
1885               title = guide->strTitle;
1886             }
1887             StringUtils::ToLower(title);
1888             guide->cScraperUrl.SetTitle(title);
1889             titles.push_back(title);
1890           }
1891
1892           double matchscore;
1893           std::string loweredTitle(file->strTitle);
1894           StringUtils::ToLower(loweredTitle);
1895           int index = StringUtils::FindBestMatch(loweredTitle, titles, matchscore);
1896           if (index >= 0 && matchscore >= minscore)
1897           {
1898             guide = candidates->begin() + index;
1899             bFound = true;
1900             CLog::Log(LOGDEBUG,
1901                       "{} fuzzy title match for show: '{}', title: '{}', match: '{}', score: {:f} "
1902                       ">= {:f}",
1903                       __FUNCTION__, showInfo.m_strTitle, file->strTitle, titles[index], matchscore,
1904                       minscore);
1905           }
1906         }
1907       }
1908
1909       if (bFound)
1910       {
1911         CVideoInfoDownloader imdb(scraper);
1912         CFileItem item;
1913         item.SetPath(file->strPath);
1914         if (!imdb.GetEpisodeDetails(guide->cScraperUrl, *item.GetVideoInfoTag(), pDlgProgress))
1915           return INFO_NOT_FOUND; //! @todo should we just skip to the next episode?
1916
1917         // Only set season/epnum from filename when it is not already set by a scraper
1918         if (item.GetVideoInfoTag()->m_iSeason == -1)
1919           item.GetVideoInfoTag()->m_iSeason = guide->iSeason;
1920         if (item.GetVideoInfoTag()->m_iEpisode == -1)
1921           item.GetVideoInfoTag()->m_iEpisode = guide->iEpisode;
1922
1923         if (AddVideo(&item, CONTENT_TVSHOWS, file->isFolder, useLocal, &showInfo) < 0)
1924           return INFO_ERROR;
1925       }
1926       else
1927       {
1928         CLog::Log(
1929             LOGDEBUG,
1930             "{} - no match for show: '{}', season: {}, episode: {}.{}, airdate: '{}', title: '{}'",
1931             __FUNCTION__, showInfo.m_strTitle, file->iSeason, file->iEpisode, file->iSubepisode,
1932             file->cDate.GetAsLocalizedDate(), file->strTitle);
1933       }
1934     }
1935     return INFO_ADDED;
1936   }
1937
1938   bool CVideoInfoScanner::GetDetails(CFileItem *pItem, CScraperUrl &url,
1939                                      const ScraperPtr& scraper,
1940                                      IVideoInfoTagLoader* loader,
1941                                      CGUIDialogProgress* pDialog /* = NULL */)
1942   {
1943     CVideoInfoTag movieDetails;
1944
1945     if (m_handle && !url.GetTitle().empty())
1946       m_handle->SetText(url.GetTitle());
1947
1948     CVideoInfoDownloader imdb(scraper);
1949     bool ret = imdb.GetDetails(url, movieDetails, pDialog);
1950
1951     if (ret)
1952     {
1953       if (loader)
1954         loader->Load(movieDetails, true);
1955
1956       if (m_handle && url.GetTitle().empty())
1957         m_handle->SetText(movieDetails.m_strTitle);
1958
1959       if (pDialog)
1960       {
1961         pDialog->SetLine(1, CVariant{movieDetails.m_strTitle});
1962         pDialog->Progress();
1963       }
1964
1965       *pItem->GetVideoInfoTag() = movieDetails;
1966       return true;
1967     }
1968     return false; // no info found, or cancelled
1969   }
1970
1971   void CVideoInfoScanner::ApplyThumbToFolder(const std::string &folder, const std::string &imdbThumb)
1972   {
1973     // copy icon to folder also;
1974     if (!imdbThumb.empty())
1975     {
1976       CFileItem folderItem(folder, true);
1977       CThumbLoader loader;
1978       loader.SetCachedImage(folderItem, "thumb", imdbThumb);
1979     }
1980   }
1981
1982   int CVideoInfoScanner::GetPathHash(const CFileItemList &items, std::string &hash)
1983   {
1984     // Create a hash based on the filenames, filesize and filedate.  Also count the number of files
1985     if (0 == items.Size()) return 0;
1986     CDigest digest{CDigest::Type::MD5};
1987     int count = 0;
1988     for (int i = 0; i < items.Size(); ++i)
1989     {
1990       const CFileItemPtr pItem = items[i];
1991       digest.Update(pItem->GetPath());
1992       if (pItem->IsPlugin())
1993       {
1994         // allow plugin to calculate hash itself using strings rather than binary data for size and date
1995         // according to ListItem.setInfo() documentation date format should be "d.m.Y"
1996         if (pItem->m_dwSize)
1997           digest.Update(std::to_string(pItem->m_dwSize));
1998         if (pItem->m_dateTime.IsValid())
1999           digest.Update(StringUtils::Format("{:02}.{:02}.{:04}", pItem->m_dateTime.GetDay(),
2000                                             pItem->m_dateTime.GetMonth(),
2001                                             pItem->m_dateTime.GetYear()));
2002       }
2003       else
2004       {
2005         digest.Update(&pItem->m_dwSize, sizeof(pItem->m_dwSize));
2006         KODI::TIME::FileTime time = pItem->m_dateTime;
2007         digest.Update(&time, sizeof(KODI::TIME::FileTime));
2008       }
2009       if (pItem->IsVideo() && !pItem->IsPlayList() && !pItem->IsNFO())
2010         count++;
2011     }
2012     hash = digest.Finalize();
2013     return count;
2014   }
2015
2016   bool CVideoInfoScanner::CanFastHash(const CFileItemList &items, const std::vector<std::string> &excludes) const
2017   {
2018     if (!CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_bVideoLibraryUseFastHash || items.IsPlugin())
2019       return false;
2020
2021     for (int i = 0; i < items.Size(); ++i)
2022     {
2023       if (items[i]->m_bIsFolder && !CUtil::ExcludeFileOrFolder(items[i]->GetPath(), excludes))
2024         return false;
2025     }
2026     return true;
2027   }
2028
2029   std::string CVideoInfoScanner::GetFastHash(const std::string &directory,
2030       const std::vector<std::string> &excludes) const
2031   {
2032     CDigest digest{CDigest::Type::MD5};
2033
2034     if (excludes.size())
2035       digest.Update(StringUtils::Join(excludes, "|"));
2036
2037     struct __stat64 buffer;
2038     if (XFILE::CFile::Stat(directory, &buffer) == 0)
2039     {
2040       int64_t time = buffer.st_mtime;
2041       if (!time)
2042         time = buffer.st_ctime;
2043       if (time)
2044       {
2045         digest.Update((unsigned char *)&time, sizeof(time));
2046         return digest.Finalize();
2047       }
2048     }
2049     return "";
2050   }
2051
2052   std::string CVideoInfoScanner::GetRecursiveFastHash(const std::string &directory,
2053       const std::vector<std::string> &excludes) const
2054   {
2055     CFileItemList items;
2056     items.Add(CFileItemPtr(new CFileItem(directory, true)));
2057     CUtil::GetRecursiveDirsListing(directory, items, DIR_FLAG_NO_FILE_DIRS | DIR_FLAG_NO_FILE_INFO);
2058
2059     CDigest digest{CDigest::Type::MD5};
2060
2061     if (excludes.size())
2062       digest.Update(StringUtils::Join(excludes, "|"));
2063
2064     int64_t time = 0;
2065     for (int i=0; i < items.Size(); ++i)
2066     {
2067       int64_t stat_time = 0;
2068       struct __stat64 buffer;
2069       if (XFILE::CFile::Stat(items[i]->GetPath(), &buffer) == 0)
2070       {
2071         //! @todo some filesystems may return the mtime/ctime inline, in which case this is
2072         //! unnecessarily expensive. Consider supporting Stat() in our directory cache?
2073         stat_time = buffer.st_mtime ? buffer.st_mtime : buffer.st_ctime;
2074         time += stat_time;
2075       }
2076
2077       if (!stat_time)
2078         return "";
2079     }
2080
2081     if (time)
2082     {
2083       digest.Update((unsigned char *)&time, sizeof(time));
2084       return digest.Finalize();
2085     }
2086     return "";
2087   }
2088
2089   void CVideoInfoScanner::GetSeasonThumbs(const CVideoInfoTag &show,
2090       std::map<int, std::map<std::string, std::string>> &seasonArt, const std::vector<std::string> &artTypes, bool useLocal)
2091   {
2092     int artLevel = CServiceBroker::GetSettingsComponent()->GetSettings()->
2093       GetInt(CSettings::SETTING_VIDEOLIBRARY_ARTWORK_LEVEL);
2094     bool addAll = artLevel == CSettings::VIDEOLIBRARY_ARTWORK_LEVEL_ALL;
2095     bool exactName = artLevel == CSettings::VIDEOLIBRARY_ARTWORK_LEVEL_BASIC;
2096     if (useLocal)
2097     {
2098       // find the maximum number of seasons we have local thumbs for
2099       int maxSeasons = 0;
2100       CFileItemList items;
2101       std::string extensions = CServiceBroker::GetFileExtensionProvider().GetPictureExtensions();
2102       if (!show.m_strPath.empty())
2103       {
2104         CDirectory::GetDirectory(show.m_strPath, items, extensions,
2105                                  DIR_FLAG_NO_FILE_DIRS | DIR_FLAG_READ_CACHE |
2106                                      DIR_FLAG_NO_FILE_INFO);
2107       }
2108       extensions.erase(std::remove(extensions.begin(), extensions.end(), '.'), extensions.end());
2109       CRegExp reg;
2110       if (items.Size() && reg.RegComp("season([0-9]+)(-[a-z0-9]+)?\\.(" + extensions + ")"))
2111       {
2112         for (const auto& item : items)
2113         {
2114           std::string name = URIUtils::GetFileName(item->GetPath());
2115           if (reg.RegFind(name) > -1)
2116           {
2117             int season = atoi(reg.GetMatch(1).c_str());
2118             if (season > maxSeasons)
2119               maxSeasons = season;
2120           }
2121         }
2122       }
2123       for (int season = -1; season <= maxSeasons; season++)
2124       {
2125         // skip if we already have some art
2126         std::map<int, std::map<std::string, std::string>>::const_iterator it = seasonArt.find(season);
2127         if (it != seasonArt.end() && !it->second.empty())
2128           continue;
2129
2130         std::map<std::string, std::string> art;
2131         std::string basePath;
2132         if (season == -1)
2133           basePath = "season-all";
2134         else if (season == 0)
2135           basePath = "season-specials";
2136         else
2137           basePath = StringUtils::Format("season{:02}", season);
2138
2139         AddLocalItemArtwork(art, artTypes,
2140           URIUtils::AddFileToFolder(show.m_strPath, basePath),
2141           addAll, exactName);
2142
2143         seasonArt[season] = art;
2144       }
2145     }
2146     // add online art
2147     for (const auto& url : show.m_strPictureURL.GetUrls())
2148     {
2149       if (url.m_type != CScraperUrl::UrlType::Season)
2150         continue;
2151       std::string aspect = url.m_aspect;
2152       if (aspect.empty())
2153         aspect = "thumb";
2154       std::map<std::string, std::string>& art = seasonArt[url.m_season];
2155       if ((addAll || CVideoThumbLoader::IsArtTypeInWhitelist(aspect, artTypes, exactName)) &&
2156         art.find(aspect) == art.end())
2157       {
2158         std::string image = CScraperUrl::GetThumbUrl(url);
2159         if (!image.empty())
2160           art.insert(std::make_pair(aspect, image));
2161       }
2162     }
2163   }
2164
2165   void CVideoInfoScanner::FetchActorThumbs(std::vector<SActorInfo>& actors, const std::string& strPath)
2166   {
2167     CFileItemList items;
2168     // don't try to fetch anything local with plugin source
2169     if (!URIUtils::IsPlugin(strPath))
2170     {
2171       std::string actorsDir = URIUtils::AddFileToFolder(strPath, ".actors");
2172       if (CDirectory::Exists(actorsDir))
2173         CDirectory::GetDirectory(actorsDir, items, ".png|.jpg|.tbn", DIR_FLAG_NO_FILE_DIRS |
2174                                  DIR_FLAG_NO_FILE_INFO);
2175     }
2176     for (std::vector<SActorInfo>::iterator i = actors.begin(); i != actors.end(); ++i)
2177     {
2178       if (i->thumb.empty())
2179       {
2180         std::string thumbFile = i->strName;
2181         StringUtils::Replace(thumbFile, ' ', '_');
2182         for (int j = 0; j < items.Size(); j++)
2183         {
2184           std::string compare = URIUtils::GetFileName(items[j]->GetPath());
2185           URIUtils::RemoveExtension(compare);
2186           if (!items[j]->m_bIsFolder && compare == thumbFile)
2187           {
2188             i->thumb = items[j]->GetPath();
2189             break;
2190           }
2191         }
2192         if (i->thumb.empty() && !i->thumbUrl.GetFirstUrlByType().m_url.empty())
2193           i->thumb = CScraperUrl::GetThumbUrl(i->thumbUrl.GetFirstUrlByType());
2194         if (!i->thumb.empty())
2195           CServiceBroker::GetTextureCache()->BackgroundCacheImage(i->thumb);
2196       }
2197     }
2198   }
2199
2200   bool CVideoInfoScanner::DownloadFailed(CGUIDialogProgress* pDialog)
2201   {
2202     if (CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_bVideoScannerIgnoreErrors)
2203       return true;
2204
2205     if (pDialog)
2206     {
2207       HELPERS::ShowOKDialogText(CVariant{20448}, CVariant{20449});
2208       return false;
2209     }
2210     return HELPERS::ShowYesNoDialogText(CVariant{20448}, CVariant{20450}) ==
2211            DialogResponse::CHOICE_YES;
2212   }
2213
2214   bool CVideoInfoScanner::ProgressCancelled(CGUIDialogProgress* progress, int heading, const std::string &line1)
2215   {
2216     if (progress)
2217     {
2218       progress->SetHeading(CVariant{heading});
2219       progress->SetLine(0, CVariant{line1});
2220       progress->SetLine(2, CVariant{""});
2221       progress->Progress();
2222       return progress->IsCanceled();
2223     }
2224     return m_bStop;
2225   }
2226
2227   int CVideoInfoScanner::FindVideo(const std::string &title, int year, const ScraperPtr &scraper, CScraperUrl &url, CGUIDialogProgress *progress)
2228   {
2229     MOVIELIST movielist;
2230     CVideoInfoDownloader imdb(scraper);
2231     int returncode = imdb.FindMovie(title, year, movielist, progress);
2232     if (returncode < 0 || (returncode == 0 && (m_bStop || !DownloadFailed(progress))))
2233     { // scraper reported an error, or we had an error and user wants to cancel the scan
2234       m_bStop = true;
2235       return -1; // cancelled
2236     }
2237     if (returncode > 0 && movielist.size())
2238     {
2239       url = movielist[0];
2240       return 1;  // found a movie
2241     }
2242     return 0;    // didn't find anything
2243   }
2244
2245 }