xbmc/filesystem/RSSDirectory.cpp

   1 /*
   2  *  Copyright (C) 2005-2018 Team Kodi
   3  *  This file is part of Kodi - https://kodi.tv
   4  *
   5  *  SPDX-License-Identifier: GPL-2.0-or-later
   6  *  See LICENSES/README.md for more information.
   7  */
   8
   9 #include "RSSDirectory.h"
  10
  11 #include "CurlFile.h"
  12 #include "FileItem.h"
  13 #include "ServiceBroker.h"
  14 #include "URL.h"
  15 #include "settings/AdvancedSettings.h"
  16 #include "settings/Settings.h"
  17 #include "settings/SettingsComponent.h"
  18 #include "utils/FileExtensionProvider.h"
  19 #include "utils/HTMLUtil.h"
  20 #include "utils/StringUtils.h"
  21 #include "utils/URIUtils.h"
  22 #include "utils/XBMCTinyXML.h"
  23 #include "utils/XMLUtils.h"
  24 #include "utils/log.h"
  25 #include "video/VideoInfoTag.h"
  26
  27 #include <climits>
  28 #include <mutex>
  29 #include <utility>
  30
  31 using namespace XFILE;
  32 using namespace MUSIC_INFO;
  33
  34 namespace {
  35
  36   struct SResource
  37   {
  38     std::string tag;
  39     std::string path;
  40     std::string mime;
  41     std::string lang;
  42     int        width = 0;
  43     int        height = 0;
  44     int        bitrate = 0;
  45     int        duration = 0;
  46     int64_t    size = 0;
  47   };
  48
  49   typedef std::vector<SResource> SResources;
  50
  51 }
  52
  53 std::map<std::string,CDateTime> CRSSDirectory::m_cache;
  54 CCriticalSection CRSSDirectory::m_section;
  55
  56 CRSSDirectory::CRSSDirectory() = default;
  57
  58 CRSSDirectory::~CRSSDirectory() = default;
  59
  60 bool CRSSDirectory::ContainsFiles(const CURL& url)
  61 {
  62   CFileItemList items;
  63   if(!GetDirectory(url, items))
  64     return false;
  65
  66   return items.Size() > 0;
  67 }
  68
  69 static bool IsPathToMedia(const std::string& strPath )
  70 {
  71   return URIUtils::HasExtension(strPath,
  72                                 CServiceBroker::GetFileExtensionProvider().GetVideoExtensions() + '|' +
  73                                 CServiceBroker::GetFileExtensionProvider().GetMusicExtensions() + '|' +
  74                                 CServiceBroker::GetFileExtensionProvider().GetPictureExtensions());
  75 }
  76
  77 static bool IsPathToThumbnail(const std::string& strPath )
  78 {
  79   // Currently just check if this is an image, maybe we will add some
  80   // other checks later
  81   return URIUtils::HasExtension(strPath,
  82                                 CServiceBroker::GetFileExtensionProvider().GetPictureExtensions());
  83 }
  84
  85 static time_t ParseDate(const std::string & strDate)
  86 {
  87   struct tm pubDate = {};
  88   //! @todo Handle time zone
  89   strptime(strDate.c_str(), "%a, %d %b %Y %H:%M:%S", &pubDate);
  90   // Check the difference between the time of last check and time of the item
  91   return mktime(&pubDate);
  92 }
  93 static void ParseItem(CFileItem* item, SResources& resources, TiXmlElement* root, const std::string& path);
  94
  95 static std::string GetValue(TiXmlElement *element)
  96 {
  97   if (element && !element->NoChildren())
  98     return element->FirstChild()->ValueStr();
  99   return "";
 100 }
 101
 102 static void ParseItemMRSS(CFileItem* item, SResources& resources, TiXmlElement* item_child, const std::string& name, const std::string& xmlns, const std::string& path)
 103 {
 104   CVideoInfoTag* vtag = item->GetVideoInfoTag();
 105   std::string text = GetValue(item_child);
 106
 107   if(name == "content")
 108   {
 109     SResource res;
 110     res.tag = "media:content";
 111     res.mime    = XMLUtils::GetAttribute(item_child, "type");
 112     res.path    = XMLUtils::GetAttribute(item_child, "url");
 113     item_child->Attribute("width", &res.width);
 114     item_child->Attribute("height", &res.height);
 115     item_child->Attribute("bitrate", &res.bitrate);
 116     item_child->Attribute("duration", &res.duration);
 117     if(item_child->Attribute("fileSize"))
 118       res.size = std::atoll(item_child->Attribute("fileSize"));
 119
 120     resources.push_back(res);
 121     ParseItem(item, resources, item_child, path);
 122   }
 123   else if(name == "group")
 124   {
 125     ParseItem(item, resources, item_child, path);
 126   }
 127   else if(name == "thumbnail")
 128   {
 129     if(!item_child->NoChildren() && IsPathToThumbnail(item_child->FirstChild()->ValueStr()))
 130       item->SetArt("thumb", item_child->FirstChild()->ValueStr());
 131     else
 132     {
 133       const char * url = item_child->Attribute("url");
 134       if(url && IsPathToThumbnail(url))
 135         item->SetArt("thumb", url);
 136     }
 137   }
 138   else if (name == "title")
 139   {
 140     if(text.empty())
 141       return;
 142
 143     if(text.length() > item->m_strTitle.length())
 144       item->m_strTitle = text;
 145   }
 146   else if(name == "description")
 147   {
 148     if(text.empty())
 149       return;
 150
 151     std::string description = text;
 152     if(XMLUtils::GetAttribute(item_child, "type") == "html")
 153       HTML::CHTMLUtil::RemoveTags(description);
 154     item->SetProperty("description", description);
 155   }
 156   else if(name == "category")
 157   {
 158     if(text.empty())
 159       return;
 160
 161     std::string scheme = XMLUtils::GetAttribute(item_child, "scheme");
 162
 163     /* okey this is silly, boxee what did you think?? */
 164     if     (scheme == "urn:boxee:genre")
 165       vtag->m_genre.push_back(text);
 166     else if(scheme == "urn:boxee:title-type")
 167     {
 168       if     (text == "tv")
 169         item->SetProperty("boxee:istvshow", true);
 170       else if(text == "movie")
 171         item->SetProperty("boxee:ismovie", true);
 172     }
 173     else if(scheme == "urn:boxee:episode")
 174       vtag->m_iEpisode = atoi(text.c_str());
 175     else if(scheme == "urn:boxee:season")
 176       vtag->m_iSeason  = atoi(text.c_str());
 177     else if(scheme == "urn:boxee:show-title")
 178       vtag->m_strShowTitle = text.c_str();
 179     else if(scheme == "urn:boxee:view-count")
 180       vtag->SetPlayCount(atoi(text.c_str()));
 181     else if(scheme == "urn:boxee:source")
 182       item->SetProperty("boxee:provider_source", text);
 183     else
 184       vtag->m_genre = StringUtils::Split(text, CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_videoItemSeparator);
 185   }
 186   else if(name == "rating")
 187   {
 188     std::string scheme = XMLUtils::GetAttribute(item_child, "scheme");
 189     if(scheme == "urn:user")
 190       vtag->SetRating((float)atof(text.c_str()));
 191     else
 192       vtag->m_strMPAARating = text;
 193   }
 194   else if(name == "credit")
 195   {
 196     std::string role = XMLUtils::GetAttribute(item_child, "role");
 197     if     (role == "director")
 198       vtag->m_director.push_back(text);
 199     else if(role == "author"
 200          || role == "writer")
 201       vtag->m_writingCredits.push_back(text);
 202     else if(role == "actor")
 203     {
 204       SActorInfo actor;
 205       actor.strName = text;
 206       vtag->m_cast.push_back(actor);
 207     }
 208   }
 209   else if(name == "copyright")
 210     vtag->m_studio = StringUtils::Split(text, CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_videoItemSeparator);
 211   else if(name == "keywords")
 212     item->SetProperty("keywords", text);
 213
 214 }
 215
 216 static void ParseItemItunes(CFileItem* item, SResources& resources, TiXmlElement* item_child, const std::string& name, const std::string& xmlns, const std::string& path)
 217 {
 218   CVideoInfoTag* vtag = item->GetVideoInfoTag();
 219   std::string text = GetValue(item_child);
 220
 221   if(name == "image")
 222   {
 223     const char * url = item_child->Attribute("href");
 224     if(url)
 225       item->SetArt("thumb", url);
 226     else
 227       item->SetArt("thumb", text);
 228   }
 229   else if(name == "summary")
 230     vtag->m_strPlot = text;
 231   else if(name == "subtitle")
 232     vtag->m_strPlotOutline = text;
 233   else if(name == "author")
 234     vtag->m_writingCredits.push_back(text);
 235   else if(name == "duration")
 236     vtag->SetDuration(StringUtils::TimeStringToSeconds(text));
 237   else if(name == "keywords")
 238     item->SetProperty("keywords", text);
 239 }
 240
 241 static void ParseItemRSS(CFileItem* item, SResources& resources, TiXmlElement* item_child, const std::string& name, const std::string& xmlns, const std::string& path)
 242 {
 243   std::string text = GetValue(item_child);
 244   if (name == "title")
 245   {
 246     if(text.length() > item->m_strTitle.length())
 247       item->m_strTitle = text;
 248   }
 249   else if (name == "pubDate")
 250   {
 251     CDateTime pubDate(ParseDate(text));
 252     item->m_dateTime = pubDate;
 253   }
 254   else if (name == "link")
 255   {
 256     SResource res;
 257     res.tag  = "rss:link";
 258     res.path = text;
 259     resources.push_back(res);
 260   }
 261   else if(name == "enclosure")
 262   {
 263     const char * len  = item_child->Attribute("length");
 264
 265     SResource res;
 266     res.tag = "rss:enclosure";
 267     res.path = XMLUtils::GetAttribute(item_child, "url");
 268     res.mime = XMLUtils::GetAttribute(item_child, "type");
 269     if(len)
 270       res.size = std::atoll(len);
 271
 272     resources.push_back(res);
 273   }
 274   else if(name == "description")
 275   {
 276     std::string description = text;
 277     HTML::CHTMLUtil::RemoveTags(description);
 278     item->SetProperty("description", description);
 279   }
 280   else if(name == "guid")
 281   {
 282     if(IsPathToMedia(text))
 283     {
 284       SResource res;
 285       res.tag  = "rss:guid";
 286       res.path = text;
 287       resources.push_back(res);
 288     }
 289   }
 290 }
 291
 292 static void ParseItemVoddler(CFileItem* item, SResources& resources, TiXmlElement* element, const std::string& name, const std::string& xmlns, const std::string& path)
 293 {
 294   CVideoInfoTag* vtag = item->GetVideoInfoTag();
 295   std::string text = GetValue(element);
 296
 297   if(name == "trailer")
 298   {
 299     vtag->m_strTrailer = text;
 300
 301     SResource res;
 302     res.tag  = "voddler:trailer";
 303     res.mime = XMLUtils::GetAttribute(element, "type");
 304     res.path = text;
 305     resources.push_back(res);
 306   }
 307   else if(name == "year")
 308     vtag->SetYear(atoi(text.c_str()));
 309   else if(name == "rating")
 310     vtag->SetRating((float)atof(text.c_str()));
 311   else if(name == "tagline")
 312     vtag->m_strTagLine = text;
 313   else if(name == "posterwall")
 314   {
 315     const char* url = element->Attribute("url");
 316     if(url)
 317       item->SetArt("fanart", url);
 318     else if(IsPathToThumbnail(text))
 319       item->SetArt("fanart", text);
 320   }
 321 }
 322
 323 static void ParseItemBoxee(CFileItem* item, SResources& resources, TiXmlElement* element, const std::string& name, const std::string& xmlns, const std::string& path)
 324 {
 325   CVideoInfoTag* vtag = item->GetVideoInfoTag();
 326   std::string text = GetValue(element);
 327
 328   if     (name == "image")
 329     item->SetArt("thumb", text);
 330   else if(name == "user_agent")
 331     item->SetProperty("boxee:user_agent", text);
 332   else if(name == "content_type")
 333     item->SetMimeType(text);
 334   else if(name == "runtime")
 335     vtag->SetDuration(StringUtils::TimeStringToSeconds(text));
 336   else if(name == "episode")
 337     vtag->m_iEpisode = atoi(text.c_str());
 338   else if(name == "season")
 339     vtag->m_iSeason = atoi(text.c_str());
 340   else if(name == "view-count")
 341     vtag->SetPlayCount(atoi(text.c_str()));
 342   else if(name == "tv-show-title")
 343     vtag->m_strShowTitle = text;
 344   else if(name == "release-date")
 345     item->SetProperty("boxee:releasedate", text);
 346 }
 347
 348 static void ParseItemZink(CFileItem* item, SResources& resources, TiXmlElement* element, const std::string& name, const std::string& xmlns, const std::string& path)
 349 {
 350   CVideoInfoTag* vtag = item->GetVideoInfoTag();
 351   std::string text = GetValue(element);
 352   if     (name == "episode")
 353     vtag->m_iEpisode = atoi(text.c_str());
 354   else if(name == "season")
 355     vtag->m_iSeason = atoi(text.c_str());
 356   else if(name == "views")
 357     vtag->SetPlayCount(atoi(text.c_str()));
 358   else if(name == "airdate")
 359     vtag->m_firstAired.SetFromDateString(text);
 360   else if(name == "userrating")
 361     vtag->SetRating((float)atof(text.c_str()));
 362   else if(name == "duration")
 363     vtag->SetDuration(atoi(text.c_str()));
 364   else if(name == "durationstr")
 365     vtag->SetDuration(StringUtils::TimeStringToSeconds(text));
 366 }
 367
 368 static void ParseItemSVT(CFileItem* item, SResources& resources, TiXmlElement* element, const std::string& name, const std::string& xmlns, const std::string& path)
 369 {
 370   std::string text = GetValue(element);
 371   if     (name == "xmllink")
 372   {
 373     SResource res;
 374     res.tag  = "svtplay:xmllink";
 375     res.path = text;
 376     res.mime = "application/rss+xml";
 377     resources.push_back(res);
 378   }
 379   else if (name == "broadcasts")
 380   {
 381     CURL url(path);
 382     if(StringUtils::StartsWith(url.GetFileName(), "v1/"))
 383     {
 384       SResource res;
 385       res.tag  = "svtplay:broadcasts";
 386       res.path = url.GetWithoutFilename() + "v1/video/list/" + text;
 387       res.mime = "application/rss+xml";
 388       resources.push_back(res);
 389     }
 390   }
 391 }
 392
 393 static void ParseItem(CFileItem* item, SResources& resources, TiXmlElement* root, const std::string& path)
 394 {
 395   for (TiXmlElement* child = root->FirstChildElement(); child; child = child->NextSiblingElement())
 396   {
 397     std::string name = child->Value();
 398     std::string xmlns;
 399     size_t pos = name.find(':');
 400     if(pos != std::string::npos)
 401     {
 402       xmlns = name.substr(0, pos);
 403       name.erase(0, pos+1);
 404     }
 405
 406     if      (xmlns == "media")
 407       ParseItemMRSS   (item, resources, child, name, xmlns, path);
 408     else if (xmlns == "itunes")
 409       ParseItemItunes (item, resources, child, name, xmlns, path);
 410     else if (xmlns == "voddler")
 411       ParseItemVoddler(item, resources, child, name, xmlns, path);
 412     else if (xmlns == "boxee")
 413       ParseItemBoxee  (item, resources, child, name, xmlns, path);
 414     else if (xmlns == "zn")
 415       ParseItemZink   (item, resources, child, name, xmlns, path);
 416     else if (xmlns == "svtplay")
 417       ParseItemSVT    (item, resources, child, name, xmlns, path);
 418     else
 419       ParseItemRSS    (item, resources, child, name, xmlns, path);
 420   }
 421 }
 422
 423 static bool FindMime(const SResources& resources, const std::string& mime)
 424 {
 425   for (const auto& it : resources)
 426   {
 427     if (StringUtils::StartsWithNoCase(it.mime, mime))
 428       return true;
 429   }
 430   return false;
 431 }
 432
 433 static void ParseItem(CFileItem* item, TiXmlElement* root, const std::string& path)
 434 {
 435   SResources resources;
 436   ParseItem(item, resources, root, path);
 437
 438   const char* prio[] = { "media:content", "voddler:trailer", "rss:enclosure", "svtplay:broadcasts", "svtplay:xmllink", "rss:link", "rss:guid", NULL };
 439
 440   std::string mime;
 441   if     (FindMime(resources, "video/"))
 442     mime = "video/";
 443   else if(FindMime(resources, "audio/"))
 444     mime = "audio/";
 445   else if(FindMime(resources, "application/rss"))
 446     mime = "application/rss";
 447   else if(FindMime(resources, "image/"))
 448     mime = "image/";
 449
 450   int maxrate = CServiceBroker::GetSettingsComponent()->GetSettings()->GetInt(CSettings::SETTING_NETWORK_BANDWIDTH);
 451   if(maxrate == 0)
 452     maxrate = INT_MAX;
 453
 454   SResources::iterator best = resources.end();
 455   for(const char** type = prio; *type && best == resources.end(); type++)
 456   {
 457     for (SResources::iterator it = resources.begin(); it != resources.end(); ++it)
 458     {
 459       if(!StringUtils::StartsWith(it->mime, mime))
 460         continue;
 461
 462       if(it->tag == *type)
 463       {
 464         if(best == resources.end())
 465         {
 466           best = it;
 467           continue;
 468         }
 469
 470         if(it->bitrate == best->bitrate)
 471         {
 472           if(it->width*it->height > best->width*best->height)
 473             best = it;
 474           continue;
 475         }
 476
 477         if(it->bitrate > maxrate)
 478         {
 479           if(it->bitrate < best->bitrate)
 480             best = it;
 481           continue;
 482         }
 483
 484         if(it->bitrate > best->bitrate)
 485         {
 486           best = it;
 487           continue;
 488         }
 489       }
 490     }
 491   }
 492
 493   if(best != resources.end())
 494   {
 495     item->SetMimeType(best->mime);
 496     item->SetPath(best->path);
 497     item->m_dwSize  = best->size;
 498
 499     if(best->duration)
 500       item->SetProperty("duration", StringUtils::SecondsToTimeString(best->duration));
 501
 502     /* handling of mimetypes fo directories are sub optimal at best */
 503     if(best->mime == "application/rss+xml" && StringUtils::StartsWithNoCase(item->GetPath(), "http://"))
 504       item->SetPath("rss://" + item->GetPath().substr(7));
 505
 506     if(best->mime == "application/rss+xml" && StringUtils::StartsWithNoCase(item->GetPath(), "https://"))
 507       item->SetPath("rsss://" + item->GetPath().substr(8));
 508
 509     if(StringUtils::StartsWithNoCase(item->GetPath(), "rss://")
 510       || StringUtils::StartsWithNoCase(item->GetPath(), "rsss://"))
 511       item->m_bIsFolder = true;
 512     else
 513       item->m_bIsFolder = false;
 514   }
 515
 516   if(!item->m_strTitle.empty())
 517     item->SetLabel(item->m_strTitle);
 518
 519   if(item->HasVideoInfoTag())
 520   {
 521     CVideoInfoTag* vtag = item->GetVideoInfoTag();
 522
 523     if(item->HasProperty("duration")    && !vtag->GetDuration())
 524       vtag->SetDuration(StringUtils::TimeStringToSeconds(item->GetProperty("duration").asString()));
 525
 526     if(item->HasProperty("description") && vtag->m_strPlot.empty())
 527       vtag->m_strPlot = item->GetProperty("description").asString();
 528
 529     if(vtag->m_strPlotOutline.empty() && !vtag->m_strPlot.empty())
 530     {
 531       size_t pos = vtag->m_strPlot.find('\n');
 532       if(pos != std::string::npos)
 533         vtag->m_strPlotOutline = vtag->m_strPlot.substr(0, pos);
 534       else
 535         vtag->m_strPlotOutline = vtag->m_strPlot;
 536     }
 537
 538     if(!vtag->GetDuration())
 539       item->SetLabel2(StringUtils::SecondsToTimeString(vtag->GetDuration()));
 540   }
 541 }
 542
 543 bool CRSSDirectory::GetDirectory(const CURL& url, CFileItemList &items)
 544 {
 545   const std::string pathToUrl(url.Get());
 546   std::string strPath(pathToUrl);
 547   URIUtils::RemoveSlashAtEnd(strPath);
 548   std::map<std::string,CDateTime>::iterator it;
 549   items.SetPath(strPath);
 550   std::unique_lock<CCriticalSection> lock(m_section);
 551   if ((it=m_cache.find(strPath)) != m_cache.end())
 552   {
 553     if (it->second > CDateTime::GetCurrentDateTime() &&
 554         items.Load())
 555       return true;
 556     m_cache.erase(it);
 557   }
 558   lock.unlock();
 559
 560   CXBMCTinyXML xmlDoc;
 561   if (!xmlDoc.LoadFile(strPath))
 562   {
 563     CLog::Log(LOGERROR, "failed to load xml from <{}>. error: <{}>", strPath, xmlDoc.ErrorId());
 564     return false;
 565   }
 566   if (xmlDoc.Error())
 567   {
 568     CLog::Log(LOGERROR, "error parsing xml doc from <{}>. error: <{}>", strPath, xmlDoc.ErrorId());
 569     return false;
 570   }
 571
 572   TiXmlElement* rssXmlNode = xmlDoc.RootElement();
 573
 574   if (!rssXmlNode)
 575     return false;
 576
 577   TiXmlHandle docHandle( &xmlDoc );
 578   TiXmlElement* channelXmlNode = docHandle.FirstChild( "rss" ).FirstChild( "channel" ).Element();
 579   if (channelXmlNode)
 580     ParseItem(&items, channelXmlNode, pathToUrl);
 581   else
 582     return false;
 583
 584   TiXmlElement* child = NULL;
 585   for (child = channelXmlNode->FirstChildElement("item"); child; child = child->NextSiblingElement())
 586   {
 587     // Create new item,
 588     CFileItemPtr item(new CFileItem());
 589     ParseItem(item.get(), child, pathToUrl);
 590
 591     item->SetProperty("isrss", "1");
 592     // Use channel image if item doesn't have one
 593     if (!item->HasArt("thumb") && items.HasArt("thumb"))
 594       item->SetArt("thumb", items.GetArt("thumb"));
 595
 596     if (!item->GetPath().empty())
 597       items.Add(item);
 598   }
 599
 600   items.AddSortMethod(SortByNone   , 231, LABEL_MASKS("%L", "%D", "%L", ""));    // FileName, Duration | Foldername, empty
 601   items.AddSortMethod(SortByLabel  , 551, LABEL_MASKS("%L", "%D", "%L", ""));    // FileName, Duration | Foldername, empty
 602   items.AddSortMethod(SortBySize   , 553, LABEL_MASKS("%L", "%I", "%L", "%I"));  // FileName, Size | Foldername, Size
 603   items.AddSortMethod(SortByDate   , 552, LABEL_MASKS("%L", "%J", "%L", "%J"));  // FileName, Date | Foldername, Date
 604
 605   CDateTime time = CDateTime::GetCurrentDateTime();
 606   int mins = 60;
 607   TiXmlElement* ttl = docHandle.FirstChild("rss").FirstChild("ttl").Element();
 608   if (ttl)
 609     mins = strtol(ttl->FirstChild()->Value(),NULL,10);
 610   time += CDateTimeSpan(0,0,mins,0);
 611   items.SetPath(strPath);
 612   items.Save();
 613   std::unique_lock<CCriticalSection> lock2(m_section);
 614   m_cache.insert(make_pair(strPath,time));
 615
 616   return true;
 617 }
 618
 619 bool CRSSDirectory::Exists(const CURL& url)
 620 {
 621   CCurlFile rss;
 622   return rss.Exists(url);
 623 }