xbmc/utils/RssReader.cpp

   1 /*
   2  *  Copyright (C) 2005-2018 Team Kodi
   3  *  This file is part of Kodi - https://kodi.tv
   4  *
   5  *  SPDX-License-Identifier: GPL-2.0-or-later
   6  *  See LICENSES/README.md for more information.
   7  */
   8
   9 #include "RssReader.h"
  10
  11 #include "CharsetConverter.h"
  12 #include "ServiceBroker.h"
  13 #include "URL.h"
  14 #include "filesystem/CurlFile.h"
  15 #include "filesystem/File.h"
  16 #include "guilib/GUIRSSControl.h"
  17 #include "guilib/LocalizeStrings.h"
  18 #include "log.h"
  19 #include "network/Network.h"
  20 #include "settings/AdvancedSettings.h"
  21 #include "settings/SettingsComponent.h"
  22 #include "threads/SystemClock.h"
  23 #include "utils/HTMLUtil.h"
  24 #include "utils/XTimeUtils.h"
  25
  26 #include <mutex>
  27
  28 #define RSS_COLOR_BODY      0
  29 #define RSS_COLOR_HEADLINE  1
  30 #define RSS_COLOR_CHANNEL   2
  31
  32 using namespace XFILE;
  33 using namespace std::chrono_literals;
  34
  35 //////////////////////////////////////////////////////////////////////
  36 // Construction/Destruction
  37 //////////////////////////////////////////////////////////////////////
  38
  39 CRssReader::CRssReader() : CThread("RSSReader")
  40 {
  41   m_pObserver = NULL;
  42   m_spacesBetweenFeeds = 0;
  43   m_bIsRunning = false;
  44   m_savedScrollPixelPos = 0;
  45   m_rtlText = false;
  46   m_requestRefresh = false;
  47 }
  48
  49 CRssReader::~CRssReader()
  50 {
  51   if (m_pObserver)
  52     m_pObserver->OnFeedRelease();
  53   StopThread();
  54   for (unsigned int i = 0; i < m_vecTimeStamps.size(); i++)
  55     delete m_vecTimeStamps[i];
  56 }
  57
  58 void CRssReader::Create(IRssObserver* aObserver, const std::vector<std::string>& aUrls, const std::vector<int> &times, int spacesBetweenFeeds, bool rtl)
  59 {
  60   std::unique_lock<CCriticalSection> lock(m_critical);
  61
  62   m_pObserver = aObserver;
  63   m_spacesBetweenFeeds = spacesBetweenFeeds;
  64   m_vecUrls = aUrls;
  65   m_strFeed.resize(aUrls.size());
  66   m_strColors.resize(aUrls.size());
  67   // set update times
  68   m_vecUpdateTimes = times;
  69   m_rtlText = rtl;
  70   m_requestRefresh = false;
  71
  72   // update each feed on creation
  73   for (unsigned int i = 0; i < m_vecUpdateTimes.size(); ++i)
  74   {
  75     AddToQueue(i);
  76     KODI::TIME::SystemTime* time = new KODI::TIME::SystemTime;
  77     KODI::TIME::GetLocalTime(time);
  78     m_vecTimeStamps.push_back(time);
  79   }
  80 }
  81
  82 void CRssReader::requestRefresh()
  83 {
  84   m_requestRefresh = true;
  85 }
  86
  87 void CRssReader::AddToQueue(int iAdd)
  88 {
  89   std::unique_lock<CCriticalSection> lock(m_critical);
  90   if (iAdd < (int)m_vecUrls.size())
  91     m_vecQueue.push_back(iAdd);
  92   if (!m_bIsRunning)
  93   {
  94     StopThread();
  95     m_bIsRunning = true;
  96     CThread::Create(false);
  97   }
  98 }
  99
 100 void CRssReader::OnExit()
 101 {
 102   m_bIsRunning = false;
 103 }
 104
 105 int CRssReader::GetQueueSize()
 106 {
 107   std::unique_lock<CCriticalSection> lock(m_critical);
 108   return m_vecQueue.size();
 109 }
 110
 111 void CRssReader::Process()
 112 {
 113   while (GetQueueSize())
 114   {
 115     std::unique_lock<CCriticalSection> lock(m_critical);
 116
 117     int iFeed = m_vecQueue.front();
 118     m_vecQueue.erase(m_vecQueue.begin());
 119
 120     m_strFeed[iFeed].clear();
 121     m_strColors[iFeed].clear();
 122
 123     CCurlFile http;
 124     http.SetUserAgent(CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_userAgent);
 125     http.SetTimeout(2);
 126     std::string strXML;
 127     std::string strUrl = m_vecUrls[iFeed];
 128     lock.unlock();
 129
 130     int nRetries = 3;
 131     CURL url(strUrl);
 132     std::string fileCharset;
 133
 134     // we wait for the network to come up
 135     if ((url.IsProtocol("http") || url.IsProtocol("https")) &&
 136         !CServiceBroker::GetNetwork().IsAvailable())
 137     {
 138       CLog::Log(LOGWARNING, "RSS: No network connection");
 139       strXML = "<rss><item><title>"+g_localizeStrings.Get(15301)+"</title></item></rss>";
 140     }
 141     else
 142     {
 143       XbmcThreads::EndTime<> timeout(15s);
 144       while (!m_bStop && nRetries > 0)
 145       {
 146         if (timeout.IsTimePast())
 147         {
 148           CLog::Log(LOGERROR, "Timeout while retrieving rss feed: {}", strUrl);
 149           break;
 150         }
 151         nRetries--;
 152
 153         if (!url.IsProtocol("http") && !url.IsProtocol("https"))
 154         {
 155           CFile file;
 156           std::vector<uint8_t> buffer;
 157           if (file.LoadFile(strUrl, buffer) > 0)
 158           {
 159             strXML.assign(reinterpret_cast<char*>(buffer.data()), buffer.size());
 160             break;
 161           }
 162         }
 163         else
 164         {
 165           if (http.Get(strUrl, strXML))
 166           {
 167             fileCharset = http.GetProperty(XFILE::FILE_PROPERTY_CONTENT_CHARSET);
 168             CLog::Log(LOGDEBUG, "Got rss feed: {}", strUrl);
 169             break;
 170           }
 171           else if (nRetries > 0)
 172             CThread::Sleep(5000ms); // Network problems? Retry, but not immediately.
 173           else
 174             CLog::Log(LOGERROR, "Unable to obtain rss feed: {}", strUrl);
 175         }
 176       }
 177       http.Cancel();
 178     }
 179     if (!strXML.empty() && m_pObserver)
 180     {
 181       // erase any <content:encoded> tags (also unsupported by tinyxml)
 182       size_t iStart = strXML.find("<content:encoded>");
 183       size_t iEnd = 0;
 184       while (iStart != std::string::npos)
 185       {
 186         // get <content:encoded> end position
 187         iEnd = strXML.find("</content:encoded>", iStart) + 18;
 188
 189         // erase the section
 190         strXML = strXML.erase(iStart, iEnd - iStart);
 191
 192         iStart = strXML.find("<content:encoded>");
 193       }
 194
 195       if (Parse(strXML, iFeed, fileCharset))
 196         CLog::Log(LOGDEBUG, "Parsed rss feed: {}", strUrl);
 197     }
 198   }
 199   UpdateObserver();
 200 }
 201
 202 void CRssReader::getFeed(vecText &text)
 203 {
 204   text.clear();
 205   // double the spaces at the start of the set
 206   for (int j = 0; j < m_spacesBetweenFeeds; j++)
 207     text.push_back(L' ');
 208   for (unsigned int i = 0; i < m_strFeed.size(); i++)
 209   {
 210     for (int j = 0; j < m_spacesBetweenFeeds; j++)
 211       text.push_back(L' ');
 212
 213     for (unsigned int j = 0; j < m_strFeed[i].size(); j++)
 214     {
 215       character_t letter = m_strFeed[i][j] | ((m_strColors[i][j] - 48) << 16);
 216       text.push_back(letter);
 217     }
 218   }
 219 }
 220
 221 void CRssReader::AddTag(const std::string &aString)
 222 {
 223   m_tagSet.push_back(aString);
 224 }
 225
 226 void CRssReader::AddString(std::wstring aString, int aColour, int iFeed)
 227 {
 228   if (m_rtlText)
 229     m_strFeed[iFeed] = aString + m_strFeed[iFeed];
 230   else
 231     m_strFeed[iFeed] += aString;
 232
 233   size_t nStringLength = aString.size();
 234
 235   for (size_t i = 0;i < nStringLength;i++)
 236     aString[i] = static_cast<char>(48 + aColour);
 237
 238   if (m_rtlText)
 239     m_strColors[iFeed] = aString + m_strColors[iFeed];
 240   else
 241     m_strColors[iFeed] += aString;
 242 }
 243
 244 void CRssReader::GetNewsItems(TiXmlElement* channelXmlNode, int iFeed)
 245 {
 246   HTML::CHTMLUtil html;
 247
 248   TiXmlElement * itemNode = channelXmlNode->FirstChildElement("item");
 249   std::map<std::string, std::wstring> mTagElements;
 250   typedef std::pair<std::string, std::wstring> StrPair;
 251   std::list<std::string>::iterator i;
 252
 253   // Add the title tag in if we didn't pass any tags in at all
 254   // Represents default behaviour before configurability
 255
 256   if (m_tagSet.empty())
 257     AddTag("title");
 258
 259   while (itemNode != nullptr)
 260   {
 261     TiXmlNode* childNode = itemNode->FirstChild();
 262     mTagElements.clear();
 263     while (childNode != nullptr)
 264     {
 265       std::string strName = childNode->ValueStr();
 266
 267       for (i = m_tagSet.begin(); i != m_tagSet.end(); ++i)
 268       {
 269         if (!childNode->NoChildren() && *i == strName)
 270         {
 271           std::string htmlText = childNode->FirstChild()->ValueStr();
 272
 273           // This usually happens in right-to-left languages where they want to
 274           // specify in the RSS body that the text should be RTL.
 275           // <title>
 276           //  <div dir="RTL">��� ����: ���� �� �����</div>
 277           // </title>
 278           if (htmlText == "div" || htmlText == "span")
 279             htmlText = childNode->FirstChild()->FirstChild()->ValueStr();
 280
 281           std::wstring unicodeText, unicodeText2;
 282
 283           g_charsetConverter.utf8ToW(htmlText, unicodeText2, m_rtlText);
 284           html.ConvertHTMLToW(unicodeText2, unicodeText);
 285
 286           mTagElements.insert(StrPair(*i, unicodeText));
 287         }
 288       }
 289       childNode = childNode->NextSibling();
 290     }
 291
 292     int rsscolour = RSS_COLOR_HEADLINE;
 293     for (i = m_tagSet.begin(); i != m_tagSet.end(); ++i)
 294     {
 295       std::map<std::string, std::wstring>::iterator j = mTagElements.find(*i);
 296
 297       if (j == mTagElements.end())
 298         continue;
 299
 300       std::wstring& text = j->second;
 301       AddString(text, rsscolour, iFeed);
 302       rsscolour = RSS_COLOR_BODY;
 303       text = L" - ";
 304       AddString(text, rsscolour, iFeed);
 305     }
 306     itemNode = itemNode->NextSiblingElement("item");
 307   }
 308 }
 309
 310 bool CRssReader::Parse(const std::string& data, int iFeed, const std::string& charset)
 311 {
 312   m_xml.Clear();
 313   m_xml.Parse(data, charset);
 314
 315   CLog::Log(LOGDEBUG, "RSS feed encoding: {}", m_xml.GetUsedCharset());
 316
 317   return Parse(iFeed);
 318 }
 319
 320 bool CRssReader::Parse(int iFeed)
 321 {
 322   TiXmlElement* rootXmlNode = m_xml.RootElement();
 323
 324   if (!rootXmlNode)
 325     return false;
 326
 327   TiXmlElement* rssXmlNode = NULL;
 328
 329   std::string strValue = rootXmlNode->ValueStr();
 330   if (strValue.find("rss") != std::string::npos ||
 331       strValue.find("rdf") != std::string::npos)
 332     rssXmlNode = rootXmlNode;
 333   else
 334   {
 335     // Unable to find root <rss> or <rdf> node
 336     return false;
 337   }
 338
 339   TiXmlElement* channelXmlNode = rssXmlNode->FirstChildElement("channel");
 340   if (channelXmlNode)
 341   {
 342     TiXmlElement* titleNode = channelXmlNode->FirstChildElement("title");
 343     if (titleNode && !titleNode->NoChildren())
 344     {
 345       std::string strChannel = titleNode->FirstChild()->Value();
 346       std::wstring strChannelUnicode;
 347       g_charsetConverter.utf8ToW(strChannel, strChannelUnicode, m_rtlText);
 348       AddString(strChannelUnicode, RSS_COLOR_CHANNEL, iFeed);
 349
 350       AddString(L":", RSS_COLOR_CHANNEL, iFeed);
 351       AddString(L" ", RSS_COLOR_CHANNEL, iFeed);
 352     }
 353
 354     GetNewsItems(channelXmlNode,iFeed);
 355   }
 356
 357   GetNewsItems(rssXmlNode,iFeed);
 358
 359   // avoid trailing ' - '
 360   if (m_strFeed[iFeed].size() > 3 && m_strFeed[iFeed].substr(m_strFeed[iFeed].size() - 3) == L" - ")
 361   {
 362     if (m_rtlText)
 363     {
 364       m_strFeed[iFeed].erase(0, 3);
 365       m_strColors[iFeed].erase(0, 3);
 366     }
 367     else
 368     {
 369       m_strFeed[iFeed].erase(m_strFeed[iFeed].length() - 3);
 370       m_strColors[iFeed].erase(m_strColors[iFeed].length() - 3);
 371     }
 372   }
 373   return true;
 374 }
 375
 376 void CRssReader::SetObserver(IRssObserver *observer)
 377 {
 378   m_pObserver = observer;
 379 }
 380
 381 void CRssReader::UpdateObserver()
 382 {
 383   if (!m_pObserver)
 384     return;
 385
 386   vecText feed;
 387   getFeed(feed);
 388   if (!feed.empty())
 389   {
 390     std::unique_lock<CCriticalSection> lock(CServiceBroker::GetWinSystem()->GetGfxContext());
 391     if (m_pObserver) // need to check again when locked to make sure observer wasnt removed
 392       m_pObserver->OnFeedUpdate(feed);
 393   }
 394 }
 395
 396 void CRssReader::CheckForUpdates()
 397 {
 398   KODI::TIME::SystemTime time;
 399   KODI::TIME::GetLocalTime(&time);
 400
 401   for (unsigned int i = 0;i < m_vecUpdateTimes.size(); ++i )
 402   {
 403     if (m_requestRefresh || ((time.day * 24 * 60) + (time.hour * 60) + time.minute) -
 404                                     ((m_vecTimeStamps[i]->day * 24 * 60) +
 405                                      (m_vecTimeStamps[i]->hour * 60) + m_vecTimeStamps[i]->minute) >
 406                                 m_vecUpdateTimes[i])
 407     {
 408       CLog::Log(LOGDEBUG, "Updating RSS");
 409       KODI::TIME::GetLocalTime(m_vecTimeStamps[i]);
 410       AddToQueue(i);
 411     }
 412   }
 413
 414   m_requestRefresh = false;
 415 }