src/util.cpp

   1 /*
   2   Copyright 2013-2015 Mats Sjöberg
   3
   4   This file is part of the Pumpa programme.
   5
   6   Pumpa is free software: you can redistribute it and/or modify it
   7   under the terms of the GNU General Public License as published by
   8   the Free Software Foundation, either version 3 of the License, or
   9   (at your option) any later version.
  10
  11   Pumpa is distributed in the hope that it will be useful, but WITHOUT
  12   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
  14   License for more details.
  15
  16   You should have received a copy of the GNU General Public License
  17   along with Pumpa.  If not, see <http://www.gnu.org/licenses/>.
  18 */
  19
  20 #include "util.h"
  21
  22 #include <QTextStream>
  23 #include <QStringList>
  24 #include <QRegExp>
  25 #include <QObject>
  26 #include <QDebug>
  27 #include <QFile>
  28
  29 #ifdef DEBUG_MEMORY
  30 #include <sys/resource.h>
  31 #include <unistd.h>
  32 #endif
  33
  34 #include "sundown/markdown.h"
  35 #include "sundown/html.h"
  36 #include "sundown/buffer.h"
  37
  38 #ifdef USE_TIDY_TIDY
  39 #include <tidy/tidy.h>
  40 #include <tidy/buffio.h>
  41 #endif
  42
  43 #ifdef USE_TIDY
  44 #include <tidy.h>
  45 #include <buffio.h>
  46 #endif
  47
  48 //------------------------------------------------------------------------------
  49
  50 QString markDown(QString text) {
  51   struct sd_callbacks callbacks;
  52   struct html_renderopt options;
  53   sdhtml_renderer(&callbacks, &options, 0);
  54
  55   struct sd_markdown* markdown = sd_markdown_new(0, 16, &callbacks, &options);
  56
  57   struct buf* ob = bufnew(64);
  58   // QByteArray ba = text.toLocal8Bit();
  59   QByteArray ba = text.toUtf8();
  60
  61   sd_markdown_render(ob, (const unsigned char*)ba.constData(), ba.size(),
  62                      markdown);
  63   sd_markdown_free(markdown);
  64
  65   // QString ret = QString::fromLocal8Bit((char*)ob->data, ob->size);
  66   QString ret = QString::fromUtf8((char*)ob->data, ob->size);
  67   bufrelease(ob);
  68
  69   return ret.trimmed();
  70 }
  71
  72 //------------------------------------------------------------------------------
  73
  74 QString siteUrlFixer(QString url, bool useSsl) {
  75   if (!url.startsWith("http://") && !url.startsWith("https://"))
  76     url = (useSsl ? "https://" : "http://") + url;
  77
  78   if (url.endsWith('/'))
  79     url.chop(1);
  80
  81   return url;
  82 }
  83
  84 //------------------------------------------------------------------------------
  85
  86 QString linkifyUrls(QString text, bool useMarkdown) {
  87   QRegExp rx(QString("(^|\\s)%1([\\s\\.\\,\\!\\?\\)]|$)").arg(URL_REGEX_STRICT));
  88
  89   QStringList lines = text.split('\n');
  90
  91   for (int i=0; i<lines.size(); ++i) {
  92     QString line = lines.at(i);
  93     int pos = 0;
  94     while ((pos = rx.indexIn(line, pos)) != -1) {
  95       int len = rx.matchedLength();
  96       QString before = rx.cap(1);
  97       QString url = rx.cap(2);
  98       QString after = rx.cap(3);
  99
 100       QString newText = useMarkdown ?
 101         QString("%1<%2>%3").arg(before).arg(url).arg(after) :
 102         QString("%1<a href=\"%2\">%2</a>%3").arg(before).arg(url).arg(after);
 103
 104       line.replace(pos, len, newText);
 105       pos += newText.count();
 106     }
 107     lines[i] = line;
 108   }
 109   return lines.join("\n");
 110 }
 111
 112 //------------------------------------------------------------------------------
 113
 114 QString changePairedTags(QString text,
 115                          QString begin, QString end,
 116                          QString newBegin, QString newEnd,
 117                          QString nogoItems) {
 118   QRegExp rx(QString(MD_PAIR_REGEX).arg(begin).arg(end).arg(nogoItems));
 119   int pos = 0;
 120   while ((pos = rx.indexIn(text, pos)) != -1) {
 121     int len = rx.matchedLength();
 122     QString newText = QString(newBegin + rx.cap(1) + newEnd);
 123     text.replace(pos, len, newText);
 124     pos += newText.count();
 125   }
 126   return text;
 127 }
 128
 129 //------------------------------------------------------------------------------
 130
 131 QString siteUrlToAccountId(QString username, QString url) {
 132   if (url.startsWith("http://"))
 133     url.remove(0, 7);
 134   if (url.startsWith("https://"))
 135     url.remove(0, 8);
 136
 137   if (url.endsWith('/'))
 138     url.chop(1);
 139
 140   return username + "@" + url;
 141 }
 142
 143 //------------------------------------------------------------------------------
 144
 145 QString relativeFuzzyTime(QDateTime sTime, bool longTime) {
 146   QString dateStr = sTime.toString("ddd d MMMM yyyy");
 147
 148   int secs = sTime.secsTo(QDateTime::currentDateTime().toUTC());
 149   if (secs < 0)
 150     secs = 0;
 151   float t = (float)secs;
 152   t /= 60; int mins = qRound(t);
 153   t /= 60; int hours = qRound(t);
 154   t /= 24; int days = qRound(t);
 155   t /= 7;  int weeks = qRound(t);
 156   t /= 4.33; int months = qRound(t);
 157   t /= 12; int years = qRound(t);
 158
 159   if (secs < 60) {
 160     dateStr = QObject::tr("a few seconds ago");
 161   } else if (mins == 1) {
 162     dateStr = QObject::tr("one minute ago");
 163   } else if (hours < 1) {
 164     dateStr = QObject::tr("%n minutes ago", 0, mins);
 165   } else if (hours == 1) {
 166     dateStr = QObject::tr("one hour ago", 0, hours);
 167   } else if (days < 1) {
 168     dateStr = QObject::tr("%n hours ago", 0, hours);
 169   } else if (longTime) {
 170     if (days == 1) {
 171       dateStr = QObject::tr("one day ago", 0, days);
 172     } else if (weeks < 1) {
 173       dateStr = QObject::tr("%n days ago", 0, days);
 174     } else if (weeks == 1) {
 175       dateStr = QObject::tr("one week ago", 0, weeks);
 176     } else if (months < 1) {
 177       dateStr = QObject::tr("%n weeks ago", 0, weeks);
 178     } else if (months == 1) {
 179       dateStr = QObject::tr("one month ago", 0, months);
 180     } else if (years < 1) {
 181       dateStr = QObject::tr("%n months ago", 0, months);
 182     } else if (years == 1) {
 183       dateStr = QObject::tr("one year ago", 0, years);
 184     } else {
 185       dateStr = QObject::tr("%n years ago", 0, years);
 186     }
 187   }
 188   return dateStr;
 189 }
 190
 191 //------------------------------------------------------------------------------
 192
 193 bool splitWebfingerId(QString accountId, QString& username, QString& server) {
 194   static QRegExp rx("^([\\w\\._-+]+)@([\\w\\._-+]+)$");
 195   if (!rx.exactMatch(accountId.trimmed()))
 196     return false;
 197
 198   username = rx.cap(1);
 199   server = rx.cap(2);
 200   return true;
 201 }
 202
 203 //------------------------------------------------------------------------------
 204
 205 long getMaxRSS() {
 206 #ifdef DEBUG_MEMORY
 207   struct rusage rusage;
 208   getrusage(RUSAGE_SELF, &rusage);
 209   return rusage.ru_maxrss;
 210 #else
 211   return 0;
 212 #endif
 213 }
 214
 215 //------------------------------------------------------------------------------
 216
 217 long getCurrentRSS() {
 218 #ifdef DEBUG_MEMORY
 219   QFile fp("/proc/self/statm");
 220   if (!fp.open(QIODevice::ReadOnly))
 221     return -1;
 222
 223   QTextStream in(&fp);
 224   QString line = in.readLine();
 225   QStringList parts = line.split(" ");
 226
 227   return parts[1].toLong() * sysconf( _SC_PAGESIZE);
 228 #else
 229   return 0;
 230 #endif
 231 }
 232
 233 //------------------------------------------------------------------------------
 234
 235 void checkMemory(QString desc) {
 236   static long oldMem = -1;
 237
 238   long mem = getCurrentRSS();
 239   long diff = 0;
 240   if (oldMem > 0)
 241     diff = mem-oldMem;
 242
 243   QString msg("RESIDENT MEMORY");
 244   if (!desc.isEmpty())
 245     msg += " (" + desc + ")";
 246   msg += QString(": %1 KB").arg((float)mem/1024.0, 0, 'f', 2);
 247   if (diff != 0)
 248     msg += QString(" (%2%1)").arg(diff).arg(diff > 0 ? '+' : '-');
 249
 250   qDebug() << msg;
 251 }
 252
 253 //------------------------------------------------------------------------------
 254
 255 QString removeHtml(QString origText) {
 256   QString text = origText;
 257
 258   // Remove any inline HTML tags
 259   // text.replace(QRegExp(HTML_TAG_REGEX), "&lt;\\1&gt;");
 260   QRegExp rx(HTML_TAG_REGEX);
 261   QRegExp urlRx(URL_REGEX);
 262   int pos = 0;
 263
 264   while ((pos = rx.indexIn(text, pos)) != -1) {
 265     int len = rx.matchedLength();
 266     QString tag = rx.cap(1);
 267     if (urlRx.exactMatch(tag)) {
 268       pos += len;
 269     } else {
 270       QString newText = "&lt;" + tag + "&gt;";
 271       text.replace(pos, len, newText);
 272       pos += newText.length();
 273     }
 274   }
 275   return text;
 276 }
 277
 278 //------------------------------------------------------------------------------
 279
 280 #ifndef NO_TIDY
 281 // Tidy changed TidyBodyOnly type from bool to int at 2007-05-24.
 282 // Returns true when a new version (int) is found
 283 bool isTidyWithIntBodyOnlyCheck() {
 284   QString releaseDateStr(tidyReleaseDate());
 285   int yearPos = releaseDateStr.indexOf(QRegExp(" [0-9]{4}"));
 286   if (yearPos != -1)
 287     releaseDateStr = releaseDateStr.left(yearPos + 5);
 288   QDate releaseDate = QLocale::c().toDate(releaseDateStr, "d MMMM yyyy");
 289   QDate changeDate = QLocale::c().toDate("24 May 2007", "d MMMM yyyy");
 290   bool isNewer = releaseDate > changeDate;
 291
 292 #ifdef DEBUG_MARKUP
 293   qDebug() << "\n[DEBUG] tidy release date:" << releaseDate.toString(Qt::ISODate);
 294   qDebug() << "\n[DEBUG] use API with new TidyBodyOnly as int :" << isNewer;
 295 #endif
 296
 297   return isNewer;
 298 }
 299 #endif
 300
 301 //------------------------------------------------------------------------------
 302
 303 QString tidyHtml(QString str, bool& ok) {
 304 #ifdef NO_TIDY
 305   ok = true;
 306   return str;
 307 #else
 308   QString res = str;
 309   ok = false;
 310
 311   static bool isTidyWithIntBodyOnly = isTidyWithIntBodyOnlyCheck();
 312
 313   TidyDoc tdoc = tidyCreate();
 314   TidyBuffer output;
 315   TidyBuffer errbuf;
 316
 317   tidyBufInit(&output);
 318   tidyBufInit(&errbuf);
 319
 320   bool configOk =
 321     tidyOptSetBool(tdoc, TidyXhtmlOut, yes) &&
 322     tidyOptSetBool(tdoc, TidyForceOutput, yes) &&
 323     tidyOptSetBool(tdoc, TidyMark, no) &&
 324     (isTidyWithIntBodyOnly
 325      ? tidyOptSetInt(tdoc, TidyBodyOnly, 1)
 326      : tidyOptSetBool(tdoc, TidyBodyOnly, yes)) &&
 327     tidyOptSetInt(tdoc, TidyWrapLen, 0) &&
 328     tidyOptSetInt(tdoc, TidyDoctypeMode, TidyDoctypeOmit);
 329
 330   if (configOk &&
 331       (tidySetCharEncoding(tdoc, "utf8") >= 0) &&
 332       (tidySetErrorBuffer(tdoc, &errbuf) >= 0) &&
 333       (tidyParseString(tdoc, str.toUtf8().data()) >= 0) &&
 334       (tidyCleanAndRepair(tdoc) >= 0) &&
 335       (tidyRunDiagnostics(tdoc) >= 0) &&
 336       (tidySaveBuffer(tdoc, &output) >= 0) &&
 337       (output.bp != 0 && output.size > 0)) {
 338     res = QString::fromUtf8((char*)output.bp, output.size);
 339
 340     ok = true;
 341   }
 342
 343 #ifdef DEBUG_MARKUP
 344   if (errbuf.size > 0) {
 345     QString errStr =  QString::fromUtf8((char*)errbuf.bp, errbuf.size);
 346     qDebug() << "\n[DEBUG] MARKUP, libtidy errors and warnings:\n" << errStr;
 347   }
 348 #endif
 349
 350   if (output.bp != 0)
 351     tidyBufFree(&output);
 352   if (errbuf.bp != 0)
 353     tidyBufFree(&errbuf);
 354   tidyRelease(tdoc);
 355
 356   return res.trimmed();
 357 #endif
 358 }
 359
 360 //------------------------------------------------------------------------------
 361
 362 QString addTextMarkup(QString text, bool useMarkdown) {
 363   QString oldText = text;
 364
 365 #ifdef DEBUG_MARKUP
 366   qDebug() << "\n[DEBUG] MARKUP\n" << text;
 367 #endif
 368
 369 #ifdef NO_TIDY
 370   text = removeHtml(text);
 371 # ifdef DEBUG_MARKUP
 372   qDebug() << "\n[DEBUG] MARKUP (clean inline HTML)\n" << text;
 373 # endif
 374 #endif
 375
 376   if (useMarkdown) {
 377     // apply markdown
 378     text = markDown(text);
 379   } else {
 380     // linkify plain URLs
 381     text = linkifyUrls(text, useMarkdown);
 382
 383 #ifdef DEBUG_MARKUP
 384     qDebug() << "\n[DEBUG] MARKUP (linkify plain URLs)\n" << text;
 385 #endif
 386
 387     // This is a bit of a hack: if the text doesn't certain tags we
 388     // replace newlines with breaks.
 389
 390     QRegExp rx("<p>|<ul>", Qt::CaseInsensitive);
 391     if (rx.indexIn(text) == -1)
 392       text.replace("\n", "<br/>");
 393   }
 394
 395 #ifdef DEBUG_MARKUP
 396   qDebug() << "\n[DEBUG] MARKUP (apply"
 397            << (useMarkdown?"Markdown)":"text conversion)") << "\n" << text;
 398 #endif
 399
 400 #ifndef NO_TIDY
 401   bool tidyOk = false;
 402   text = tidyHtml(text, tidyOk);
 403 #endif
 404
 405 #ifdef DEBUG_MARKUP
 406   qDebug() << "\n[DEBUG] MARKUP (libtidy)\n" << text;
 407 #endif
 408
 409   return text;
 410 }
 411
 412 //------------------------------------------------------------------------------
 413
 414 QString processTitle(QString text, bool removeLtgt) {
 415   // text = removeHtml(text);
 416   if (removeLtgt) {
 417     text.replace("<", "&lt;");
 418     text.replace(">", "&gt;");
 419   }
 420   text.replace("\n", " ");
 421   return text.trimmed();
 422 }
 423
 424 //------------------------------------------------------------------------------
 425
 426 QString slashify(QString path) {
 427   QString ret = path;
 428   if (!ret.endsWith('/'))
 429     ret.append('/');
 430   return ret;
 431 }