sc/source/ui/docshell/datastream.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  */
   9
  10 #include <datastream.hxx>
  11 #include <datastreamgettime.hxx>
  12
  13 #include <com/sun/star/frame/XLayoutManager.hpp>
  14 #include <osl/conditn.hxx>
  15 #include <osl/time.h>
  16 #include <salhelper/thread.hxx>
  17 #include <sfx2/viewfrm.hxx>
  18 #include <tools/stream.hxx>
  19 #include <vcl/svapp.hxx>
  20 #include <docsh.hxx>
  21 #include <tabvwsh.hxx>
  22 #include <viewdata.hxx>
  23 #include <stringutil.hxx>
  24 #include <documentlinkmgr.hxx>
  25 #include <o3tl/enumarray.hxx>
  26
  27 #include <officecfg/Office/Calc.hxx>
  28
  29 #include <orcus/csv_parser.hpp>
  30
  31 #include <queue>
  32
  33 namespace com::sun::star::ui { class XUIElement; }
  34
  35 namespace sc {
  36
  37 static o3tl::enumarray<DebugTime, double> fTimes { 0.0, 0.0, 0.0 };
  38
  39 double datastream_get_time(DebugTime nIdx)
  40 {
  41     return fTimes[ nIdx ];
  42 }
  43
  44 namespace {
  45
  46 double getNow()
  47 {
  48     TimeValue now;
  49     osl_getSystemTime(&now);
  50     return static_cast<double>(now.Seconds) + static_cast<double>(now.Nanosec) / 1000000000.0;
  51 }
  52
  53 class CSVHandler
  54 {
  55     DataStream::Line& mrLine;
  56     size_t mnColCount;
  57     size_t mnCols;
  58     const char* mpLineHead;
  59
  60 public:
  61     CSVHandler( DataStream::Line& rLine, size_t nColCount ) :
  62         mrLine(rLine), mnColCount(nColCount), mnCols(0), mpLineHead(rLine.maLine.getStr()) {}
  63
  64     static void begin_parse() {}
  65     static void end_parse() {}
  66     static void begin_row() {}
  67     static void end_row() {}
  68
  69     void cell(const char* p, size_t n, bool /*transient*/)
  70     {
  71         if (mnCols >= mnColCount)
  72             return;
  73
  74         DataStream::Cell aCell;
  75         if (ScStringUtil::parseSimpleNumber(p, n, '.', ',', aCell.mfValue))
  76         {
  77             aCell.mbValue = true;
  78         }
  79         else
  80         {
  81             aCell.mbValue = false;
  82             aCell.maStr.Pos = std::distance(mpLineHead, p);
  83             aCell.maStr.Size = n;
  84         }
  85         mrLine.maCells.push_back(aCell);
  86
  87         ++mnCols;
  88     }
  89 };
  90
  91 }
  92
  93 namespace datastreams {
  94
  95 class ReaderThread : public salhelper::Thread
  96 {
  97     std::unique_ptr<SvStream> mpStream;
  98     size_t mnColCount;
  99     bool mbTerminate;
 100     osl::Mutex maMtxTerminate;
 101
 102     std::queue<std::unique_ptr<DataStream::LinesType>> maPendingLines;
 103     std::queue<std::unique_ptr<DataStream::LinesType>> maUsedLines;
 104     osl::Mutex maMtxLines;
 105
 106     osl::Condition maCondReadStream;
 107     osl::Condition maCondConsume;
 108
 109     orcus::csv::parser_config maConfig;
 110
 111 public:
 112
 113     ReaderThread(std::unique_ptr<SvStream> pData, size_t nColCount):
 114         Thread("ReaderThread"),
 115         mpStream(std::move(pData)),
 116         mnColCount(nColCount),
 117         mbTerminate(false)
 118     {
 119         maConfig.delimiters.push_back(',');
 120         maConfig.text_qualifier = '"';
 121     }
 122
 123     bool isTerminateRequested()
 124     {
 125         osl::MutexGuard aGuard(maMtxTerminate);
 126         return mbTerminate;
 127     }
 128
 129     void requestTerminate()
 130     {
 131         osl::MutexGuard aGuard(maMtxTerminate);
 132         mbTerminate = true;
 133     }
 134
 135     void endThread()
 136     {
 137         requestTerminate();
 138         maCondReadStream.set();
 139     }
 140
 141     void waitForNewLines()
 142     {
 143         maCondConsume.wait();
 144         maCondConsume.reset();
 145     }
 146
 147     std::unique_ptr<DataStream::LinesType> popNewLines()
 148     {
 149         auto pLines = std::move(maPendingLines.front());
 150         maPendingLines.pop();
 151         return pLines;
 152     }
 153
 154     void resumeReadStream()
 155     {
 156         if (maPendingLines.size() <= 4)
 157             maCondReadStream.set(); // start producer again
 158     }
 159
 160     bool hasNewLines() const
 161     {
 162         return !maPendingLines.empty();
 163     }
 164
 165     void pushUsedLines( std::unique_ptr<DataStream::LinesType> pLines )
 166     {
 167         maUsedLines.push(std::move(pLines));
 168     }
 169
 170     osl::Mutex& getLinesMutex()
 171     {
 172         return maMtxLines;
 173     }
 174
 175 private:
 176     virtual void execute() override
 177     {
 178         while (!isTerminateRequested())
 179         {
 180             std::unique_ptr<DataStream::LinesType> pLines;
 181             osl::ResettableMutexGuard aGuard(maMtxLines);
 182
 183             if (!maUsedLines.empty())
 184             {
 185                 // Re-use lines from previous runs.
 186                 pLines = std::move(maUsedLines.front());
 187                 maUsedLines.pop();
 188                 aGuard.clear(); // unlock
 189             }
 190             else
 191             {
 192                 aGuard.clear(); // unlock
 193                 pLines.reset(new DataStream::LinesType(10));
 194             }
 195
 196             // Read & store new lines from stream.
 197             for (DataStream::Line & rLine : *pLines)
 198             {
 199                 rLine.maCells.clear();
 200                 mpStream->ReadLine(rLine.maLine);
 201                 CSVHandler aHdl(rLine, mnColCount);
 202                 orcus::csv_parser<CSVHandler> parser(rLine.maLine.getStr(), rLine.maLine.getLength(), aHdl, maConfig);
 203                 parser.parse();
 204             }
 205
 206             aGuard.reset(); // lock
 207             while (!isTerminateRequested() && maPendingLines.size() >= 8)
 208             {
 209                 // pause reading for a bit
 210                 aGuard.clear(); // unlock
 211                 maCondReadStream.wait();
 212                 maCondReadStream.reset();
 213                 aGuard.reset(); // lock
 214             }
 215             maPendingLines.push(std::move(pLines));
 216             maCondConsume.set();
 217             if (!mpStream->good())
 218                 requestTerminate();
 219         }
 220     }
 221 };
 222
 223 }
 224
 225 DataStream::Cell::Cell() : mfValue(0.0), mbValue(true) {}
 226
 227 DataStream::Cell::Cell( const Cell& r ) : mbValue(r.mbValue)
 228 {
 229     if (r.mbValue)
 230         mfValue = r.mfValue;
 231     else
 232     {
 233         maStr.Pos = r.maStr.Pos;
 234         maStr.Size = r.maStr.Size;
 235     }
 236 }
 237
 238 void DataStream::MakeToolbarVisible()
 239 {
 240     ScViewData* pViewData = ScDocShell::GetViewData();
 241     if (!pViewData)
 242         return;
 243
 244     css::uno::Reference< css::frame::XFrame > xFrame =
 245         pViewData->GetViewShell()->GetViewFrame()->GetFrame().GetFrameInterface();
 246     if (!xFrame.is())
 247         return;
 248
 249     css::uno::Reference< css::beans::XPropertySet > xPropSet(xFrame, css::uno::UNO_QUERY);
 250     if (!xPropSet.is())
 251         return;
 252
 253     css::uno::Reference< css::frame::XLayoutManager > xLayoutManager;
 254     xPropSet->getPropertyValue("LayoutManager") >>= xLayoutManager;
 255     if (!xLayoutManager.is())
 256         return;
 257
 258     static const OUStringLiteral sResourceURL( u"private:resource/toolbar/datastreams" );
 259     css::uno::Reference< css::ui::XUIElement > xUIElement = xLayoutManager->getElement(sResourceURL);
 260     if (!xUIElement.is())
 261     {
 262         xLayoutManager->createElement( sResourceURL );
 263         xLayoutManager->showElement( sResourceURL );
 264     }
 265 }
 266
 267 DataStream* DataStream::Set(
 268     ScDocShell *pShell, const OUString& rURL, const ScRange& rRange,
 269     sal_Int32 nLimit, MoveType eMove, sal_uInt32 nSettings)
 270 {
 271     DataStream* pLink = new DataStream(pShell, rURL, rRange, nLimit, eMove, nSettings);
 272     sc::DocumentLinkManager& rMgr = pShell->GetDocument().GetDocLinkManager();
 273     rMgr.setDataStream(pLink);
 274     return pLink;
 275 }
 276
 277 DataStream::DataStream(ScDocShell *pShell, const OUString& rURL, const ScRange& rRange,
 278         sal_Int32 nLimit, MoveType eMove, sal_uInt32 nSettings) :
 279     mpDocShell(pShell),
 280     maDocAccess(mpDocShell->GetDocument()),
 281     meOrigMove(NO_MOVE),
 282     meMove(NO_MOVE),
 283     mbRunning(false),
 284     mbValuesInLine(false),
 285     mbRefreshOnEmptyLine(false),
 286     mnLinesCount(0),
 287     mnLinesSinceRefresh(0),
 288     mfLastRefreshTime(0.0),
 289     mnCurRow(0),
 290     maImportTimer("sc DataStream maImportTimer"),
 291     mbIsFirst(true),
 292     mbIsUpdate(false)
 293 {
 294     maImportTimer.SetTimeout(0);
 295     maImportTimer.SetInvokeHandler( LINK(this, DataStream, ImportTimerHdl) );
 296
 297     Decode(rURL, rRange, nLimit, eMove, nSettings);
 298 }
 299
 300 DataStream::~DataStream()
 301 {
 302     if (mbRunning)
 303         StopImport();
 304
 305     if (mxReaderThread.is())
 306     {
 307         mxReaderThread->endThread();
 308         mxReaderThread->join();
 309     }
 310     mpLines.reset();
 311 }
 312
 313 DataStream::Line DataStream::ConsumeLine()
 314 {
 315     if (!mpLines || mnLinesCount >= mpLines->size())
 316     {
 317         mnLinesCount = 0;
 318         if (mxReaderThread->isTerminateRequested())
 319             return Line();
 320
 321         osl::ResettableMutexGuard aGuard(mxReaderThread->getLinesMutex());
 322         if (mpLines)
 323             mxReaderThread->pushUsedLines(std::move(mpLines));
 324
 325         while (!mxReaderThread->hasNewLines())
 326         {
 327             aGuard.clear(); // unlock
 328             mxReaderThread->waitForNewLines();
 329             aGuard.reset(); // lock
 330         }
 331
 332         mpLines = mxReaderThread->popNewLines();
 333         mxReaderThread->resumeReadStream();
 334     }
 335     return mpLines->at(mnLinesCount++);
 336 }
 337
 338 ScRange DataStream::GetRange() const
 339 {
 340     ScRange aRange = maStartRange;
 341     aRange.aEnd = maEndRange.aEnd;
 342     return aRange;
 343 }
 344
 345 void DataStream::Decode(const OUString& rURL, const ScRange& rRange,
 346         sal_Int32 nLimit, MoveType eMove, const sal_uInt32 nSettings)
 347 {
 348     msURL = rURL;
 349     meMove = eMove;
 350     meOrigMove = eMove;
 351     mnSettings = nSettings;
 352
 353     mbValuesInLine = true; // always true.
 354
 355     mnCurRow = rRange.aStart.Row();
 356
 357     ScRange aRange = rRange;
 358     if (aRange.aStart.Row() != aRange.aEnd.Row())
 359         // We only allow this range to be one row tall.
 360         aRange.aEnd.SetRow(aRange.aStart.Row());
 361
 362     maStartRange = aRange;
 363     maEndRange = aRange;
 364     const auto & rDoc = mpDocShell->GetDocument();
 365     if (nLimit == 0)
 366     {
 367         // Unlimited
 368         maEndRange.aStart.SetRow(rDoc.MaxRow());
 369     }
 370     else if (nLimit > 0)
 371     {
 372         // Limited.
 373         maEndRange.aStart.IncRow(nLimit-1);
 374         if (maEndRange.aStart.Row() > rDoc.MaxRow())
 375             maEndRange.aStart.SetRow(rDoc.MaxRow());
 376     }
 377
 378     maEndRange.aEnd.SetRow(maEndRange.aStart.Row());
 379 }
 380
 381 void DataStream::StartImport()
 382 {
 383     if (mbRunning)
 384         return;
 385
 386     if (!mxReaderThread.is())
 387     {
 388         std::unique_ptr<SvStream> pStream(new SvFileStream(msURL, StreamMode::READ));
 389         mxReaderThread = new datastreams::ReaderThread(std::move(pStream), maStartRange.aEnd.Col() - maStartRange.aStart.Col() + 1);
 390         mxReaderThread->launch();
 391     }
 392     mbRunning = true;
 393     maDocAccess.reset();
 394
 395     maImportTimer.Start();
 396 }
 397
 398 void DataStream::StopImport()
 399 {
 400     if (!mbRunning)
 401         return;
 402
 403     mbRunning = false;
 404     Refresh();
 405     maImportTimer.Stop();
 406 }
 407
 408 void DataStream::SetRefreshOnEmptyLine( bool bVal )
 409 {
 410     mbRefreshOnEmptyLine = bVal;
 411 }
 412
 413 void DataStream::Refresh()
 414 {
 415     Application::Yield();
 416
 417     double fStart = getNow();
 418
 419     // Hard recalc will repaint the grid area.
 420     mpDocShell->DoHardRecalc();
 421     mpDocShell->SetDocumentModified();
 422
 423     fTimes[ DebugTime::Recalc ] = getNow() - fStart;
 424
 425     mfLastRefreshTime = getNow();
 426     mnLinesSinceRefresh = 0;
 427 }
 428
 429 void DataStream::MoveData()
 430 {
 431     switch (meMove)
 432     {
 433         case RANGE_DOWN:
 434         {
 435             if (mnCurRow == maEndRange.aStart.Row())
 436                 meMove = MOVE_UP;
 437         }
 438         break;
 439         case MOVE_UP:
 440         {
 441             mbIsUpdate = true;
 442             // Remove the top row and shift the remaining rows upward. Then
 443             // insert a new row at the end row position.
 444             ScRange aRange = maStartRange;
 445             aRange.aEnd = maEndRange.aEnd;
 446             maDocAccess.shiftRangeUp(aRange);
 447         }
 448         break;
 449         case MOVE_DOWN:
 450         {
 451             mbIsUpdate = true;
 452             // Remove the end row and shift the remaining rows downward by
 453             // inserting a new row at the top row.
 454             ScRange aRange = maStartRange;
 455             aRange.aEnd = maEndRange.aEnd;
 456             maDocAccess.shiftRangeDown(aRange);
 457         }
 458         break;
 459         case NO_MOVE:
 460         default:
 461             ;
 462     }
 463     if(mbIsFirst && mbIsUpdate)
 464     {
 465         sal_Int32 nStreamTimeout = officecfg::Office::Calc::DataStream::UpdateTimeout::get();
 466         maImportTimer.SetTimeout(nStreamTimeout);
 467         mbIsFirst = false;
 468     }
 469 }
 470
 471 void DataStream::Text2Doc()
 472 {
 473     Line aLine = ConsumeLine();
 474     if (aLine.maCells.empty() && mbRefreshOnEmptyLine)
 475     {
 476         // Empty line detected.  Trigger refresh and discard it.
 477         Refresh();
 478         return;
 479     }
 480
 481     double fStart = getNow();
 482
 483     MoveData();
 484     {
 485         SCCOL nCol = maStartRange.aStart.Col();
 486         const char* pLineHead = aLine.maLine.getStr();
 487         for (const Cell& rCell : aLine.maCells)
 488         {
 489             if (rCell.mbValue)
 490             {
 491                 maDocAccess.setNumericCell(
 492                     ScAddress(nCol, mnCurRow, maStartRange.aStart.Tab()), rCell.mfValue);
 493             }
 494             else
 495             {
 496                 maDocAccess.setStringCell(
 497                     ScAddress(nCol, mnCurRow, maStartRange.aStart.Tab()),
 498                     OUString(pLineHead+rCell.maStr.Pos, rCell.maStr.Size, RTL_TEXTENCODING_UTF8));
 499             }
 500             ++nCol;
 501         }
 502     }
 503
 504     fTimes[ DebugTime::Import ] = getNow() - fStart;
 505
 506     if (meMove == NO_MOVE)
 507         return;
 508
 509     if (meMove == RANGE_DOWN)
 510     {
 511         ++mnCurRow;
 512 //      mpDocShell->GetViewData().GetView()->AlignToCursor(
 513 //              maStartRange.aStart.Col(), mnCurRow, SC_FOLLOW_JUMP);
 514     }
 515
 516     if (getNow() - mfLastRefreshTime > 0.1 && mnLinesSinceRefresh > 200)
 517         // Refresh no more frequently than every 0.1 second, and wait until at
 518         // least we have processed 200 lines.
 519         Refresh();
 520
 521     ++mnLinesSinceRefresh;
 522 }
 523
 524 bool DataStream::ImportData()
 525 {
 526     if (!mbValuesInLine)
 527         // We no longer support this mode. To be deleted later.
 528         return false;
 529
 530     ScViewData* pViewData = ScDocShell::GetViewData();
 531     if (!pViewData)
 532         return false;
 533
 534     if (pViewData->GetViewShell()->NeedsRepaint())
 535         return mbRunning;
 536
 537     Text2Doc();
 538     return mbRunning;
 539 }
 540
 541 IMPL_LINK_NOARG(DataStream, ImportTimerHdl, Timer *, void)
 542 {
 543     if (ImportData())
 544         maImportTimer.Start();
 545 }
 546
 547 } // namespace sc
 548
 549 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */