sc/source/ui/docshell/datastream.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  */
   9
  10 #include <datastream.hxx>
  11 #include <datastreamgettime.hxx>
  12
  13 #include <com/sun/star/frame/XLayoutManager.hpp>
  14 #include <com/sun/star/ui/XUIElement.hpp>
  15 #include <officecfg/Office/Common.hxx>
  16 #include <osl/conditn.hxx>
  17 #include <osl/time.h>
  18 #include <rtl/strbuf.hxx>
  19 #include <salhelper/thread.hxx>
  20 #include <sfx2/viewfrm.hxx>
  21 #include <datastreamdlg.hxx>
  22 #include <docsh.hxx>
  23 #include <rangelst.hxx>
  24 #include <tabvwsh.hxx>
  25 #include <viewdata.hxx>
  26 #include <stringutil.hxx>
  27 #include <documentlinkmgr.hxx>
  28
  29 #include <config_orcus.h>
  30
  31 #if ENABLE_ORCUS
  32 #if defined WNT
  33 #define __ORCUS_STATIC_LIB
  34 #endif
  35 #include <orcus/csv_parser.hpp>
  36 #endif
  37
  38 #include <queue>
  39
  40 namespace sc {
  41
  42 enum {
  43     DEBUG_TIME_IMPORT,
  44     DEBUG_TIME_RECALC,
  45     DEBUG_TIME_RENDER,
  46     DEBUG_TIME_MAX
  47 };
  48
  49 static double fTimes[DEBUG_TIME_MAX] = { 0.0, 0.0, 0.0 };
  50
  51 double datastream_get_time(int nIdx)
  52 {
  53     if( nIdx < 0 || nIdx >= (int)SAL_N_ELEMENTS( fTimes ) )
  54         return -1;
  55     return fTimes[ nIdx ];
  56 }
  57
  58 namespace {
  59
  60 inline double getNow()
  61 {
  62     TimeValue now;
  63     osl_getSystemTime(&now);
  64     return static_cast<double>(now.Seconds) + static_cast<double>(now.Nanosec) / 1000000000.0;
  65 }
  66
  67 #if ENABLE_ORCUS
  68
  69 class CSVHandler
  70 {
  71     DataStream::Line& mrLine;
  72     size_t mnColCount;
  73     size_t mnCols;
  74     const char* mpLineHead;
  75
  76 public:
  77     CSVHandler( DataStream::Line& rLine, size_t nColCount ) :
  78         mrLine(rLine), mnColCount(nColCount), mnCols(0), mpLineHead(rLine.maLine.getStr()) {}
  79
  80     void begin_parse() {}
  81     void end_parse() {}
  82     void begin_row() {}
  83     void end_row() {}
  84
  85     void cell(const char* p, size_t n)
  86     {
  87         if (mnCols >= mnColCount)
  88             return;
  89
  90         DataStream::Cell aCell;
  91         if (ScStringUtil::parseSimpleNumber(p, n, '.', ',', aCell.mfValue))
  92         {
  93             aCell.mbValue = true;
  94         }
  95         else
  96         {
  97             aCell.mbValue = false;
  98             aCell.maStr.Pos = std::distance(mpLineHead, p);
  99             aCell.maStr.Size = n;
 100         }
 101         mrLine.maCells.push_back(aCell);
 102
 103         ++mnCols;
 104     }
 105 };
 106
 107 #endif
 108
 109 }
 110
 111 namespace datastreams {
 112
 113 void emptyLineQueue( std::queue<DataStream::LinesType*>& rQueue )
 114 {
 115     while (!rQueue.empty())
 116     {
 117         delete rQueue.front();
 118         rQueue.pop();
 119     }
 120 }
 121
 122 class ReaderThread : public salhelper::Thread
 123 {
 124     SvStream *mpStream;
 125     size_t mnColCount;
 126     bool mbTerminate;
 127     osl::Mutex maMtxTerminate;
 128
 129     std::queue<DataStream::LinesType*> maPendingLines;
 130     std::queue<DataStream::LinesType*> maUsedLines;
 131     osl::Mutex maMtxLines;
 132
 133     osl::Condition maCondReadStream;
 134     osl::Condition maCondConsume;
 135
 136 #if ENABLE_ORCUS
 137     orcus::csv::parser_config maConfig;
 138 #endif
 139
 140 public:
 141
 142     ReaderThread(SvStream *pData, size_t nColCount):
 143         Thread("ReaderThread"),
 144         mpStream(pData),
 145         mnColCount(nColCount),
 146         mbTerminate(false)
 147     {
 148 #if ENABLE_ORCUS
 149         maConfig.delimiters.push_back(',');
 150         maConfig.text_qualifier = '"';
 151 #endif
 152     }
 153
 154     virtual ~ReaderThread()
 155     {
 156         delete mpStream;
 157         emptyLineQueue(maPendingLines);
 158         emptyLineQueue(maUsedLines);
 159     }
 160
 161     bool isTerminateRequested()
 162     {
 163         osl::MutexGuard aGuard(maMtxTerminate);
 164         return mbTerminate;
 165     }
 166
 167     void requestTerminate()
 168     {
 169         osl::MutexGuard aGuard(maMtxTerminate);
 170         mbTerminate = true;
 171     }
 172
 173     void endThread()
 174     {
 175         requestTerminate();
 176         maCondReadStream.set();
 177     }
 178
 179     void waitForNewLines()
 180     {
 181         maCondConsume.wait();
 182         maCondConsume.reset();
 183     }
 184
 185     DataStream::LinesType* popNewLines()
 186     {
 187         DataStream::LinesType* pLines = maPendingLines.front();
 188         maPendingLines.pop();
 189         return pLines;
 190     }
 191
 192     void resumeReadStream()
 193     {
 194         if (maPendingLines.size() <= 4)
 195             maCondReadStream.set(); // start producer again
 196     }
 197
 198     bool hasNewLines()
 199     {
 200         return !maPendingLines.empty();
 201     }
 202
 203     void pushUsedLines( DataStream::LinesType* pLines )
 204     {
 205         maUsedLines.push(pLines);
 206     }
 207
 208     osl::Mutex& getLinesMutex()
 209     {
 210         return maMtxLines;
 211     }
 212
 213 private:
 214     virtual void execute() SAL_OVERRIDE
 215     {
 216         while (!isTerminateRequested())
 217         {
 218             DataStream::LinesType* pLines = NULL;
 219             osl::ResettableMutexGuard aGuard(maMtxLines);
 220
 221             if (!maUsedLines.empty())
 222             {
 223                 // Re-use lines from previous runs.
 224                 pLines = maUsedLines.front();
 225                 maUsedLines.pop();
 226                 aGuard.clear(); // unlock
 227             }
 228             else
 229             {
 230                 aGuard.clear(); // unlock
 231                 pLines = new DataStream::LinesType(10);
 232             }
 233
 234             // Read & store new lines from stream.
 235             for (size_t i = 0, n = pLines->size(); i < n; ++i)
 236             {
 237                 DataStream::Line& rLine = (*pLines)[i];
 238                 rLine.maCells.clear();
 239                 mpStream->ReadLine(rLine.maLine);
 240 #if ENABLE_ORCUS
 241                 CSVHandler aHdl(rLine, mnColCount);
 242                 orcus::csv_parser<CSVHandler> parser(rLine.maLine.getStr(), rLine.maLine.getLength(), aHdl, maConfig);
 243                 parser.parse();
 244 #endif
 245             }
 246
 247             aGuard.reset(); // lock
 248             while (!isTerminateRequested() && maPendingLines.size() >= 8)
 249             {
 250                 // pause reading for a bit
 251                 aGuard.clear(); // unlock
 252                 maCondReadStream.wait();
 253                 maCondReadStream.reset();
 254                 aGuard.reset(); // lock
 255             }
 256             maPendingLines.push(pLines);
 257             maCondConsume.set();
 258             if (!mpStream->good())
 259                 requestTerminate();
 260         }
 261     }
 262 };
 263
 264 }
 265
 266 DataStream::Cell::Cell() : mfValue(0.0), mbValue(true) {}
 267
 268 DataStream::Cell::Cell( const Cell& r ) : mbValue(r.mbValue)
 269 {
 270     if (r.mbValue)
 271         mfValue = r.mfValue;
 272     else
 273     {
 274         maStr.Pos = r.maStr.Pos;
 275         maStr.Size = r.maStr.Size;
 276     }
 277 }
 278
 279 void DataStream::MakeToolbarVisible()
 280 {
 281     css::uno::Reference< css::frame::XFrame > xFrame =
 282         ScDocShell::GetViewData()->GetViewShell()->GetViewFrame()->GetFrame().GetFrameInterface();
 283     if (!xFrame.is())
 284         return;
 285
 286     css::uno::Reference< css::beans::XPropertySet > xPropSet(xFrame, css::uno::UNO_QUERY);
 287     if (!xPropSet.is())
 288         return;
 289
 290     css::uno::Reference< css::frame::XLayoutManager > xLayoutManager;
 291     xPropSet->getPropertyValue("LayoutManager") >>= xLayoutManager;
 292     if (!xLayoutManager.is())
 293         return;
 294
 295     const OUString sResourceURL( "private:resource/toolbar/datastreams" );
 296     css::uno::Reference< css::ui::XUIElement > xUIElement = xLayoutManager->getElement(sResourceURL);
 297     if (!xUIElement.is())
 298     {
 299         xLayoutManager->createElement( sResourceURL );
 300         xLayoutManager->showElement( sResourceURL );
 301     }
 302 }
 303
 304 DataStream* DataStream::Set(
 305     ScDocShell *pShell, const OUString& rURL, const ScRange& rRange,
 306     sal_Int32 nLimit, MoveType eMove, sal_uInt32 nSettings)
 307 {
 308     DataStream* pLink = new DataStream(pShell, rURL, rRange, nLimit, eMove, nSettings);
 309     sc::DocumentLinkManager& rMgr = pShell->GetDocument()->GetDocLinkManager();
 310     rMgr.setDataStream(pLink);
 311     return pLink;
 312 }
 313
 314 DataStream::DataStream(ScDocShell *pShell, const OUString& rURL, const ScRange& rRange,
 315         sal_Int32 nLimit, MoveType eMove, sal_uInt32 nSettings) :
 316     mpDocShell(pShell),
 317     mpDoc(mpDocShell->GetDocument()),
 318     maDocAccess(*mpDoc),
 319     meOrigMove(NO_MOVE),
 320     meMove(NO_MOVE),
 321     mbRunning(false),
 322     mbValuesInLine(false),
 323     mbRefreshOnEmptyLine(false),
 324     mpLines(0),
 325     mnLinesCount(0),
 326     mnLinesSinceRefresh(0),
 327     mfLastRefreshTime(0.0),
 328     mnCurRow(0)
 329 {
 330     maImportTimer.SetTimeout(0);
 331     maImportTimer.SetTimeoutHdl( LINK(this, DataStream, ImportTimerHdl) );
 332
 333     Decode(rURL, rRange, nLimit, eMove, nSettings);
 334 }
 335
 336 DataStream::~DataStream()
 337 {
 338     if (mbRunning)
 339         StopImport();
 340
 341     if (mxReaderThread.is())
 342     {
 343         mxReaderThread->endThread();
 344         mxReaderThread->join();
 345     }
 346     delete mpLines;
 347 }
 348
 349 DataStream::Line DataStream::ConsumeLine()
 350 {
 351     if (!mpLines || mnLinesCount >= mpLines->size())
 352     {
 353         mnLinesCount = 0;
 354         if (mxReaderThread->isTerminateRequested())
 355             return Line();
 356
 357         osl::ResettableMutexGuard aGuard(mxReaderThread->getLinesMutex());
 358         if (mpLines)
 359             mxReaderThread->pushUsedLines(mpLines);
 360
 361         while (!mxReaderThread->hasNewLines())
 362         {
 363             aGuard.clear(); // unlock
 364             mxReaderThread->waitForNewLines();
 365             aGuard.reset(); // lock
 366         }
 367
 368         mpLines = mxReaderThread->popNewLines();
 369         mxReaderThread->resumeReadStream();
 370     }
 371     return mpLines->at(mnLinesCount++);
 372 }
 373
 374 ScRange DataStream::GetRange() const
 375 {
 376     ScRange aRange = maStartRange;
 377     aRange.aEnd = maEndRange.aEnd;
 378     return aRange;
 379 }
 380
 381 bool DataStream::IsRefreshOnEmptyLine() const
 382 {
 383     return mbRefreshOnEmptyLine;
 384 }
 385
 386 DataStream::MoveType DataStream::GetMove() const
 387 {
 388     return meOrigMove;
 389 }
 390
 391 void DataStream::Decode(const OUString& rURL, const ScRange& rRange,
 392         sal_Int32 nLimit, MoveType eMove, const sal_uInt32 nSettings)
 393 {
 394     msURL = rURL;
 395     mnLimit = nLimit;
 396     meMove = eMove;
 397     meOrigMove = eMove;
 398     mnSettings = nSettings;
 399
 400     mbValuesInLine = true; // always true.
 401
 402     mnCurRow = rRange.aStart.Row();
 403
 404     ScRange aRange = rRange;
 405     if (aRange.aStart.Row() != aRange.aEnd.Row())
 406         // We only allow this range to be one row tall.
 407         aRange.aEnd.SetRow(aRange.aStart.Row());
 408
 409     maStartRange = aRange;
 410     maEndRange = aRange;
 411     if (nLimit == 0)
 412     {
 413         // Unlimited
 414         maEndRange.aStart.SetRow(MAXROW);
 415     }
 416     else if (nLimit > 0)
 417     {
 418         // Limited.
 419         maEndRange.aStart.IncRow(nLimit-1);
 420         if (maEndRange.aStart.Row() > MAXROW)
 421             maEndRange.aStart.SetRow(MAXROW);
 422     }
 423
 424     maEndRange.aEnd.SetRow(maEndRange.aStart.Row());
 425 }
 426
 427 void DataStream::StartImport()
 428 {
 429     if (mbRunning)
 430         return;
 431
 432     if (!mxReaderThread.is())
 433     {
 434         SvStream *pStream = 0;
 435         if (mnSettings & SCRIPT_STREAM)
 436             pStream = new SvScriptStream(msURL);
 437         else
 438             pStream = new SvFileStream(msURL, STREAM_READ);
 439         mxReaderThread = new datastreams::ReaderThread(pStream, maStartRange.aEnd.Col() - maStartRange.aStart.Col() + 1);
 440         mxReaderThread->launch();
 441     }
 442     mbRunning = true;
 443     maDocAccess.reset();
 444
 445     maImportTimer.Start();
 446 }
 447
 448 void DataStream::StopImport()
 449 {
 450     if (!mbRunning)
 451         return;
 452
 453     mbRunning = false;
 454     Refresh();
 455     maImportTimer.Stop();
 456 }
 457
 458 void DataStream::SetRefreshOnEmptyLine( bool bVal )
 459 {
 460     mbRefreshOnEmptyLine = bVal;
 461 }
 462
 463 void DataStream::Refresh()
 464 {
 465     Application::Yield();
 466
 467     double fStart = getNow();
 468
 469     // Hard recalc will repaint the grid area.
 470     mpDocShell->DoHardRecalc(true);
 471     mpDocShell->SetDocumentModified(true);
 472
 473     fTimes[ DEBUG_TIME_RECALC ] = getNow() - fStart;
 474
 475     mfLastRefreshTime = getNow();
 476     mnLinesSinceRefresh = 0;
 477 }
 478
 479 void DataStream::MoveData()
 480 {
 481     switch (meMove)
 482     {
 483         case RANGE_DOWN:
 484         {
 485             if (mnCurRow == maEndRange.aStart.Row())
 486                 meMove = MOVE_UP;
 487         }
 488         break;
 489         case MOVE_UP:
 490         {
 491             // Remove the top row and shift the remaining rows upward. Then
 492             // insert a new row at the end row position.
 493             ScRange aRange = maStartRange;
 494             aRange.aEnd = maEndRange.aEnd;
 495             maDocAccess.shiftRangeUp(aRange);
 496         }
 497         break;
 498         case MOVE_DOWN:
 499         {
 500             // Remove the end row and shift the remaining rows downward by
 501             // inserting a new row at the top row.
 502             ScRange aRange = maStartRange;
 503             aRange.aEnd = maEndRange.aEnd;
 504             maDocAccess.shiftRangeDown(aRange);
 505         }
 506         break;
 507         case NO_MOVE:
 508         default:
 509             ;
 510     }
 511 }
 512
 513 #if ENABLE_ORCUS
 514
 515 void DataStream::Text2Doc()
 516 {
 517     Line aLine = ConsumeLine();
 518     if (aLine.maCells.empty() && mbRefreshOnEmptyLine)
 519     {
 520         // Empty line detected.  Trigger refresh and discard it.
 521         Refresh();
 522         return;
 523     }
 524
 525     double fStart = getNow();
 526
 527     MoveData();
 528     {
 529         std::vector<Cell>::const_iterator it = aLine.maCells.begin(), itEnd = aLine.maCells.end();
 530         SCCOL nCol = maStartRange.aStart.Col();
 531         const char* pLineHead = aLine.maLine.getStr();
 532         for (; it != itEnd; ++it, ++nCol)
 533         {
 534             const Cell& rCell = *it;
 535             if (rCell.mbValue)
 536             {
 537                 maDocAccess.setNumericCell(
 538                     ScAddress(nCol, mnCurRow, maStartRange.aStart.Tab()), rCell.mfValue);
 539             }
 540             else
 541             {
 542                 maDocAccess.setStringCell(
 543                     ScAddress(nCol, mnCurRow, maStartRange.aStart.Tab()),
 544                     OUString(pLineHead+rCell.maStr.Pos, rCell.maStr.Size, RTL_TEXTENCODING_UTF8));
 545             }
 546         }
 547     }
 548
 549     fTimes[ DEBUG_TIME_IMPORT ] = getNow() - fStart;
 550
 551     if (meMove == NO_MOVE)
 552         return;
 553
 554     if (meMove == RANGE_DOWN)
 555     {
 556         ++mnCurRow;
 557 //      mpDocShell->GetViewData()->GetView()->AlignToCursor(
 558 //              maStartRange.aStart.Col(), mnCurRow, SC_FOLLOW_JUMP);
 559     }
 560
 561     if (getNow() - mfLastRefreshTime > 0.1 && mnLinesSinceRefresh > 200)
 562         // Refresh no more frequently than every 0.1 second, and wait until at
 563         // least we have processed 200 lines.
 564         Refresh();
 565
 566     ++mnLinesSinceRefresh;
 567 }
 568
 569 #else
 570
 571 void DataStream::Text2Doc() {}
 572
 573 #endif
 574
 575 bool DataStream::ImportData()
 576 {
 577     if (!mbValuesInLine)
 578         // We no longer support this mode. To be deleted later.
 579         return false;
 580
 581     if (ScDocShell::GetViewData()->GetViewShell()->NeedsRepaint())
 582         return mbRunning;
 583
 584     Text2Doc();
 585     return mbRunning;
 586 }
 587
 588 IMPL_LINK_NOARG(DataStream, ImportTimerHdl)
 589 {
 590     if (ImportData())
 591         maImportTimer.Start();
 592
 593     return 0;
 594 }
 595
 596 } // namespace sc
 597
 598 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */