sc/source/ui/docshell/datastream.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  */
   9
  10 #include <datastream.hxx>
  11 #include <datastreamgettime.hxx>
  12
  13 #include <com/sun/star/frame/XLayoutManager.hpp>
  14 #include <com/sun/star/ui/XUIElement.hpp>
  15 #include <officecfg/Office/Common.hxx>
  16 #include <osl/conditn.hxx>
  17 #include <osl/time.h>
  18 #include <rtl/strbuf.hxx>
  19 #include <salhelper/thread.hxx>
  20 #include <sfx2/viewfrm.hxx>
  21 #include <datastreamdlg.hxx>
  22 #include <docsh.hxx>
  23 #include <rangelst.hxx>
  24 #include <tabvwsh.hxx>
  25 #include <viewdata.hxx>
  26 #include <stringutil.hxx>
  27 #include <documentlinkmgr.hxx>
  28
  29 #include <config_orcus.h>
  30 #include "officecfg/Office/Calc.hxx"
  31
  32
  33 #if ENABLE_ORCUS
  34 #if defined WNT
  35 #define __ORCUS_STATIC_LIB
  36 #endif
  37 #include <orcus/csv_parser.hpp>
  38 #endif
  39
  40 #include <queue>
  41
  42 namespace sc {
  43
  44 enum {
  45     DEBUG_TIME_IMPORT,
  46     DEBUG_TIME_RECALC,
  47     DEBUG_TIME_RENDER,
  48     DEBUG_TIME_MAX
  49 };
  50
  51 static double fTimes[DEBUG_TIME_MAX] = { 0.0, 0.0, 0.0 };
  52
  53 double datastream_get_time(int nIdx)
  54 {
  55     if( nIdx < 0 || nIdx >= (int)SAL_N_ELEMENTS( fTimes ) )
  56         return -1;
  57     return fTimes[ nIdx ];
  58 }
  59
  60 namespace {
  61
  62 inline double getNow()
  63 {
  64     TimeValue now;
  65     osl_getSystemTime(&now);
  66     return static_cast<double>(now.Seconds) + static_cast<double>(now.Nanosec) / 1000000000.0;
  67 }
  68
  69 #if ENABLE_ORCUS
  70
  71 class CSVHandler
  72 {
  73     DataStream::Line& mrLine;
  74     size_t mnColCount;
  75     size_t mnCols;
  76     const char* mpLineHead;
  77
  78 public:
  79     CSVHandler( DataStream::Line& rLine, size_t nColCount ) :
  80         mrLine(rLine), mnColCount(nColCount), mnCols(0), mpLineHead(rLine.maLine.getStr()) {}
  81
  82     static void begin_parse() {}
  83     static void end_parse() {}
  84     static void begin_row() {}
  85     static void end_row() {}
  86
  87     void cell(const char* p, size_t n)
  88     {
  89         if (mnCols >= mnColCount)
  90             return;
  91
  92         DataStream::Cell aCell;
  93         if (ScStringUtil::parseSimpleNumber(p, n, '.', ',', aCell.mfValue))
  94         {
  95             aCell.mbValue = true;
  96         }
  97         else
  98         {
  99             aCell.mbValue = false;
 100             aCell.maStr.Pos = std::distance(mpLineHead, p);
 101             aCell.maStr.Size = n;
 102         }
 103         mrLine.maCells.push_back(aCell);
 104
 105         ++mnCols;
 106     }
 107 };
 108
 109 #endif
 110
 111 }
 112
 113 namespace datastreams {
 114
 115 void emptyLineQueue( std::queue<DataStream::LinesType*>& rQueue )
 116 {
 117     while (!rQueue.empty())
 118     {
 119         delete rQueue.front();
 120         rQueue.pop();
 121     }
 122 }
 123
 124 class ReaderThread : public salhelper::Thread
 125 {
 126     SvStream *mpStream;
 127     size_t mnColCount;
 128     bool mbTerminate;
 129     osl::Mutex maMtxTerminate;
 130
 131     std::queue<DataStream::LinesType*> maPendingLines;
 132     std::queue<DataStream::LinesType*> maUsedLines;
 133     osl::Mutex maMtxLines;
 134
 135     osl::Condition maCondReadStream;
 136     osl::Condition maCondConsume;
 137
 138 #if ENABLE_ORCUS
 139     orcus::csv::parser_config maConfig;
 140 #endif
 141
 142 public:
 143
 144     ReaderThread(SvStream *pData, size_t nColCount):
 145         Thread("ReaderThread"),
 146         mpStream(pData),
 147         mnColCount(nColCount),
 148         mbTerminate(false)
 149     {
 150 #if ENABLE_ORCUS
 151         maConfig.delimiters.push_back(',');
 152         maConfig.text_qualifier = '"';
 153 #endif
 154     }
 155
 156     virtual ~ReaderThread()
 157     {
 158         delete mpStream;
 159         emptyLineQueue(maPendingLines);
 160         emptyLineQueue(maUsedLines);
 161     }
 162
 163     bool isTerminateRequested()
 164     {
 165         osl::MutexGuard aGuard(maMtxTerminate);
 166         return mbTerminate;
 167     }
 168
 169     void requestTerminate()
 170     {
 171         osl::MutexGuard aGuard(maMtxTerminate);
 172         mbTerminate = true;
 173     }
 174
 175     void endThread()
 176     {
 177         requestTerminate();
 178         maCondReadStream.set();
 179     }
 180
 181     void waitForNewLines()
 182     {
 183         maCondConsume.wait();
 184         maCondConsume.reset();
 185     }
 186
 187     DataStream::LinesType* popNewLines()
 188     {
 189         DataStream::LinesType* pLines = maPendingLines.front();
 190         maPendingLines.pop();
 191         return pLines;
 192     }
 193
 194     void resumeReadStream()
 195     {
 196         if (maPendingLines.size() <= 4)
 197             maCondReadStream.set(); // start producer again
 198     }
 199
 200     bool hasNewLines()
 201     {
 202         return !maPendingLines.empty();
 203     }
 204
 205     void pushUsedLines( DataStream::LinesType* pLines )
 206     {
 207         maUsedLines.push(pLines);
 208     }
 209
 210     osl::Mutex& getLinesMutex()
 211     {
 212         return maMtxLines;
 213     }
 214
 215 private:
 216     virtual void execute() SAL_OVERRIDE
 217     {
 218         while (!isTerminateRequested())
 219         {
 220             DataStream::LinesType* pLines = NULL;
 221             osl::ResettableMutexGuard aGuard(maMtxLines);
 222
 223             if (!maUsedLines.empty())
 224             {
 225                 // Re-use lines from previous runs.
 226                 pLines = maUsedLines.front();
 227                 maUsedLines.pop();
 228                 aGuard.clear(); // unlock
 229             }
 230             else
 231             {
 232                 aGuard.clear(); // unlock
 233                 pLines = new DataStream::LinesType(10);
 234             }
 235
 236             // Read & store new lines from stream.
 237             for (size_t i = 0, n = pLines->size(); i < n; ++i)
 238             {
 239                 DataStream::Line& rLine = (*pLines)[i];
 240                 rLine.maCells.clear();
 241                 mpStream->ReadLine(rLine.maLine);
 242 #if ENABLE_ORCUS
 243                 CSVHandler aHdl(rLine, mnColCount);
 244                 orcus::csv_parser<CSVHandler> parser(rLine.maLine.getStr(), rLine.maLine.getLength(), aHdl, maConfig);
 245                 parser.parse();
 246 #endif
 247             }
 248
 249             aGuard.reset(); // lock
 250             while (!isTerminateRequested() && maPendingLines.size() >= 8)
 251             {
 252                 // pause reading for a bit
 253                 aGuard.clear(); // unlock
 254                 maCondReadStream.wait();
 255                 maCondReadStream.reset();
 256                 aGuard.reset(); // lock
 257             }
 258             maPendingLines.push(pLines);
 259             maCondConsume.set();
 260             if (!mpStream->good())
 261                 requestTerminate();
 262         }
 263     }
 264 };
 265
 266 }
 267
 268 DataStream::Cell::Cell() : mfValue(0.0), mbValue(true) {}
 269
 270 DataStream::Cell::Cell( const Cell& r ) : mbValue(r.mbValue)
 271 {
 272     if (r.mbValue)
 273         mfValue = r.mfValue;
 274     else
 275     {
 276         maStr.Pos = r.maStr.Pos;
 277         maStr.Size = r.maStr.Size;
 278     }
 279 }
 280
 281 void DataStream::MakeToolbarVisible()
 282 {
 283     css::uno::Reference< css::frame::XFrame > xFrame =
 284         ScDocShell::GetViewData()->GetViewShell()->GetViewFrame()->GetFrame().GetFrameInterface();
 285     if (!xFrame.is())
 286         return;
 287
 288     css::uno::Reference< css::beans::XPropertySet > xPropSet(xFrame, css::uno::UNO_QUERY);
 289     if (!xPropSet.is())
 290         return;
 291
 292     css::uno::Reference< css::frame::XLayoutManager > xLayoutManager;
 293     xPropSet->getPropertyValue("LayoutManager") >>= xLayoutManager;
 294     if (!xLayoutManager.is())
 295         return;
 296
 297     const OUString sResourceURL( "private:resource/toolbar/datastreams" );
 298     css::uno::Reference< css::ui::XUIElement > xUIElement = xLayoutManager->getElement(sResourceURL);
 299     if (!xUIElement.is())
 300     {
 301         xLayoutManager->createElement( sResourceURL );
 302         xLayoutManager->showElement( sResourceURL );
 303     }
 304 }
 305
 306 DataStream* DataStream::Set(
 307     ScDocShell *pShell, const OUString& rURL, const ScRange& rRange,
 308     sal_Int32 nLimit, MoveType eMove, sal_uInt32 nSettings)
 309 {
 310     DataStream* pLink = new DataStream(pShell, rURL, rRange, nLimit, eMove, nSettings);
 311     sc::DocumentLinkManager& rMgr = pShell->GetDocument().GetDocLinkManager();
 312     rMgr.setDataStream(pLink);
 313     return pLink;
 314 }
 315
 316 DataStream::DataStream(ScDocShell *pShell, const OUString& rURL, const ScRange& rRange,
 317         sal_Int32 nLimit, MoveType eMove, sal_uInt32 nSettings) :
 318     mpDocShell(pShell),
 319     maDocAccess(mpDocShell->GetDocument()),
 320     meOrigMove(NO_MOVE),
 321     meMove(NO_MOVE),
 322     mbRunning(false),
 323     mbValuesInLine(false),
 324     mbRefreshOnEmptyLine(false),
 325     mpLines(0),
 326     mnLinesCount(0),
 327     mnLinesSinceRefresh(0),
 328     mfLastRefreshTime(0.0),
 329     mnCurRow(0),
 330     mbIsFirst(true),
 331     mbIsUpdate(false)
 332 {
 333     maImportTimer.SetTimeout(0);
 334     maImportTimer.SetTimeoutHdl( LINK(this, DataStream, ImportTimerHdl) );
 335
 336     Decode(rURL, rRange, nLimit, eMove, nSettings);
 337 }
 338
 339 DataStream::~DataStream()
 340 {
 341     if (mbRunning)
 342         StopImport();
 343
 344     if (mxReaderThread.is())
 345     {
 346         mxReaderThread->endThread();
 347         mxReaderThread->join();
 348     }
 349     delete mpLines;
 350 }
 351
 352 DataStream::Line DataStream::ConsumeLine()
 353 {
 354     if (!mpLines || mnLinesCount >= mpLines->size())
 355     {
 356         mnLinesCount = 0;
 357         if (mxReaderThread->isTerminateRequested())
 358             return Line();
 359
 360         osl::ResettableMutexGuard aGuard(mxReaderThread->getLinesMutex());
 361         if (mpLines)
 362             mxReaderThread->pushUsedLines(mpLines);
 363
 364         while (!mxReaderThread->hasNewLines())
 365         {
 366             aGuard.clear(); // unlock
 367             mxReaderThread->waitForNewLines();
 368             aGuard.reset(); // lock
 369         }
 370
 371         mpLines = mxReaderThread->popNewLines();
 372         mxReaderThread->resumeReadStream();
 373     }
 374     return mpLines->at(mnLinesCount++);
 375 }
 376
 377 ScRange DataStream::GetRange() const
 378 {
 379     ScRange aRange = maStartRange;
 380     aRange.aEnd = maEndRange.aEnd;
 381     return aRange;
 382 }
 383
 384 void DataStream::Decode(const OUString& rURL, const ScRange& rRange,
 385         sal_Int32 nLimit, MoveType eMove, const sal_uInt32 nSettings)
 386 {
 387     msURL = rURL;
 388     mnLimit = nLimit;
 389     meMove = eMove;
 390     meOrigMove = eMove;
 391     mnSettings = nSettings;
 392
 393     mbValuesInLine = true; // always true.
 394
 395     mnCurRow = rRange.aStart.Row();
 396
 397     ScRange aRange = rRange;
 398     if (aRange.aStart.Row() != aRange.aEnd.Row())
 399         // We only allow this range to be one row tall.
 400         aRange.aEnd.SetRow(aRange.aStart.Row());
 401
 402     maStartRange = aRange;
 403     maEndRange = aRange;
 404     if (nLimit == 0)
 405     {
 406         // Unlimited
 407         maEndRange.aStart.SetRow(MAXROW);
 408     }
 409     else if (nLimit > 0)
 410     {
 411         // Limited.
 412         maEndRange.aStart.IncRow(nLimit-1);
 413         if (maEndRange.aStart.Row() > MAXROW)
 414             maEndRange.aStart.SetRow(MAXROW);
 415     }
 416
 417     maEndRange.aEnd.SetRow(maEndRange.aStart.Row());
 418 }
 419
 420 void DataStream::StartImport()
 421 {
 422     if (mbRunning)
 423         return;
 424
 425     if (!mxReaderThread.is())
 426     {
 427         SvStream *pStream = 0;
 428         if (mnSettings & SCRIPT_STREAM)
 429             pStream = new SvScriptStream(msURL);
 430         else
 431             pStream = new SvFileStream(msURL, StreamMode::READ);
 432         mxReaderThread = new datastreams::ReaderThread(pStream, maStartRange.aEnd.Col() - maStartRange.aStart.Col() + 1);
 433         mxReaderThread->launch();
 434     }
 435     mbRunning = true;
 436     maDocAccess.reset();
 437
 438     maImportTimer.Start();
 439 }
 440
 441 void DataStream::StopImport()
 442 {
 443     if (!mbRunning)
 444         return;
 445
 446     mbRunning = false;
 447     Refresh();
 448     maImportTimer.Stop();
 449 }
 450
 451 void DataStream::SetRefreshOnEmptyLine( bool bVal )
 452 {
 453     mbRefreshOnEmptyLine = bVal;
 454 }
 455
 456 void DataStream::Refresh()
 457 {
 458     Application::Yield();
 459
 460     double fStart = getNow();
 461
 462     // Hard recalc will repaint the grid area.
 463     mpDocShell->DoHardRecalc(true);
 464     mpDocShell->SetDocumentModified(true);
 465
 466     fTimes[ DEBUG_TIME_RECALC ] = getNow() - fStart;
 467
 468     mfLastRefreshTime = getNow();
 469     mnLinesSinceRefresh = 0;
 470 }
 471
 472 void DataStream::MoveData()
 473 {
 474     switch (meMove)
 475     {
 476         case RANGE_DOWN:
 477         {
 478             if (mnCurRow == maEndRange.aStart.Row())
 479                 meMove = MOVE_UP;
 480         }
 481         break;
 482         case MOVE_UP:
 483         {
 484             mbIsUpdate = true;
 485             // Remove the top row and shift the remaining rows upward. Then
 486             // insert a new row at the end row position.
 487             ScRange aRange = maStartRange;
 488             aRange.aEnd = maEndRange.aEnd;
 489             maDocAccess.shiftRangeUp(aRange);
 490         }
 491         break;
 492         case MOVE_DOWN:
 493         {
 494             mbIsUpdate = true;
 495             // Remove the end row and shift the remaining rows downward by
 496             // inserting a new row at the top row.
 497             ScRange aRange = maStartRange;
 498             aRange.aEnd = maEndRange.aEnd;
 499             maDocAccess.shiftRangeDown(aRange);
 500         }
 501         break;
 502         case NO_MOVE:
 503         default:
 504             ;
 505     }
 506     if(mbIsFirst && mbIsUpdate)
 507     {
 508         sal_Int32 nStreamTimeout = officecfg::Office::Calc::DataStream::UpdateTimeout::get();
 509         maImportTimer.SetTimeout(nStreamTimeout);
 510         mbIsFirst = false;
 511     }
 512 }
 513
 514 #if ENABLE_ORCUS
 515
 516 void DataStream::Text2Doc()
 517 {
 518     Line aLine = ConsumeLine();
 519     if (aLine.maCells.empty() && mbRefreshOnEmptyLine)
 520     {
 521         // Empty line detected.  Trigger refresh and discard it.
 522         Refresh();
 523         return;
 524     }
 525
 526     double fStart = getNow();
 527
 528     MoveData();
 529     {
 530         std::vector<Cell>::const_iterator it = aLine.maCells.begin(), itEnd = aLine.maCells.end();
 531         SCCOL nCol = maStartRange.aStart.Col();
 532         const char* pLineHead = aLine.maLine.getStr();
 533         for (; it != itEnd; ++it, ++nCol)
 534         {
 535             const Cell& rCell = *it;
 536             if (rCell.mbValue)
 537             {
 538                 maDocAccess.setNumericCell(
 539                     ScAddress(nCol, mnCurRow, maStartRange.aStart.Tab()), rCell.mfValue);
 540             }
 541             else
 542             {
 543                 maDocAccess.setStringCell(
 544                     ScAddress(nCol, mnCurRow, maStartRange.aStart.Tab()),
 545                     OUString(pLineHead+rCell.maStr.Pos, rCell.maStr.Size, RTL_TEXTENCODING_UTF8));
 546             }
 547         }
 548     }
 549
 550     fTimes[ DEBUG_TIME_IMPORT ] = getNow() - fStart;
 551
 552     if (meMove == NO_MOVE)
 553         return;
 554
 555     if (meMove == RANGE_DOWN)
 556     {
 557         ++mnCurRow;
 558 //      mpDocShell->GetViewData().GetView()->AlignToCursor(
 559 //              maStartRange.aStart.Col(), mnCurRow, SC_FOLLOW_JUMP);
 560     }
 561
 562     if (getNow() - mfLastRefreshTime > 0.1 && mnLinesSinceRefresh > 200)
 563         // Refresh no more frequently than every 0.1 second, and wait until at
 564         // least we have processed 200 lines.
 565         Refresh();
 566
 567     ++mnLinesSinceRefresh;
 568 }
 569
 570 #else
 571
 572 void DataStream::Text2Doc() {}
 573
 574 #endif
 575
 576 bool DataStream::ImportData()
 577 {
 578     if (!mbValuesInLine)
 579         // We no longer support this mode. To be deleted later.
 580         return false;
 581
 582     if (ScDocShell::GetViewData()->GetViewShell()->NeedsRepaint())
 583         return mbRunning;
 584
 585     Text2Doc();
 586     return mbRunning;
 587 }
 588
 589 IMPL_LINK_NOARG_TYPED(DataStream, ImportTimerHdl, Timer *, void)
 590 {
 591     if (ImportData())
 592         maImportTimer.Start();
 593 }
 594
 595 } // namespace sc
 596
 597 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */