sc/source/ui/docshell/externalrefmgr.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  *
   9  * This file incorporates work covered by the following license notice:
  10  *
  11  *   Licensed to the Apache Software Foundation (ASF) under one or more
  12  *   contributor license agreements. See the NOTICE file distributed
  13  *   with this work for additional information regarding copyright
  14  *   ownership. The ASF licenses this file to you under the Apache
  15  *   License, Version 2.0 (the "License"); you may not use this file
  16  *   except in compliance with the License. You may obtain a copy of
  17  *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
  18  */
  19
  20 #include <externalrefmgr.hxx>
  21 #include <document.hxx>
  22 #include <token.hxx>
  23 #include <tokenarray.hxx>
  24 #include <address.hxx>
  25 #include <tablink.hxx>
  26 #include <docsh.hxx>
  27 #include <scextopt.hxx>
  28 #include <rangenam.hxx>
  29 #include <formulacell.hxx>
  30 #include <utility>
  31 #include <viewdata.hxx>
  32 #include <tabvwsh.hxx>
  33 #include <sc.hrc>
  34 #include <globstr.hrc>
  35 #include <scresid.hxx>
  36 #include <cellvalue.hxx>
  37 #include <defaultsoptions.hxx>
  38 #include <scmod.hxx>
  39
  40 #include <o3tl/safeint.hxx>
  41 #include <osl/diagnose.h>
  42 #include <osl/file.hxx>
  43 #include <sfx2/app.hxx>
  44 #include <sfx2/docfile.hxx>
  45 #include <sfx2/event.hxx>
  46 #include <sfx2/fcontnr.hxx>
  47 #include <sfx2/objsh.hxx>
  48 #include <svl/itemset.hxx>
  49 #include <svl/numformat.hxx>
  50 #include <svl/stritem.hxx>
  51 #include <svl/urihelper.hxx>
  52 #include <svl/sharedstringpool.hxx>
  53 #include <sfx2/linkmgr.hxx>
  54 #include <tools/hostfilter.hxx>
  55 #include <tools/urlobj.hxx>
  56 #include <unotools/charclass.hxx>
  57 #include <comphelper/configuration.hxx>
  58 #include <unotools/ucbhelper.hxx>
  59 #include <vcl/svapp.hxx>
  60 #include <vcl/weld.hxx>
  61 #include <stringutil.hxx>
  62 #include <scmatrix.hxx>
  63 #include <columnspanset.hxx>
  64 #include <column.hxx>
  65 #include <com/sun/star/document/MacroExecMode.hpp>
  66 #include <com/sun/star/document/UpdateDocMode.hpp>
  67 #include <sal/log.hxx>
  68
  69 #include <memory>
  70 #include <algorithm>
  71
  72 using ::std::unique_ptr;
  73 using ::com::sun::star::uno::Any;
  74 using ::std::vector;
  75 using ::std::find_if;
  76 using ::std::for_each;
  77 using ::std::distance;
  78 using ::std::pair;
  79 using namespace formula;
  80
  81 #define SRCDOC_LIFE_SPAN     30000      // 5 minutes (in 100th of a sec)
  82 #define SRCDOC_SCAN_INTERVAL 1000*30    // every 30 seconds (in msec)
  83
  84 namespace {
  85
  86 class TabNameSearchPredicate
  87 {
  88 public:
  89     explicit TabNameSearchPredicate(const OUString& rSearchName) :
  90         maSearchName(ScGlobal::getCharClass().uppercase(rSearchName))
  91     {
  92     }
  93
  94     bool operator()(const ScExternalRefCache::TableName& rTabNameSet) const
  95     {
  96         // Ok, I'm doing case insensitive search here.
  97         return rTabNameSet.maUpperName == maSearchName;
  98     }
  99
 100 private:
 101     OUString maSearchName;
 102 };
 103
 104 class FindSrcFileByName
 105 {
 106 public:
 107     explicit FindSrcFileByName(const OUString& rMatchName) :
 108         mrMatchName(rMatchName)
 109     {
 110     }
 111
 112     bool operator()(const ScExternalRefManager::SrcFileData& rSrcData) const
 113     {
 114         return rSrcData.maFileName == mrMatchName;
 115     }
 116
 117 private:
 118     const OUString& mrMatchName;
 119 };
 120
 121 class NotifyLinkListener
 122 {
 123 public:
 124     NotifyLinkListener(sal_uInt16 nFileId, ScExternalRefManager::LinkUpdateType eType) :
 125         mnFileId(nFileId), meType(eType) {}
 126
 127     void operator() (ScExternalRefManager::LinkListener* p) const
 128     {
 129         p->notify(mnFileId, meType);
 130     }
 131 private:
 132     sal_uInt16 mnFileId;
 133     ScExternalRefManager::LinkUpdateType meType;
 134 };
 135
 136 struct UpdateFormulaCell
 137 {
 138     void operator() (ScFormulaCell* pCell) const
 139     {
 140         // Check to make sure the cell really contains svExternal*.
 141         // External names, external cell and range references all have a
 142         // token of svExternal*. Additionally check for INDIRECT() that can be
 143         // called with any constructed URI string.
 144         ScTokenArray* pCode = pCell->GetCode();
 145         if (!pCode->HasExternalRef() && !pCode->HasOpCode(ocIndirect))
 146             return;
 147
 148         if (pCode->GetCodeError() != FormulaError::NONE)
 149         {
 150             // Clear the error code, or a cell with error won't get re-compiled.
 151             pCode->SetCodeError(FormulaError::NONE);
 152             pCell->SetCompile(true);
 153             pCell->CompileTokenArray();
 154         }
 155
 156         pCell->SetDirty();
 157     }
 158 };
 159
 160 class RemoveFormulaCell
 161 {
 162 public:
 163     explicit RemoveFormulaCell(ScFormulaCell* p) : mpCell(p) {}
 164     void operator() (pair<const sal_uInt16, ScExternalRefManager::RefCellSet>& r) const
 165     {
 166         r.second.erase(mpCell);
 167     }
 168 private:
 169     ScFormulaCell* mpCell;
 170 };
 171
 172 class ConvertFormulaToStatic
 173 {
 174 public:
 175     explicit ConvertFormulaToStatic(ScDocument* pDoc) : mpDoc(pDoc) {}
 176     void operator() (ScFormulaCell* pCell) const
 177     {
 178         ScAddress aPos = pCell->aPos;
 179
 180         // We don't check for empty cells because empty external cells are
 181         // treated as having a value of 0.
 182
 183         if (pCell->IsValue())
 184         {
 185             // Turn this into value cell.
 186             mpDoc->SetValue(aPos, pCell->GetValue());
 187         }
 188         else
 189         {
 190             // string cell otherwise.
 191             ScSetStringParam aParam;
 192             aParam.setTextInput();
 193             mpDoc->SetString(aPos, pCell->GetString().getString(), &aParam);
 194         }
 195     }
 196 private:
 197     ScDocument* mpDoc;
 198 };
 199
 200 /**
 201  * Check whether a named range contains an external reference to a
 202  * particular document.
 203  */
 204 bool hasRefsToSrcDoc(ScRangeData& rData, sal_uInt16 nFileId)
 205 {
 206     ScTokenArray* pArray = rData.GetCode();
 207     if (!pArray)
 208         return false;
 209
 210     formula::FormulaTokenArrayPlainIterator aIter(*pArray);
 211     formula::FormulaToken* p = aIter.GetNextReference();
 212     for (; p; p = aIter.GetNextReference())
 213     {
 214         if (!p->IsExternalRef())
 215             continue;
 216
 217         if (p->GetIndex() == nFileId)
 218             return true;
 219     }
 220     return false;
 221 }
 222
 223 class EraseRangeByIterator
 224 {
 225     ScRangeName& mrRanges;
 226 public:
 227     explicit EraseRangeByIterator(ScRangeName& rRanges) : mrRanges(rRanges) {}
 228     void operator() (const ScRangeName::const_iterator& itr)
 229     {
 230         mrRanges.erase(itr);
 231     }
 232 };
 233
 234 /**
 235  * Remove all named ranges that contain references to specified source
 236  * document.
 237  */
 238 void removeRangeNamesBySrcDoc(ScRangeName& rRanges, sal_uInt16 nFileId)
 239 {
 240     ScRangeName::const_iterator itr = rRanges.begin(), itrEnd = rRanges.end();
 241     vector<ScRangeName::const_iterator> v;
 242     for (; itr != itrEnd; ++itr)
 243     {
 244         if (hasRefsToSrcDoc(*itr->second, nFileId))
 245             v.push_back(itr);
 246     }
 247     for_each(v.begin(), v.end(), EraseRangeByIterator(rRanges));
 248 }
 249
 250 }
 251
 252 ScExternalRefCache::Table::Table()
 253     : mbReferenced( true )
 254       // Prevent accidental data loss due to lack of knowledge.
 255 {
 256 }
 257
 258 ScExternalRefCache::Table::~Table()
 259 {
 260 }
 261
 262 void ScExternalRefCache::Table::clear()
 263 {
 264     maRows.clear();
 265     maCachedRanges.RemoveAll();
 266     mbReferenced = true;
 267 }
 268
 269 void ScExternalRefCache::Table::setReferenced( bool bReferenced )
 270 {
 271     mbReferenced = bReferenced;
 272 }
 273
 274 bool ScExternalRefCache::Table::isReferenced() const
 275 {
 276     return mbReferenced;
 277 }
 278
 279 void ScExternalRefCache::Table::setCell(SCCOL nCol, SCROW nRow, TokenRef const & pToken, sal_uLong nFmtIndex, bool bSetCacheRange)
 280 {
 281     using ::std::pair;
 282     RowsDataType::iterator itrRow = maRows.find(nRow);
 283     if (itrRow == maRows.end())
 284     {
 285         // This row does not exist yet.
 286         pair<RowsDataType::iterator, bool> res = maRows.emplace(
 287             nRow, RowDataType());
 288
 289         if (!res.second)
 290             return;
 291
 292         itrRow = res.first;
 293     }
 294
 295     // Insert this token into the specified column location.  I don't need to
 296     // check for existing data.  Just overwrite it.
 297     RowDataType& rRow = itrRow->second;
 298     ScExternalRefCache::Cell aCell;
 299     aCell.mxToken = pToken;
 300     aCell.mnFmtIndex = nFmtIndex;
 301     rRow.emplace(nCol, aCell);
 302     if (bSetCacheRange)
 303         setCachedCell(nCol, nRow);
 304 }
 305
 306 ScExternalRefCache::TokenRef ScExternalRefCache::Table::getCell(SCCOL nCol, SCROW nRow, sal_uInt32* pnFmtIndex) const
 307 {
 308     RowsDataType::const_iterator itrTable = maRows.find(nRow);
 309     if (itrTable == maRows.end())
 310     {
 311         // this table doesn't have the specified row.
 312         return getEmptyOrNullToken(nCol, nRow);
 313     }
 314
 315     const RowDataType& rRowData = itrTable->second;
 316     RowDataType::const_iterator itrRow = rRowData.find(nCol);
 317     if (itrRow == rRowData.end())
 318     {
 319         // this row doesn't have the specified column.
 320         return getEmptyOrNullToken(nCol, nRow);
 321     }
 322
 323     const Cell& rCell = itrRow->second;
 324     if (pnFmtIndex)
 325         *pnFmtIndex = rCell.mnFmtIndex;
 326
 327     return rCell.mxToken;
 328 }
 329
 330 bool ScExternalRefCache::Table::hasRow( SCROW nRow ) const
 331 {
 332     RowsDataType::const_iterator itrRow = maRows.find(nRow);
 333     return itrRow != maRows.end();
 334 }
 335
 336 template< typename P >
 337 void ScExternalRefCache::Table::getAllRows(vector<SCROW>& rRows, P predicate) const
 338 {
 339     vector<SCROW> aRows;
 340     aRows.reserve(maRows.size());
 341     for (const auto& rEntry : maRows)
 342         if (predicate(rEntry))
 343             aRows.push_back(rEntry.first);
 344
 345     // hash map is not ordered, so we need to explicitly sort it.
 346     ::std::sort(aRows.begin(), aRows.end());
 347     rRows.swap(aRows);
 348 }
 349
 350 void ScExternalRefCache::Table::getAllRows(vector<SCROW>& rRows, SCROW nLow, SCROW nHigh) const
 351 {
 352     getAllRows(rRows,
 353         [nLow, nHigh](std::pair<SCROW, RowDataType> rEntry) { return (nLow <= rEntry.first && rEntry.first <= nHigh); });
 354 }
 355
 356 void ScExternalRefCache::Table::getAllRows(vector<SCROW>& rRows) const
 357 {
 358     getAllRows(rRows, [](std::pair<SCROW, RowDataType>) { return true; } );
 359 }
 360
 361 ::std::pair< SCROW, SCROW > ScExternalRefCache::Table::getRowRange() const
 362 {
 363     ::std::pair< SCROW, SCROW > aRange( 0, 0 );
 364     if( !maRows.empty() )
 365     {
 366         // iterate over entire container (hash map is not sorted by key)
 367         auto itMinMax = std::minmax_element(maRows.begin(), maRows.end(),
 368             [](const RowsDataType::value_type& a, const RowsDataType::value_type& b) { return a.first < b.first; });
 369         aRange.first = itMinMax.first->first;
 370         aRange.second = itMinMax.second->first + 1;
 371     }
 372     return aRange;
 373 }
 374
 375 template< typename P >
 376 void ScExternalRefCache::Table::getAllCols(SCROW nRow, vector<SCCOL>& rCols, P predicate) const
 377 {
 378     RowsDataType::const_iterator itrRow = maRows.find(nRow);
 379     if (itrRow == maRows.end())
 380         // this table doesn't have the specified row.
 381         return;
 382
 383     const RowDataType& rRowData = itrRow->second;
 384     vector<SCCOL> aCols;
 385     aCols.reserve(rRowData.size());
 386     for (const auto& rCol : rRowData)
 387         if (predicate(rCol))
 388             aCols.push_back(rCol.first);
 389
 390     // hash map is not ordered, so we need to explicitly sort it.
 391     ::std::sort(aCols.begin(), aCols.end());
 392     rCols.swap(aCols);
 393 }
 394
 395 void ScExternalRefCache::Table::getAllCols(SCROW nRow, vector<SCCOL>& rCols, SCCOL nLow, SCCOL nHigh) const
 396 {
 397     getAllCols(nRow, rCols,
 398         [nLow, nHigh](std::pair<SCCOL, Cell> rCol) { return nLow <= rCol.first && rCol.first <= nHigh; } );
 399 }
 400
 401 void ScExternalRefCache::Table::getAllCols(SCROW nRow, vector<SCCOL>& rCols) const
 402 {
 403     getAllCols(nRow, rCols, [](std::pair<SCCOL, Cell>) { return true; } );
 404 }
 405
 406 ::std::pair< SCCOL, SCCOL > ScExternalRefCache::Table::getColRange( SCROW nRow ) const
 407 {
 408     ::std::pair< SCCOL, SCCOL > aRange( 0, 0 );
 409
 410     RowsDataType::const_iterator itrRow = maRows.find( nRow );
 411     if (itrRow == maRows.end())
 412         // this table doesn't have the specified row.
 413         return aRange;
 414
 415     const RowDataType& rRowData = itrRow->second;
 416     if( !rRowData.empty() )
 417     {
 418         // iterate over entire container (hash map is not sorted by key)
 419         auto itMinMax = std::minmax_element(rRowData.begin(), rRowData.end(),
 420             [](const RowDataType::value_type& a, const RowDataType::value_type& b) { return a.first < b.first; });
 421         aRange.first = itMinMax.first->first;
 422         aRange.second = itMinMax.second->first + 1;
 423     }
 424     return aRange;
 425 }
 426
 427 void ScExternalRefCache::Table::getAllNumberFormats(vector<sal_uInt32>& rNumFmts) const
 428 {
 429     for (const auto& rRow : maRows)
 430     {
 431         const RowDataType& rRowData = rRow.second;
 432         for (const auto& rCol : rRowData)
 433         {
 434             const Cell& rCell = rCol.second;
 435             rNumFmts.push_back(rCell.mnFmtIndex);
 436         }
 437     }
 438 }
 439
 440 bool ScExternalRefCache::Table::isRangeCached(SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2) const
 441 {
 442     return maCachedRanges.Contains(ScRange(nCol1, nRow1, 0, nCol2, nRow2, 0));
 443 }
 444
 445 void ScExternalRefCache::Table::setCachedCell(SCCOL nCol, SCROW nRow)
 446 {
 447     setCachedCellRange(nCol, nRow, nCol, nRow);
 448 }
 449
 450 void ScExternalRefCache::Table::setCachedCellRange(SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2)
 451 {
 452     ScRange aRange(nCol1, nRow1, 0, nCol2, nRow2, 0);
 453     maCachedRanges.Join(aRange);
 454 }
 455
 456 void ScExternalRefCache::Table::setWholeTableCached()
 457 {
 458     setCachedCellRange(0, 0, MAXCOL, MAXROW);
 459 }
 460
 461 bool ScExternalRefCache::Table::isInCachedRanges(SCCOL nCol, SCROW nRow) const
 462 {
 463     return maCachedRanges.Contains(ScRange(nCol, nRow, 0, nCol, nRow, 0));
 464 }
 465
 466 ScExternalRefCache::TokenRef ScExternalRefCache::Table::getEmptyOrNullToken(
 467     SCCOL nCol, SCROW nRow) const
 468 {
 469     if (isInCachedRanges(nCol, nRow))
 470     {
 471         TokenRef p(new ScEmptyCellToken(false, false));
 472         return p;
 473     }
 474     return TokenRef();
 475 }
 476
 477 ScExternalRefCache::TableName::TableName(OUString aUpper, OUString aReal) :
 478     maUpperName(std::move(aUpper)), maRealName(std::move(aReal))
 479 {
 480 }
 481
 482 ScExternalRefCache::CellFormat::CellFormat() :
 483     mbIsSet(false), mnType(SvNumFormatType::ALL), mnIndex(0)
 484 {
 485 }
 486
 487 ScExternalRefCache::ScExternalRefCache(const ScDocument& rDoc)
 488     : mrDoc(rDoc)
 489 {
 490 }
 491
 492 ScExternalRefCache::~ScExternalRefCache() {}
 493
 494 const OUString* ScExternalRefCache::getRealTableName(sal_uInt16 nFileId, const OUString& rTabName) const
 495 {
 496     std::unique_lock aGuard(maMtxDocs);
 497
 498     DocDataType::const_iterator itrDoc = maDocs.find(nFileId);
 499     if (itrDoc == maDocs.end())
 500     {
 501         // specified document is not cached.
 502         return nullptr;
 503     }
 504
 505     const DocItem& rDoc = itrDoc->second;
 506     TableNameIndexMap::const_iterator itrTabId = rDoc.findTableNameIndex( rTabName);
 507     if (itrTabId == rDoc.maTableNameIndex.end())
 508     {
 509         // the specified table is not in cache.
 510         return nullptr;
 511     }
 512
 513     return &rDoc.maTableNames[itrTabId->second].maRealName;
 514 }
 515
 516 const OUString* ScExternalRefCache::getRealRangeName(sal_uInt16 nFileId, const OUString& rRangeName) const
 517 {
 518     std::unique_lock aGuard(maMtxDocs);
 519
 520     DocDataType::const_iterator itrDoc = maDocs.find(nFileId);
 521     if (itrDoc == maDocs.end())
 522     {
 523         // specified document is not cached.
 524         return nullptr;
 525     }
 526
 527     const DocItem& rDoc = itrDoc->second;
 528     NamePairMap::const_iterator itr = rDoc.maRealRangeNameMap.find(
 529         ScGlobal::getCharClass().uppercase(rRangeName));
 530     if (itr == rDoc.maRealRangeNameMap.end())
 531         // range name not found.
 532         return nullptr;
 533
 534     return &itr->second;
 535 }
 536
 537 ScExternalRefCache::TokenRef ScExternalRefCache::getCellData(
 538     sal_uInt16 nFileId, const OUString& rTabName, SCCOL nCol, SCROW nRow, sal_uInt32* pnFmtIndex)
 539 {
 540     std::unique_lock aGuard(maMtxDocs);
 541
 542     DocDataType::const_iterator itrDoc = maDocs.find(nFileId);
 543     if (itrDoc == maDocs.end())
 544     {
 545         // specified document is not cached.
 546         return TokenRef();
 547     }
 548
 549     const DocItem& rDoc = itrDoc->second;
 550     TableNameIndexMap::const_iterator itrTabId = rDoc.findTableNameIndex( rTabName);
 551     if (itrTabId == rDoc.maTableNameIndex.end())
 552     {
 553         // the specified table is not in cache.
 554         return TokenRef();
 555     }
 556
 557     const TableTypeRef& pTableData = rDoc.maTables[itrTabId->second];
 558     if (!pTableData)
 559     {
 560         // the table data is not instantiated yet.
 561         return TokenRef();
 562     }
 563
 564     return pTableData->getCell(nCol, nRow, pnFmtIndex);
 565 }
 566
 567 ScExternalRefCache::TokenArrayRef ScExternalRefCache::getCellRangeData(
 568     sal_uInt16 nFileId, const OUString& rTabName, const ScRange& rRange)
 569 {
 570     std::unique_lock aGuard(maMtxDocs);
 571
 572     DocDataType::iterator itrDoc = maDocs.find(nFileId);
 573     if (itrDoc == maDocs.end())
 574         // specified document is not cached.
 575         return TokenArrayRef();
 576
 577     DocItem& rDoc = itrDoc->second;
 578
 579     TableNameIndexMap::const_iterator itrTabId = rDoc.findTableNameIndex( rTabName);
 580     if (itrTabId == rDoc.maTableNameIndex.end())
 581         // the specified table is not in cache.
 582         return TokenArrayRef();
 583
 584     const ScAddress& s = rRange.aStart;
 585     const ScAddress& e = rRange.aEnd;
 586
 587     const SCTAB nTab1 = s.Tab(), nTab2 = e.Tab();
 588     const SCCOL nCol1 = s.Col(), nCol2 = e.Col();
 589     const SCROW nRow1 = s.Row(), nRow2 = e.Row();
 590
 591     // Make sure I have all the tables cached.
 592     size_t nTabFirstId = itrTabId->second;
 593     size_t nTabLastId  = nTabFirstId + nTab2 - nTab1;
 594     if (nTabLastId >= rDoc.maTables.size())
 595         // not all tables are cached.
 596         return TokenArrayRef();
 597
 598     ScRange aCacheRange( nCol1, nRow1, static_cast<SCTAB>(nTabFirstId), nCol2, nRow2, static_cast<SCTAB>(nTabLastId));
 599
 600     RangeArrayMap::const_iterator itrRange = rDoc.maRangeArrays.find( aCacheRange);
 601     if (itrRange != rDoc.maRangeArrays.end())
 602         // Cache hit!
 603         return itrRange->second;
 604
 605     std::unique_ptr<ScRange> pNewRange;
 606     TokenArrayRef pArray;
 607     bool bFirstTab = true;
 608     for (size_t nTab = nTabFirstId; nTab <= nTabLastId; ++nTab)
 609     {
 610         TableTypeRef pTab = rDoc.maTables[nTab];
 611         if (!pTab)
 612             return TokenArrayRef();
 613
 614         SCCOL nDataCol1 = nCol1, nDataCol2 = nCol2;
 615         SCROW nDataRow1 = nRow1, nDataRow2 = nRow2;
 616
 617         if (!pTab->isRangeCached(nDataCol1, nDataRow1, nDataCol2, nDataRow2))
 618         {
 619             // specified range is not entirely within cached ranges.
 620             return TokenArrayRef();
 621         }
 622
 623         SCSIZE nMatrixColumns = static_cast<SCSIZE>(nDataCol2-nDataCol1+1);
 624         SCSIZE nMatrixRows = static_cast<SCSIZE>(nDataRow2-nDataRow1+1);
 625         ScMatrixRef xMat = new ScMatrix( nMatrixColumns, nMatrixRows);
 626
 627         // Needed in shrink and fill.
 628         vector<SCROW> aRows;
 629         pTab->getAllRows(aRows, nDataRow1, nDataRow2);
 630         bool bFill = true;
 631
 632         // Check if size could be allocated and if not skip the fill, there's
 633         // one error element instead. But retry first with the actual data area
 634         // if that is smaller than the original range, which works for most
 635         // functions just not some that operate/compare with the original size
 636         // and expect empty values in non-data areas.
 637         // Restrict this though to ranges of entire columns or rows, other
 638         // ranges might be on purpose. (Other special cases to handle?)
 639         /* TODO: sparse matrix could help */
 640         SCSIZE nMatCols, nMatRows;
 641         xMat->GetDimensions( nMatCols, nMatRows);
 642         if (nMatCols != nMatrixColumns || nMatRows != nMatrixRows)
 643         {
 644             bFill = false;
 645             if (aRows.empty())
 646             {
 647                 // There's no data at all. Set the one matrix element to empty
 648                 // for column-repeated and row-repeated access.
 649                 xMat->PutEmpty(0,0);
 650             }
 651             else if ((nCol1 == 0 && nCol2 == MAXCOL) || (nRow1 == 0 && nRow2 == MAXROW))
 652             {
 653                 nDataRow1 = aRows.front();
 654                 nDataRow2 = aRows.back();
 655                 SCCOL nMinCol = std::numeric_limits<SCCOL>::max();
 656                 SCCOL nMaxCol = std::numeric_limits<SCCOL>::min();
 657                 for (const auto& rRow : aRows)
 658                 {
 659                     vector<SCCOL> aCols;
 660                     pTab->getAllCols(rRow, aCols, nDataCol1, nDataCol2);
 661                     if (!aCols.empty())
 662                     {
 663                         nMinCol = std::min( nMinCol, aCols.front());
 664                         nMaxCol = std::max( nMaxCol, aCols.back());
 665                     }
 666                 }
 667
 668                 if (nMinCol <= nMaxCol && ((o3tl::make_unsigned(nMaxCol-nMinCol+1) < nMatrixColumns) ||
 669                             (o3tl::make_unsigned(nDataRow2-nDataRow1+1) < nMatrixRows)))
 670                 {
 671                     nMatrixColumns = static_cast<SCSIZE>(nMaxCol-nMinCol+1);
 672                     nMatrixRows = static_cast<SCSIZE>(nDataRow2-nDataRow1+1);
 673                     xMat = new ScMatrix( nMatrixColumns, nMatrixRows);
 674                     xMat->GetDimensions( nMatCols, nMatRows);
 675                     if (nMatCols == nMatrixColumns && nMatRows == nMatrixRows)
 676                     {
 677                         nDataCol1 = nMinCol;
 678                         nDataCol2 = nMaxCol;
 679                         bFill = true;
 680                     }
 681                 }
 682             }
 683         }
 684
 685         if (bFill)
 686         {
 687             // Only fill non-empty cells, for better performance.
 688             for (SCROW nRow : aRows)
 689             {
 690                 vector<SCCOL> aCols;
 691                 pTab->getAllCols(nRow, aCols, nDataCol1, nDataCol2);
 692                 for (SCCOL nCol : aCols)
 693                 {
 694                     TokenRef pToken = pTab->getCell(nCol, nRow);
 695                     if (!pToken)
 696                         // This should never happen!
 697                         return TokenArrayRef();
 698
 699                     SCSIZE nC = nCol - nDataCol1, nR = nRow - nDataRow1;
 700                     switch (pToken->GetType())
 701                     {
 702                         case svDouble:
 703                             xMat->PutDouble(pToken->GetDouble(), nC, nR);
 704                             break;
 705                         case svString:
 706                             xMat->PutString(pToken->GetString(), nC, nR);
 707                             break;
 708                         default:
 709                             ;
 710                     }
 711                 }
 712             }
 713
 714             if (!bFirstTab)
 715                 pArray->AddOpCode(ocSep);
 716
 717             ScMatrixToken aToken(std::move(xMat));
 718             if (!pArray)
 719                 pArray = std::make_shared<ScTokenArray>(mrDoc);
 720             pArray->AddToken(aToken);
 721
 722             bFirstTab = false;
 723
 724             if (!pNewRange)
 725                 pNewRange.reset(new ScRange(nDataCol1, nDataRow1, nTab, nDataCol2, nDataRow2, nTab));
 726             else
 727                 pNewRange->ExtendTo(ScRange(nDataCol1, nDataRow1, nTab, nDataCol2, nDataRow2, nTab));
 728         }
 729     }
 730
 731     rDoc.maRangeArrays.emplace(aCacheRange, pArray);
 732     if (pNewRange && *pNewRange != aCacheRange)
 733         rDoc.maRangeArrays.emplace(*pNewRange, pArray);
 734
 735     return pArray;
 736 }
 737
 738 ScExternalRefCache::TokenArrayRef ScExternalRefCache::getRangeNameTokens(sal_uInt16 nFileId, const OUString& rName)
 739 {
 740     std::unique_lock aGuard(maMtxDocs);
 741
 742     DocItem* pDoc = getDocItem(aGuard, nFileId);
 743     if (!pDoc)
 744         return TokenArrayRef();
 745
 746     RangeNameMap& rMap = pDoc->maRangeNames;
 747     RangeNameMap::const_iterator itr = rMap.find(
 748         ScGlobal::getCharClass().uppercase(rName));
 749     if (itr == rMap.end())
 750         return TokenArrayRef();
 751
 752     return itr->second;
 753 }
 754
 755 void ScExternalRefCache::setRangeNameTokens(sal_uInt16 nFileId, const OUString& rName, const TokenArrayRef& pArray)
 756 {
 757     std::unique_lock aGuard(maMtxDocs);
 758
 759     DocItem* pDoc = getDocItem(aGuard, nFileId);
 760     if (!pDoc)
 761         return;
 762
 763     OUString aUpperName = ScGlobal::getCharClass().uppercase(rName);
 764     RangeNameMap& rMap = pDoc->maRangeNames;
 765     rMap.emplace(aUpperName, pArray);
 766     pDoc->maRealRangeNameMap.emplace(aUpperName, rName);
 767 }
 768
 769 bool ScExternalRefCache::isValidRangeName(sal_uInt16 nFileId, const OUString& rName) const
 770 {
 771     std::unique_lock aGuard(maMtxDocs);
 772
 773     DocItem* pDoc = getDocItem(aGuard, nFileId);
 774     if (!pDoc)
 775         return false;
 776
 777     OUString aUpperName = ScGlobal::getCharClass().uppercase(rName);
 778     const RangeNameMap& rMap = pDoc->maRangeNames;
 779     return rMap.count(aUpperName) > 0;
 780 }
 781
 782 void ScExternalRefCache::setRangeName(sal_uInt16 nFileId, const OUString& rName)
 783 {
 784     std::unique_lock aGuard(maMtxDocs);
 785
 786     DocItem* pDoc = getDocItem(aGuard, nFileId);
 787     if (!pDoc)
 788         return;
 789
 790     OUString aUpperName = ScGlobal::getCharClass().uppercase(rName);
 791     pDoc->maRealRangeNameMap.emplace(aUpperName, rName);
 792 }
 793
 794 void ScExternalRefCache::setCellData(sal_uInt16 nFileId, const OUString& rTabName, SCCOL nCol, SCROW nRow,
 795                                      TokenRef const & pToken, sal_uLong nFmtIndex)
 796 {
 797     if (!isDocInitialized(nFileId))
 798         return;
 799
 800     using ::std::pair;
 801     DocItem* pDocItem = getDocItem(nFileId);
 802     if (!pDocItem)
 803         return;
 804
 805     DocItem& rDoc = *pDocItem;
 806
 807     // See if the table by this name already exists.
 808     TableNameIndexMap::const_iterator itrTabName = rDoc.findTableNameIndex( rTabName);
 809     if (itrTabName == rDoc.maTableNameIndex.end())
 810         // Table not found.  Maybe the table name or the file id is wrong ???
 811         return;
 812
 813     TableTypeRef& pTableData = rDoc.maTables[itrTabName->second];
 814     if (!pTableData)
 815         pTableData = std::make_shared<Table>();
 816
 817     pTableData->setCell(nCol, nRow, pToken, nFmtIndex);
 818     pTableData->setCachedCell(nCol, nRow);
 819 }
 820
 821 void ScExternalRefCache::setCellRangeData(sal_uInt16 nFileId, const ScRange& rRange, const vector<SingleRangeData>& rData,
 822                                           const TokenArrayRef& pArray)
 823 {
 824     using ::std::pair;
 825     if (rData.empty() || !isDocInitialized(nFileId))
 826         // nothing to cache
 827         return;
 828
 829     // First, get the document item for the given file ID.
 830     DocItem* pDocItem = getDocItem(nFileId);
 831     if (!pDocItem)
 832         return;
 833
 834     DocItem& rDoc = *pDocItem;
 835
 836     // Now, find the table position of the first table to cache.
 837     const OUString& rFirstTabName = rData.front().maTableName;
 838     TableNameIndexMap::const_iterator itrTabName = rDoc.findTableNameIndex( rFirstTabName);
 839     if (itrTabName == rDoc.maTableNameIndex.end())
 840     {
 841         // table index not found.
 842         return;
 843     }
 844
 845     size_t nTabFirstId = itrTabName->second;
 846     SCROW nRow1 = rRange.aStart.Row(), nRow2 = rRange.aEnd.Row();
 847     SCCOL nCol1 = rRange.aStart.Col(), nCol2 = rRange.aEnd.Col();
 848     size_t i = nTabFirstId;
 849     for (const auto& rItem : rData)
 850     {
 851         TableTypeRef& pTabData = rDoc.maTables[i];
 852         if (!pTabData)
 853             pTabData = std::make_shared<Table>();
 854
 855         const ScMatrixRef& pMat = rItem.mpRangeData;
 856         SCSIZE nMatCols, nMatRows;
 857         pMat->GetDimensions( nMatCols, nMatRows);
 858         if (nMatCols > o3tl::make_unsigned(nCol2 - nCol1) && nMatRows > o3tl::make_unsigned(nRow2 - nRow1))
 859         {
 860             ScMatrix::DoubleOpFunction aDoubleFunc = [=](size_t row, size_t col, double val) -> void
 861             {
 862                 pTabData->setCell(col + nCol1, row + nRow1, new formula::FormulaDoubleToken(val), 0, false);
 863             };
 864             ScMatrix::BoolOpFunction aBoolFunc = [=](size_t row, size_t col, bool val) -> void
 865             {
 866                 pTabData->setCell(col + nCol1, row + nRow1, new formula::FormulaDoubleToken(val ? 1.0 : 0.0), 0, false);
 867             };
 868             ScMatrix::StringOpFunction aStringFunc = [=](size_t row, size_t col, svl::SharedString val) -> void
 869             {
 870                 pTabData->setCell(col + nCol1, row + nRow1, new formula::FormulaStringToken(std::move(val)), 0, false);
 871             };
 872             ScMatrix::EmptyOpFunction aEmptyFunc = [](size_t /*row*/, size_t /*col*/) -> void
 873             {
 874                 // Nothing. Empty cell.
 875             };
 876             pMat->ExecuteOperation(std::pair<size_t, size_t>(0, 0),
 877                     std::pair<size_t, size_t>(nRow2-nRow1, nCol2-nCol1),
 878                     std::move(aDoubleFunc), std::move(aBoolFunc), std::move(aStringFunc), std::move(aEmptyFunc));
 879             // Mark the whole range 'cached'.
 880             pTabData->setCachedCellRange(nCol1, nRow1, nCol2, nRow2);
 881         }
 882         else
 883         {
 884             // This may happen due to a matrix not been allocated earlier, in
 885             // which case it should have exactly one error element.
 886             SAL_WARN("sc.ui","ScExternalRefCache::setCellRangeData - matrix size mismatch");
 887             if (nMatCols != 1 || nMatRows != 1)
 888                 SAL_WARN("sc.ui","ScExternalRefCache::setCellRangeData - not a one element matrix");
 889             else
 890             {
 891                 FormulaError nErr = GetDoubleErrorValue( pMat->GetDouble(0,0));
 892                 SAL_WARN("sc.ui","ScExternalRefCache::setCellRangeData - matrix error value is " << static_cast<int>(nErr) <<
 893                         (nErr == FormulaError::MatrixSize ? ", ok" : ", not ok"));
 894             }
 895         }
 896         ++i;
 897     }
 898
 899     size_t nTabLastId = nTabFirstId + rRange.aEnd.Tab() - rRange.aStart.Tab();
 900     ScRange aCacheRange( nCol1, nRow1, static_cast<SCTAB>(nTabFirstId), nCol2, nRow2, static_cast<SCTAB>(nTabLastId));
 901
 902     rDoc.maRangeArrays.emplace(aCacheRange, pArray);
 903 }
 904
 905 bool ScExternalRefCache::isDocInitialized(sal_uInt16 nFileId)
 906 {
 907     DocItem* pDoc = getDocItem(nFileId);
 908     if (!pDoc)
 909         return false;
 910
 911     return pDoc->mbInitFromSource;
 912 }
 913
 914 static bool lcl_getStrictTableDataIndex(const ScExternalRefCache::TableNameIndexMap& rMap, const OUString& rName, size_t& rIndex)
 915 {
 916     ScExternalRefCache::TableNameIndexMap::const_iterator itr = rMap.find(rName);
 917     if (itr == rMap.end())
 918         return false;
 919
 920     rIndex = itr->second;
 921     return true;
 922 }
 923
 924 bool ScExternalRefCache::DocItem::getTableDataIndex( const OUString& rTabName, size_t& rIndex ) const
 925 {
 926     ScExternalRefCache::TableNameIndexMap::const_iterator itr = findTableNameIndex(rTabName);
 927     if (itr == maTableNameIndex.end())
 928         return false;
 929
 930     rIndex = itr->second;
 931     return true;
 932 }
 933
 934 namespace {
 935 OUString getFirstSheetName()
 936 {
 937     // Get Custom prefix.
 938     const ScDefaultsOptions& rOpt = ScModule::get()->GetDefaultsOptions();
 939     // Form sheet name identical to the first generated sheet name when
 940     // creating an internal document, e.g. 'Sheet1'.
 941     return rOpt.GetInitTabPrefix() + "1";
 942 }
 943 }
 944
 945 void ScExternalRefCache::initializeDoc(sal_uInt16 nFileId, const vector<OUString>& rTabNames,
 946         const OUString& rBaseName)
 947 {
 948     DocItem* pDoc = getDocItem(nFileId);
 949     if (!pDoc)
 950         return;
 951
 952     size_t n = rTabNames.size();
 953
 954     // table name list - the list must include all table names in the source
 955     // document and only to be populated when loading the source document, not
 956     // when loading cached data from, say, Excel XCT/CRN records.
 957     vector<TableName> aNewTabNames;
 958     aNewTabNames.reserve(n);
 959     for (const auto& rTabName : rTabNames)
 960     {
 961         TableName aNameItem(ScGlobal::getCharClass().uppercase(rTabName), rTabName);
 962         aNewTabNames.push_back(aNameItem);
 963     }
 964     pDoc->maTableNames.swap(aNewTabNames);
 965
 966     // data tables - preserve any existing data that may have been set during
 967     // file import.
 968     vector<TableTypeRef> aNewTables(n);
 969     for (size_t i = 0; i < n; ++i)
 970     {
 971         size_t nIndex;
 972         if (lcl_getStrictTableDataIndex(pDoc->maTableNameIndex, pDoc->maTableNames[i].maUpperName, nIndex))
 973         {
 974             aNewTables[i] = pDoc->maTables[nIndex];
 975         }
 976     }
 977     pDoc->maTables.swap(aNewTables);
 978
 979     // name index map
 980     TableNameIndexMap aNewNameIndex;
 981     for (size_t i = 0; i < n; ++i)
 982         aNewNameIndex.emplace(pDoc->maTableNames[i].maUpperName, i);
 983     pDoc->maTableNameIndex.swap(aNewNameIndex);
 984
 985     // Setup name for Sheet1 vs base name to be able to load documents
 986     // that store the base name as table name, or vice versa.
 987     pDoc->maSingleTableNameAlias.clear();
 988     if (!rBaseName.isEmpty() && pDoc->maTableNames.size() == 1)
 989     {
 990         OUString aSheetName = getFirstSheetName();
 991         // If the one and only table name matches exactly, carry on the base
 992         // file name for further alias use. If instead the table name matches
 993         // the base name, carry on the sheet name as alias.
 994         if (ScGlobal::GetTransliteration().isEqual( pDoc->maTableNames[0].maRealName, aSheetName))
 995             pDoc->maSingleTableNameAlias = rBaseName;
 996         else if (ScGlobal::GetTransliteration().isEqual( pDoc->maTableNames[0].maRealName, rBaseName))
 997             pDoc->maSingleTableNameAlias = aSheetName;
 998     }
 999
1000     pDoc->mbInitFromSource = true;
1001 }
1002
1003 ScExternalRefCache::TableNameIndexMap::const_iterator ScExternalRefCache::DocItem::findTableNameIndex(
1004         const OUString& rTabName ) const
1005 {
1006     const OUString aTabNameUpper = ScGlobal::getCharClass().uppercase( rTabName);
1007     TableNameIndexMap::const_iterator itrTabName = maTableNameIndex.find( aTabNameUpper);
1008     if (itrTabName != maTableNameIndex.end())
1009         return itrTabName;
1010
1011     // Since some time for external references to CSV files the base name is
1012     // used as sheet name instead of Sheet1, check if we can resolve that.
1013     // Also helps users that got accustomed to one or the other way.
1014     if (maSingleTableNameAlias.isEmpty() || maTableNameIndex.size() != 1)
1015         return itrTabName;
1016
1017     // maSingleTableNameAlias has been set up only if the original file loaded
1018     // had exactly one sheet and internal sheet name was Sheet1 or localized or
1019     // customized equivalent, or base name.
1020     if (aTabNameUpper == ScGlobal::getCharClass().uppercase( maSingleTableNameAlias))
1021         return maTableNameIndex.begin();
1022
1023     return itrTabName;
1024 }
1025
1026 bool ScExternalRefCache::DocItem::getSingleTableNameAlternative( OUString& rTabName ) const
1027 {
1028     if (maSingleTableNameAlias.isEmpty() || maTableNames.size() != 1)
1029         return false;
1030     if (ScGlobal::GetTransliteration().isEqual( rTabName, maTableNames[0].maRealName))
1031     {
1032         rTabName = maSingleTableNameAlias;
1033         return true;
1034     }
1035     if (ScGlobal::GetTransliteration().isEqual( rTabName, maSingleTableNameAlias))
1036     {
1037         rTabName = maTableNames[0].maRealName;
1038         return true;
1039     }
1040     return false;
1041 }
1042
1043 bool ScExternalRefCache::getSrcDocTable( const ScDocument& rSrcDoc, const OUString& rTabName, SCTAB& rTab,
1044         sal_uInt16 nFileId ) const
1045 {
1046     bool bFound = rSrcDoc.GetTable( rTabName, rTab);
1047     if (!bFound)
1048     {
1049         // Check the one table alias alternative.
1050         const DocItem* pDoc = getDocItem( nFileId );
1051         if (pDoc)
1052         {
1053             OUString aTabName( rTabName);
1054             if (pDoc->getSingleTableNameAlternative( aTabName))
1055                 bFound = rSrcDoc.GetTable( aTabName, rTab);
1056         }
1057     }
1058     return bFound;
1059 }
1060
1061 OUString ScExternalRefCache::getTableName(sal_uInt16 nFileId, size_t nCacheId) const
1062 {
1063     if( DocItem* pDoc = getDocItem( nFileId ) )
1064         if( nCacheId < pDoc->maTableNames.size() )
1065             return pDoc->maTableNames[ nCacheId ].maRealName;
1066     return OUString();
1067 }
1068
1069 void ScExternalRefCache::getAllTableNames(sal_uInt16 nFileId, vector<OUString>& rTabNames) const
1070 {
1071     rTabNames.clear();
1072     DocItem* pDoc = getDocItem(nFileId);
1073     if (!pDoc)
1074         return;
1075
1076     size_t n = pDoc->maTableNames.size();
1077     rTabNames.reserve(n);
1078     for (const auto& rTableName : pDoc->maTableNames)
1079         rTabNames.push_back(rTableName.maRealName);
1080 }
1081
1082 SCTAB ScExternalRefCache::getTabSpan( sal_uInt16 nFileId, const OUString& rStartTabName, const OUString& rEndTabName ) const
1083 {
1084     DocItem* pDoc = getDocItem(nFileId);
1085     if (!pDoc)
1086         return -1;
1087
1088     vector<TableName>::const_iterator itrBeg = pDoc->maTableNames.begin();
1089     vector<TableName>::const_iterator itrEnd = pDoc->maTableNames.end();
1090
1091     vector<TableName>::const_iterator itrStartTab = ::std::find_if( itrBeg, itrEnd,
1092             TabNameSearchPredicate( rStartTabName));
1093     if (itrStartTab == itrEnd)
1094         return -1;
1095
1096     vector<TableName>::const_iterator itrEndTab = ::std::find_if( itrBeg, itrEnd,
1097             TabNameSearchPredicate( rEndTabName));
1098     if (itrEndTab == itrEnd)
1099         return 0;
1100
1101     size_t nStartDist = ::std::distance( itrBeg, itrStartTab);
1102     size_t nEndDist = ::std::distance( itrBeg, itrEndTab);
1103     return nStartDist <= nEndDist ? static_cast<SCTAB>(nEndDist - nStartDist + 1) : -static_cast<SCTAB>(nStartDist - nEndDist + 1);
1104 }
1105
1106 void ScExternalRefCache::getAllNumberFormats(vector<sal_uInt32>& rNumFmts) const
1107 {
1108     std::unique_lock aGuard(maMtxDocs);
1109
1110     using ::std::sort;
1111     using ::std::unique;
1112
1113     vector<sal_uInt32> aNumFmts;
1114     for (const auto& rEntry : maDocs)
1115     {
1116         const vector<TableTypeRef>& rTables = rEntry.second.maTables;
1117         for (const TableTypeRef& pTab : rTables)
1118         {
1119             if (!pTab)
1120                 continue;
1121
1122             pTab->getAllNumberFormats(aNumFmts);
1123         }
1124     }
1125
1126     // remove duplicates.
1127     sort(aNumFmts.begin(), aNumFmts.end());
1128     aNumFmts.erase(unique(aNumFmts.begin(), aNumFmts.end()), aNumFmts.end());
1129     rNumFmts.swap(aNumFmts);
1130 }
1131
1132 bool ScExternalRefCache::setCacheDocReferenced( sal_uInt16 nFileId )
1133 {
1134     DocItem* pDocItem = getDocItem(nFileId);
1135     if (!pDocItem)
1136         return areAllCacheTablesReferenced();
1137
1138     for (auto& rxTab : pDocItem->maTables)
1139     {
1140         if (rxTab)
1141             rxTab->setReferenced(true);
1142     }
1143     addCacheDocToReferenced( nFileId);
1144     return areAllCacheTablesReferenced();
1145 }
1146
1147 bool ScExternalRefCache::setCacheTableReferenced( sal_uInt16 nFileId, const OUString& rTabName, size_t nSheets )
1148 {
1149     DocItem* pDoc = getDocItem(nFileId);
1150     if (pDoc)
1151     {
1152         size_t nIndex = 0;
1153         if (pDoc->getTableDataIndex( rTabName, nIndex))
1154         {
1155             size_t nStop = ::std::min( nIndex + nSheets, pDoc->maTables.size());
1156             for (size_t i = nIndex; i < nStop; ++i)
1157             {
1158                 TableTypeRef pTab = pDoc->maTables[i];
1159                 if (pTab)
1160                 {
1161                     if (!pTab->isReferenced())
1162                     {
1163                         pTab->setReferenced(true);
1164                         addCacheTableToReferenced( nFileId, i);
1165                     }
1166                 }
1167             }
1168         }
1169     }
1170     return areAllCacheTablesReferenced();
1171 }
1172
1173 void ScExternalRefCache::setAllCacheTableReferencedStati( bool bReferenced )
1174 {
1175     std::unique_lock aGuard(maMtxDocs);
1176
1177     if (bReferenced)
1178     {
1179         maReferenced.reset(0);
1180         for (auto& rEntry : maDocs)
1181         {
1182             ScExternalRefCache::DocItem& rDocItem = rEntry.second;
1183             for (auto& rxTab : rDocItem.maTables)
1184             {
1185                 if (rxTab)
1186                     rxTab->setReferenced(true);
1187             }
1188         }
1189     }
1190     else
1191     {
1192         size_t nDocs = 0;
1193         auto itrMax = std::max_element(maDocs.begin(), maDocs.end(),
1194             [](const DocDataType::value_type& a, const DocDataType::value_type& b) { return a.first < b.first; });
1195         if (itrMax != maDocs.end())
1196             nDocs = itrMax->first + 1;
1197         maReferenced.reset( nDocs);
1198
1199         for (auto& [nFileId, rDocItem] : maDocs)
1200         {
1201             size_t nTables = rDocItem.maTables.size();
1202             ReferencedStatus::DocReferenced & rDocReferenced = maReferenced.maDocs[nFileId];
1203             // All referenced => non-existing tables evaluate as completed.
1204             rDocReferenced.maTables.resize( nTables, true);
1205             for (size_t i=0; i < nTables; ++i)
1206             {
1207                 TableTypeRef & xTab = rDocItem.maTables[i];
1208                 if (xTab)
1209                 {
1210                     xTab->setReferenced(false);
1211                     rDocReferenced.maTables[i] = false;
1212                     rDocReferenced.mbAllTablesReferenced = false;
1213                     // An addCacheTableToReferenced() actually may have
1214                     // resulted in mbAllReferenced been set. Clear it.
1215                     maReferenced.mbAllReferenced = false;
1216                 }
1217             }
1218         }
1219     }
1220 }
1221
1222 void ScExternalRefCache::addCacheTableToReferenced( sal_uInt16 nFileId, size_t nIndex )
1223 {
1224     if (nFileId >= maReferenced.maDocs.size())
1225         return;
1226
1227     ::std::vector<bool> & rTables = maReferenced.maDocs[nFileId].maTables;
1228     size_t nTables = rTables.size();
1229     if (nIndex >= nTables)
1230         return;
1231
1232     if (!rTables[nIndex])
1233     {
1234         rTables[nIndex] = true;
1235         size_t i = 0;
1236         while (i < nTables && rTables[i])
1237             ++i;
1238         if (i == nTables)
1239         {
1240             maReferenced.maDocs[nFileId].mbAllTablesReferenced = true;
1241             maReferenced.checkAllDocs();
1242         }
1243     }
1244 }
1245
1246 void ScExternalRefCache::addCacheDocToReferenced( sal_uInt16 nFileId )
1247 {
1248     if (nFileId >= maReferenced.maDocs.size())
1249         return;
1250
1251     if (!maReferenced.maDocs[nFileId].mbAllTablesReferenced)
1252     {
1253         ::std::vector<bool> & rTables = maReferenced.maDocs[nFileId].maTables;
1254         size_t nSize = rTables.size();
1255         for (size_t i=0; i < nSize; ++i)
1256             rTables[i] = true;
1257         maReferenced.maDocs[nFileId].mbAllTablesReferenced = true;
1258         maReferenced.checkAllDocs();
1259     }
1260 }
1261
1262 void ScExternalRefCache::getAllCachedDataSpans( const ScDocument& rSrcDoc, sal_uInt16 nFileId, sc::ColumnSpanSet& rSet ) const
1263 {
1264     const DocItem* pDocItem = getDocItem(nFileId);
1265     if (!pDocItem)
1266         // This document is not cached.
1267         return;
1268
1269     const std::vector<TableTypeRef>& rTables = pDocItem->maTables;
1270     for (size_t nTab = 0, nTabCount = rTables.size(); nTab < nTabCount; ++nTab)
1271     {
1272         TableTypeRef pTab = rTables[nTab];
1273         if (!pTab)
1274             continue;
1275
1276         std::vector<SCROW> aRows;
1277         pTab->getAllRows(aRows);
1278         for (SCROW nRow : aRows)
1279         {
1280             std::vector<SCCOL> aCols;
1281             pTab->getAllCols(nRow, aCols);
1282             for (SCCOL nCol : aCols)
1283             {
1284                 rSet.set(rSrcDoc, nTab, nCol, nRow, true);
1285             }
1286         }
1287     }
1288 }
1289
1290 ScExternalRefCache::ReferencedStatus::ReferencedStatus() :
1291     mbAllReferenced(false)
1292 {
1293     reset(0);
1294 }
1295
1296 void ScExternalRefCache::ReferencedStatus::reset( size_t nDocs )
1297 {
1298     if (nDocs)
1299     {
1300         mbAllReferenced = false;
1301         DocReferencedVec aRefs( nDocs);
1302         maDocs.swap( aRefs);
1303     }
1304     else
1305     {
1306         mbAllReferenced = true;
1307         DocReferencedVec aRefs;
1308         maDocs.swap( aRefs);
1309     }
1310 }
1311
1312 void ScExternalRefCache::ReferencedStatus::checkAllDocs()
1313 {
1314     if (std::all_of(maDocs.begin(), maDocs.end(), [](const DocReferenced& rDoc) { return rDoc.mbAllTablesReferenced; }))
1315         mbAllReferenced = true;
1316 }
1317
1318 ScExternalRefCache::TableTypeRef ScExternalRefCache::getCacheTable(sal_uInt16 nFileId, size_t nTabIndex) const
1319 {
1320     DocItem* pDoc = getDocItem(nFileId);
1321     if (!pDoc || nTabIndex >= pDoc->maTables.size())
1322         return TableTypeRef();
1323
1324     return pDoc->maTables[nTabIndex];
1325 }
1326
1327 ScExternalRefCache::TableTypeRef ScExternalRefCache::getCacheTable(sal_uInt16 nFileId, const OUString& rTabName,
1328         bool bCreateNew, size_t* pnIndex, const OUString* pExtUrl)
1329 {
1330     // In API, the index is transported as cached sheet ID of type sal_Int32 in
1331     // sheet::SingleReference.Sheet or sheet::ComplexReference.Reference1.Sheet
1332     // in a sheet::FormulaToken, choose a sensible value for N/A. Effectively
1333     // being 0xffffffff
1334     const size_t nNotAvailable = static_cast<size_t>( static_cast<sal_Int32>( -1));
1335
1336     DocItem* pDoc = getDocItem(nFileId);
1337     if (!pDoc)
1338     {
1339         if (pnIndex) *pnIndex = nNotAvailable;
1340         return TableTypeRef();
1341     }
1342
1343     DocItem& rDoc = *pDoc;
1344
1345     size_t nIndex;
1346     if (rDoc.getTableDataIndex(rTabName, nIndex))
1347     {
1348         // specified table found.
1349         if( pnIndex ) *pnIndex = nIndex;
1350         if (bCreateNew && !rDoc.maTables[nIndex])
1351             rDoc.maTables[nIndex] = std::make_shared<Table>();
1352
1353         return rDoc.maTables[nIndex];
1354     }
1355
1356     if (!bCreateNew)
1357     {
1358         if (pnIndex) *pnIndex = nNotAvailable;
1359         return TableTypeRef();
1360     }
1361
1362     // If this is the first table to be created propagate the base name or
1363     // Sheet1 as an alias. For subsequent tables remove it again.
1364     if (rDoc.maTableNames.empty())
1365     {
1366         if (pExtUrl)
1367         {
1368             const OUString aBaseName( INetURLObject( *pExtUrl).GetBase());
1369             const OUString aSheetName( getFirstSheetName());
1370             if (ScGlobal::GetTransliteration().isEqual( rTabName, aSheetName))
1371                 pDoc->maSingleTableNameAlias = aBaseName;
1372             else if (ScGlobal::GetTransliteration().isEqual( rTabName, aBaseName))
1373                 pDoc->maSingleTableNameAlias = aSheetName;
1374         }
1375     }
1376     else
1377     {
1378         rDoc.maSingleTableNameAlias.clear();
1379     }
1380
1381     // Specified table doesn't exist yet.  Create one.
1382     OUString aTabNameUpper = ScGlobal::getCharClass().uppercase(rTabName);
1383     nIndex = rDoc.maTables.size();
1384     if( pnIndex ) *pnIndex = nIndex;
1385     TableTypeRef pTab = std::make_shared<Table>();
1386     rDoc.maTables.push_back(pTab);
1387     rDoc.maTableNames.emplace_back(aTabNameUpper, rTabName);
1388     rDoc.maTableNameIndex.emplace(aTabNameUpper, nIndex);
1389     return pTab;
1390 }
1391
1392 void ScExternalRefCache::clearCache(sal_uInt16 nFileId)
1393 {
1394     std::unique_lock aGuard(maMtxDocs);
1395     maDocs.erase(nFileId);
1396 }
1397
1398 void ScExternalRefCache::clearCacheTables(sal_uInt16 nFileId)
1399 {
1400     std::unique_lock aGuard(maMtxDocs);
1401     DocItem* pDocItem = getDocItem(aGuard, nFileId);
1402     if (!pDocItem)
1403         // This document is not cached at all.
1404         return;
1405
1406     // Clear all cache table content, but keep the tables.
1407     std::vector<TableTypeRef>& rTabs = pDocItem->maTables;
1408     for (TableTypeRef & pTab : rTabs)
1409     {
1410         if (!pTab)
1411             continue;
1412
1413         pTab->clear();
1414     }
1415
1416     // Clear the external range name caches.
1417     pDocItem->maRangeNames.clear();
1418     pDocItem->maRangeArrays.clear();
1419     pDocItem->maRealRangeNameMap.clear();
1420 }
1421
1422 ScExternalRefCache::DocItem* ScExternalRefCache::getDocItem(sal_uInt16 nFileId) const
1423 {
1424     std::unique_lock aGuard(maMtxDocs);
1425     return getDocItem(aGuard, nFileId);
1426 }
1427
1428 ScExternalRefCache::DocItem* ScExternalRefCache::getDocItem(std::unique_lock<std::mutex>& /*rGuard*/, sal_uInt16 nFileId) const
1429 {
1430
1431     using ::std::pair;
1432     DocDataType::iterator itrDoc = maDocs.find(nFileId);
1433     if (itrDoc == maDocs.end())
1434     {
1435         // specified document is not cached.
1436         pair<DocDataType::iterator, bool> res = maDocs.emplace(
1437                 nFileId, DocItem());
1438
1439         if (!res.second)
1440             // insertion failed.
1441             return nullptr;
1442
1443         itrDoc = res.first;
1444     }
1445
1446     return &itrDoc->second;
1447 }
1448
1449 ScExternalRefLink::ScExternalRefLink(ScDocument& rDoc, sal_uInt16 nFileId) :
1450     ::sfx2::SvBaseLink(::SfxLinkUpdateMode::ONCALL, SotClipboardFormatId::SIMPLE_FILE),
1451     mnFileId(nFileId),
1452     mrDoc(rDoc),
1453     mbDoRefresh(true)
1454 {
1455 }
1456
1457 ScExternalRefLink::~ScExternalRefLink()
1458 {
1459 }
1460
1461 void ScExternalRefLink::Closed()
1462 {
1463     ScExternalRefManager* pMgr = mrDoc.GetExternalRefManager();
1464     pMgr->breakLink(mnFileId);
1465 }
1466
1467 ::sfx2::SvBaseLink::UpdateResult ScExternalRefLink::DataChanged(const OUString& /*rMimeType*/, const Any& /*rValue*/)
1468 {
1469     if (!mbDoRefresh)
1470         return SUCCESS;
1471
1472     OUString aFile, aFilter;
1473     sfx2::LinkManager::GetDisplayNames(this, nullptr, &aFile, nullptr, &aFilter);
1474     ScExternalRefManager* pMgr = mrDoc.GetExternalRefManager();
1475
1476     if (!pMgr->isFileLoadable(aFile))
1477         return ERROR_GENERAL;
1478
1479     const OUString* pCurFile = pMgr->getExternalFileName(mnFileId);
1480     if (!pCurFile)
1481         return ERROR_GENERAL;
1482
1483     if (*pCurFile == aFile)
1484     {
1485         // Refresh the current source document.
1486         if (!pMgr->refreshSrcDocument(mnFileId))
1487             return ERROR_GENERAL;
1488     }
1489     else
1490     {
1491         // The source document has changed.
1492         ScViewData* pViewData = ScDocShell::GetViewData();
1493         if (!pViewData)
1494             return ERROR_GENERAL;
1495
1496         ScDocShell* pDocShell = pViewData->GetDocShell();
1497         ScDocShellModificator aMod(*pDocShell);
1498         pMgr->switchSrcFile(mnFileId, aFile, aFilter);
1499         aMod.SetDocumentModified();
1500     }
1501
1502     return SUCCESS;
1503 }
1504
1505 void ScExternalRefLink::Edit(weld::Window* pParent, const Link<SvBaseLink&,void>& /*rEndEditHdl*/)
1506 {
1507     SvBaseLink::Edit(pParent, Link<SvBaseLink&,void>());
1508 }
1509
1510 void ScExternalRefLink::SetDoRefresh(bool b)
1511 {
1512     mbDoRefresh = b;
1513 }
1514
1515 static FormulaToken* convertToToken( ScDocument& rHostDoc, const ScDocument& rSrcDoc, ScRefCellValue& rCell )
1516 {
1517     if (rCell.hasEmptyValue())
1518     {
1519         bool bInherited = (rCell.getType() == CELLTYPE_FORMULA);
1520         return new ScEmptyCellToken(bInherited, false);
1521     }
1522
1523     switch (rCell.getType())
1524     {
1525         case CELLTYPE_EDIT:
1526         case CELLTYPE_STRING:
1527         {
1528             OUString aStr = rCell.getString(&rSrcDoc);
1529             svl::SharedString aSS = rHostDoc.GetSharedStringPool().intern(aStr);
1530             return new formula::FormulaStringToken(std::move(aSS));
1531         }
1532         case CELLTYPE_VALUE:
1533             return new formula::FormulaDoubleToken(rCell.getDouble());
1534         case CELLTYPE_FORMULA:
1535         {
1536             ScFormulaCell* pFCell = rCell.getFormula();
1537             FormulaError nError = pFCell->GetErrCode();
1538             if (nError != FormulaError::NONE)
1539                 return new FormulaErrorToken( nError);
1540             else if (pFCell->IsValue())
1541             {
1542                 double fVal = pFCell->GetValue();
1543                 return new formula::FormulaDoubleToken(fVal);
1544             }
1545             else
1546             {
1547                 svl::SharedString aSS = rHostDoc.GetSharedStringPool().intern( pFCell->GetString().getString());
1548                 return new formula::FormulaStringToken(std::move(aSS));
1549             }
1550         }
1551         default:
1552             OSL_FAIL("attempted to convert an unknown cell type.");
1553     }
1554
1555     return nullptr;
1556 }
1557
1558 static std::unique_ptr<ScTokenArray> convertToTokenArray(
1559     ScDocument& rHostDoc, const ScDocument& rSrcDoc, ScRange& rRange, vector<ScExternalRefCache::SingleRangeData>& rCacheData )
1560 {
1561     ScAddress& s = rRange.aStart;
1562     ScAddress& e = rRange.aEnd;
1563
1564     const SCTAB nTab1 = s.Tab(), nTab2 = e.Tab();
1565     const SCCOL nCol1 = s.Col(), nCol2 = e.Col();
1566     const SCROW nRow1 = s.Row(), nRow2 = e.Row();
1567
1568     if (nTab2 != nTab1)
1569         // For now, we don't support multi-sheet ranges intentionally because
1570         // we don't have a way to express them in a single token.  In the
1571         // future we can introduce a new stack variable type svMatrixList with
1572         // a new token type that can store a 3D matrix value and convert a 3D
1573         // range to it.
1574         return nullptr;
1575
1576     std::unique_ptr<ScRange> pUsedRange;
1577
1578     unique_ptr<ScTokenArray> pArray(new ScTokenArray(rSrcDoc));
1579     bool bFirstTab = true;
1580     vector<ScExternalRefCache::SingleRangeData>::iterator
1581         itrCache = rCacheData.begin(), itrCacheEnd = rCacheData.end();
1582
1583     for (SCTAB nTab = nTab1; nTab <= nTab2 && itrCache != itrCacheEnd; ++nTab, ++itrCache)
1584     {
1585         // Only loop within the data area.
1586         SCCOL nDataCol1 = nCol1, nDataCol2 = nCol2;
1587         SCROW nDataRow1 = nRow1, nDataRow2 = nRow2;
1588         bool bShrunk;
1589         if (!rSrcDoc.ShrinkToUsedDataArea( bShrunk, nTab, nDataCol1, nDataRow1, nDataCol2, nDataRow2, false))
1590             // no data within specified range.
1591             continue;
1592
1593         if (pUsedRange)
1594             // Make sure the used area only grows, not shrinks.
1595             pUsedRange->ExtendTo(ScRange(nDataCol1, nDataRow1, 0, nDataCol2, nDataRow2, 0));
1596         else
1597             pUsedRange.reset(new ScRange(nDataCol1, nDataRow1, 0, nDataCol2, nDataRow2, 0));
1598
1599         SCSIZE nMatrixColumns = static_cast<SCSIZE>(nCol2-nCol1+1);
1600         SCSIZE nMatrixRows = static_cast<SCSIZE>(nRow2-nRow1+1);
1601         ScMatrixRef xMat = new ScMatrix( nMatrixColumns, nMatrixRows);
1602
1603         // Check if size could be allocated and if not skip the fill, there's
1604         // one error element instead. But retry first with the actual data area
1605         // if that is smaller than the original range, which works for most
1606         // functions just not some that operate/compare with the original size
1607         // and expect empty values in non-data areas.
1608         // Restrict this though to ranges of entire columns or rows, other
1609         // ranges might be on purpose. (Other special cases to handle?)
1610         /* TODO: sparse matrix could help */
1611         SCSIZE nMatCols, nMatRows;
1612         xMat->GetDimensions( nMatCols, nMatRows);
1613         if (nMatCols == nMatrixColumns && nMatRows == nMatrixRows)
1614         {
1615             rSrcDoc.FillMatrix(*xMat, nTab, nCol1, nRow1, nCol2, nRow2, &rHostDoc.GetSharedStringPool());
1616         }
1617         else if ((nCol1 == 0 && nCol2 == rSrcDoc.MaxCol()) || (nRow1 == 0 && nRow2 == rSrcDoc.MaxRow()))
1618         {
1619             if ((o3tl::make_unsigned(nDataCol2-nDataCol1+1) < nMatrixColumns) ||
1620                 (o3tl::make_unsigned(nDataRow2-nDataRow1+1) < nMatrixRows))
1621             {
1622                 nMatrixColumns = static_cast<SCSIZE>(nDataCol2-nDataCol1+1);
1623                 nMatrixRows = static_cast<SCSIZE>(nDataRow2-nDataRow1+1);
1624                 xMat = new ScMatrix( nMatrixColumns, nMatrixRows);
1625                 xMat->GetDimensions( nMatCols, nMatRows);
1626                 if (nMatCols == nMatrixColumns && nMatRows == nMatrixRows)
1627                     rSrcDoc.FillMatrix(*xMat, nTab, nDataCol1, nDataRow1, nDataCol2, nDataRow2, &rHostDoc.GetSharedStringPool());
1628             }
1629         }
1630
1631         if (!bFirstTab)
1632             pArray->AddOpCode(ocSep);
1633
1634         ScMatrixToken aToken(xMat);
1635         pArray->AddToken(aToken);
1636
1637         itrCache->mpRangeData = std::move(xMat);
1638
1639         bFirstTab = false;
1640     }
1641
1642     if (!pUsedRange)
1643         return nullptr;
1644
1645     s.SetCol(pUsedRange->aStart.Col());
1646     s.SetRow(pUsedRange->aStart.Row());
1647     e.SetCol(pUsedRange->aEnd.Col());
1648     e.SetRow(pUsedRange->aEnd.Row());
1649
1650     return pArray;
1651 }
1652
1653 static std::unique_ptr<ScTokenArray> lcl_fillEmptyMatrix(const ScDocument& rDoc, const ScRange& rRange)
1654 {
1655     SCSIZE nC = static_cast<SCSIZE>(rRange.aEnd.Col()-rRange.aStart.Col()+1);
1656     SCSIZE nR = static_cast<SCSIZE>(rRange.aEnd.Row()-rRange.aStart.Row()+1);
1657     ScMatrixRef xMat = new ScMatrix(nC, nR);
1658
1659     ScMatrixToken aToken(std::move(xMat));
1660     unique_ptr<ScTokenArray> pArray(new ScTokenArray(rDoc));
1661     pArray->AddToken(aToken);
1662     return pArray;
1663 }
1664
1665 namespace {
1666 bool isLinkUpdateAllowedInDoc(const ScDocument& rDoc)
1667 {
1668     ScDocShell* pDocShell = rDoc.GetDocumentShell();
1669     if (!pDocShell)
1670         return rDoc.IsFunctionAccess();
1671
1672     return pDocShell->GetEmbeddedObjectContainer().getUserAllowsLinkUpdate();
1673 }
1674 }
1675
1676 ScExternalRefManager::ScExternalRefManager(ScDocument& rDoc) :
1677     mrDoc(rDoc),
1678     maRefCache(rDoc),
1679     mbInReferenceMarking(false),
1680     mbUserInteractionEnabled(true),
1681     mbDocTimerEnabled(true),
1682     maSrcDocTimer( "sc::ScExternalRefManager maSrcDocTimer" )
1683 {
1684     maSrcDocTimer.SetInvokeHandler( LINK(this, ScExternalRefManager, TimeOutHdl) );
1685     maSrcDocTimer.SetTimeout(SRCDOC_SCAN_INTERVAL);
1686 }
1687
1688 ScExternalRefManager::~ScExternalRefManager()
1689 {
1690     clear();
1691 }
1692
1693 OUString ScExternalRefManager::getCacheTableName(sal_uInt16 nFileId, size_t nTabIndex) const
1694 {
1695     return maRefCache.getTableName(nFileId, nTabIndex);
1696 }
1697
1698 ScExternalRefCache::TableTypeRef ScExternalRefManager::getCacheTable(sal_uInt16 nFileId, size_t nTabIndex) const
1699 {
1700     return maRefCache.getCacheTable(nFileId, nTabIndex);
1701 }
1702
1703 ScExternalRefCache::TableTypeRef ScExternalRefManager::getCacheTable(
1704     sal_uInt16 nFileId, const OUString& rTabName, bool bCreateNew, size_t* pnIndex, const OUString* pExtUrl)
1705 {
1706     return maRefCache.getCacheTable(nFileId, rTabName, bCreateNew, pnIndex, pExtUrl);
1707 }
1708
1709 ScExternalRefManager::LinkListener::LinkListener()
1710 {
1711 }
1712
1713 ScExternalRefManager::LinkListener::~LinkListener()
1714 {
1715 }
1716
1717 ScExternalRefManager::ApiGuard::ApiGuard(const ScDocument& rDoc) :
1718     mpMgr(rDoc.GetExternalRefManager()),
1719     mbOldInteractionEnabled(mpMgr->mbUserInteractionEnabled)
1720 {
1721     // We don't want user interaction handled in the API.
1722     mpMgr->mbUserInteractionEnabled = false;
1723 }
1724
1725 ScExternalRefManager::ApiGuard::~ApiGuard()
1726 {
1727     // Restore old value.
1728     mpMgr->mbUserInteractionEnabled = mbOldInteractionEnabled;
1729 }
1730
1731 void ScExternalRefManager::getAllCachedTableNames(sal_uInt16 nFileId, vector<OUString>& rTabNames) const
1732 {
1733     maRefCache.getAllTableNames(nFileId, rTabNames);
1734 }
1735
1736 SCTAB ScExternalRefManager::getCachedTabSpan( sal_uInt16 nFileId, const OUString& rStartTabName, const OUString& rEndTabName ) const
1737 {
1738     return maRefCache.getTabSpan( nFileId, rStartTabName, rEndTabName);
1739 }
1740
1741 void ScExternalRefManager::getAllCachedNumberFormats(vector<sal_uInt32>& rNumFmts) const
1742 {
1743     maRefCache.getAllNumberFormats(rNumFmts);
1744 }
1745
1746 sal_uInt16 ScExternalRefManager::getExternalFileCount() const
1747 {
1748     return static_cast< sal_uInt16 >( maSrcFiles.size() );
1749 }
1750
1751 void ScExternalRefManager::markUsedByLinkListeners()
1752 {
1753     bool bAllMarked = false;
1754     for (const auto& [rFileId, rLinkListeners] : maLinkListeners)
1755     {
1756         if (!rLinkListeners.empty())
1757             bAllMarked = maRefCache.setCacheDocReferenced(rFileId);
1758
1759         if (bAllMarked)
1760             break;
1761         /* TODO: LinkListeners should remember the table they're listening to.
1762          * As is, listening to one table will mark all tables of the document
1763          * being referenced. */
1764     }
1765 }
1766
1767 void ScExternalRefManager::markUsedExternalRefCells()
1768 {
1769     for (const auto& rEntry : maRefCells)
1770     {
1771         for (ScFormulaCell* pCell : rEntry.second)
1772         {
1773             bool bUsed = pCell->MarkUsedExternalReferences();
1774             if (bUsed)
1775                 // Return true when at least one cell references external docs.
1776                 return;
1777         }
1778     }
1779 }
1780
1781 bool ScExternalRefManager::setCacheTableReferenced( sal_uInt16 nFileId, const OUString& rTabName, size_t nSheets )
1782 {
1783     return maRefCache.setCacheTableReferenced( nFileId, rTabName, nSheets);
1784 }
1785
1786 void ScExternalRefManager::setAllCacheTableReferencedStati( bool bReferenced )
1787 {
1788     mbInReferenceMarking = !bReferenced;
1789     maRefCache.setAllCacheTableReferencedStati( bReferenced );
1790 }
1791
1792 void ScExternalRefManager::storeRangeNameTokens(sal_uInt16 nFileId, const OUString& rName, const ScTokenArray& rArray)
1793 {
1794     ScExternalRefCache::TokenArrayRef pNewArray;
1795     if (!rArray.HasExternalRef())
1796     {
1797         // Parse all tokens in this external range data, and replace each absolute
1798         // reference token with an external reference token, and cache them.
1799         pNewArray = std::make_shared<ScTokenArray>(mrDoc);
1800         FormulaTokenArrayPlainIterator aIter(rArray);
1801         for (const FormulaToken* pToken = aIter.First(); pToken; pToken = aIter.Next())
1802         {
1803             bool bTokenAdded = false;
1804             switch (pToken->GetType())
1805             {
1806                 case svSingleRef:
1807                 {
1808                     const ScSingleRefData& rRef = *pToken->GetSingleRef();
1809                     OUString aTabName;
1810                     if (SCTAB nCacheId = rRef.Tab(); nCacheId >= 0)
1811                         aTabName = maRefCache.getTableName(nFileId, nCacheId);
1812                     ScExternalSingleRefToken aNewToken(nFileId, svl::SharedString(aTabName),   // string not interned
1813                         *pToken->GetSingleRef());
1814                     pNewArray->AddToken(aNewToken);
1815                     bTokenAdded = true;
1816                 }
1817                 break;
1818                 case svDoubleRef:
1819                 {
1820                     const ScSingleRefData& rRef = *pToken->GetSingleRef();
1821                     OUString aTabName;
1822                     if (SCTAB nCacheId = rRef.Tab(); nCacheId >= 0)
1823                         aTabName = maRefCache.getTableName(nFileId, nCacheId);
1824                     ScExternalDoubleRefToken aNewToken(nFileId, svl::SharedString(aTabName),   // string not interned
1825                         *pToken->GetDoubleRef());
1826                     pNewArray->AddToken(aNewToken);
1827                     bTokenAdded = true;
1828                 }
1829                 break;
1830                 default:
1831                     ;   // nothing
1832             }
1833
1834             if (!bTokenAdded)
1835                 pNewArray->AddToken(*pToken);
1836         }
1837     }
1838     else
1839         pNewArray = rArray.Clone();
1840
1841     maRefCache.setRangeNameTokens(nFileId, rName, pNewArray);
1842 }
1843
1844 namespace {
1845
1846 /**
1847  * Put a single cell data into internal cache table.
1848  *
1849  * @param pFmt optional cell format index that may need to be stored with
1850  *             the cell value.
1851  */
1852 void putCellDataIntoCache(
1853     ScExternalRefCache& rRefCache, const ScExternalRefCache::TokenRef& pToken,
1854     sal_uInt16 nFileId, const OUString& rTabName, const ScAddress& rCell,
1855     const ScExternalRefCache::CellFormat* pFmt)
1856 {
1857     // Now, insert the token into cache table but don't cache empty cells.
1858     if (pToken->GetType() != formula::svEmptyCell)
1859     {
1860         sal_uLong nFmtIndex = (pFmt && pFmt->mbIsSet) ? pFmt->mnIndex : 0;
1861         rRefCache.setCellData(nFileId, rTabName, rCell.Col(), rCell.Row(), pToken, nFmtIndex);
1862     }
1863 }
1864
1865 /**
1866  * Put the data into our internal cache table.
1867  *
1868  * @param rRefCache cache table set.
1869  * @param pArray single range data to be returned.
1870  * @param nFileId external file ID
1871  * @param rTabName name of the table where the data should be cached.
1872  * @param rCacheData range data to be cached.
1873  * @param rCacheRange original cache range, including the empty region if
1874  *                    any.
1875  * @param rDataRange reduced cache range that includes only the non-empty
1876  *                   data area.
1877  */
1878 void putRangeDataIntoCache(
1879     ScExternalRefCache& rRefCache, ScExternalRefCache::TokenArrayRef& pArray,
1880     sal_uInt16 nFileId, const OUString& rTabName,
1881     const vector<ScExternalRefCache::SingleRangeData>& rCacheData,
1882     const ScRange& rCacheRange, const ScRange& rDataRange)
1883 {
1884     if (pArray)
1885         // Cache these values.
1886         rRefCache.setCellRangeData(nFileId, rDataRange, rCacheData, pArray);
1887     else
1888     {
1889         // Array is empty.  Fill it with an empty matrix of the required size.
1890         pArray = lcl_fillEmptyMatrix(rRefCache.getDoc(), rCacheRange);
1891
1892         // Make sure to set this range 'cached', to prevent unnecessarily
1893         // accessing the src document time and time again.
1894         ScExternalRefCache::TableTypeRef pCacheTab =
1895             rRefCache.getCacheTable(nFileId, rTabName, true, nullptr, nullptr);
1896         if (pCacheTab)
1897             pCacheTab->setCachedCellRange(
1898                 rCacheRange.aStart.Col(), rCacheRange.aStart.Row(), rCacheRange.aEnd.Col(), rCacheRange.aEnd.Row());
1899     }
1900 }
1901
1902 /**
1903  * When accessing an external document for the first time, we need to
1904  * populate the cache with all its sheet names (whether they are referenced
1905  * or not) in the correct order.  Many client codes that use external
1906  * references make this assumption.
1907  *
1908  * @param rRefCache cache table set.
1909  * @param pSrcDoc source document instance.
1910  * @param nFileId external file ID associated with the source document.
1911  */
1912 void initDocInCache(ScExternalRefCache& rRefCache, const ScDocument* pSrcDoc, sal_uInt16 nFileId)
1913 {
1914     if (!pSrcDoc)
1915         return;
1916
1917     if (rRefCache.isDocInitialized(nFileId))
1918         // Already initialized.  No need to do this twice.
1919         return;
1920
1921     SCTAB nTabCount = pSrcDoc->GetTableCount();
1922     if (!nTabCount)
1923         return;
1924
1925     // Populate the cache with all table names in the source document.
1926     vector<OUString> aTabNames;
1927     aTabNames.reserve(nTabCount);
1928     for (SCTAB i = 0; i < nTabCount; ++i)
1929     {
1930         OUString aName;
1931         pSrcDoc->GetName(i, aName);
1932         aTabNames.push_back(aName);
1933     }
1934
1935     // Obtain the base name, don't bother if there are more than one sheets.
1936     OUString aBaseName;
1937     if (nTabCount == 1)
1938     {
1939         const ScDocShell* pShell = pSrcDoc->GetDocumentShell();
1940         if (pShell && pShell->GetMedium())
1941         {
1942             OUString aName = pShell->GetMedium()->GetName();
1943             aBaseName = INetURLObject( aName).GetBase();
1944         }
1945     }
1946
1947     rRefCache.initializeDoc(nFileId, aTabNames, aBaseName);
1948 }
1949
1950 }
1951
1952 bool ScExternalRefManager::getSrcDocTable( const ScDocument& rSrcDoc, const OUString& rTabName, SCTAB& rTab,
1953         sal_uInt16 nFileId ) const
1954 {
1955     return maRefCache.getSrcDocTable( rSrcDoc, rTabName, rTab, nFileId);
1956 }
1957
1958 ScExternalRefCache::TokenRef ScExternalRefManager::getSingleRefToken(
1959     sal_uInt16 nFileId, const OUString& rTabName, const ScAddress& rCell,
1960     const ScAddress* pCurPos, SCTAB* pTab, ScExternalRefCache::CellFormat* pFmt)
1961 {
1962     if (pCurPos)
1963         insertRefCell(nFileId, *pCurPos);
1964
1965     maybeLinkExternalFile(nFileId);
1966
1967     if (pTab)
1968         *pTab = -1;
1969
1970     if (pFmt)
1971         pFmt->mbIsSet = false;
1972
1973     ScDocument* pSrcDoc = getInMemorySrcDocument(nFileId);
1974     if (pSrcDoc)
1975     {
1976         // source document already loaded in memory.  Re-use this instance.
1977         SCTAB nTab;
1978         if (!getSrcDocTable( *pSrcDoc, rTabName, nTab, nFileId))
1979         {
1980             // specified table name doesn't exist in the source document.
1981             ScExternalRefCache::TokenRef pToken(new FormulaErrorToken(FormulaError::NoRef));
1982             return pToken;
1983         }
1984
1985         if (pTab)
1986             *pTab = nTab;
1987
1988         ScExternalRefCache::TokenRef pToken =
1989             getSingleRefTokenFromSrcDoc(
1990                 nFileId, *pSrcDoc, ScAddress(rCell.Col(),rCell.Row(),nTab), pFmt);
1991
1992         putCellDataIntoCache(maRefCache, pToken, nFileId, rTabName, rCell, pFmt);
1993         return pToken;
1994     }
1995
1996     // Check if the given table name and the cell position is cached.
1997     sal_uInt32 nFmtIndex = 0;
1998     ScExternalRefCache::TokenRef pToken = maRefCache.getCellData(
1999         nFileId, rTabName, rCell.Col(), rCell.Row(), &nFmtIndex);
2000     if (pToken)
2001     {
2002         // Cache hit !
2003         fillCellFormat(nFmtIndex, pFmt);
2004         return pToken;
2005     }
2006
2007     // reference not cached.  read from the source document.
2008     pSrcDoc = getSrcDocument(nFileId);
2009     if (!pSrcDoc)
2010     {
2011         // Source document not reachable.
2012         if (!isLinkUpdateAllowedInDoc(mrDoc))
2013         {
2014             // Indicate with specific error.
2015             pToken.reset(new FormulaErrorToken(FormulaError::LinkFormulaNeedingCheck));
2016         }
2017         else
2018         {
2019             // Throw a reference error.
2020             pToken.reset(new FormulaErrorToken(FormulaError::NoRef));
2021         }
2022         return pToken;
2023     }
2024
2025     SCTAB nTab;
2026     if (!getSrcDocTable( *pSrcDoc, rTabName, nTab, nFileId))
2027     {
2028         // specified table name doesn't exist in the source document.
2029         pToken.reset(new FormulaErrorToken(FormulaError::NoRef));
2030         return pToken;
2031     }
2032
2033     if (pTab)
2034         *pTab = nTab;
2035
2036     SCCOL nDataCol1 = 0, nDataCol2 = pSrcDoc->MaxCol();
2037     SCROW nDataRow1 = 0, nDataRow2 = pSrcDoc->MaxRow();
2038     bool bData = pSrcDoc->ShrinkToDataArea(nTab, nDataCol1, nDataRow1, nDataCol2, nDataRow2);
2039     if (!bData || rCell.Col() < nDataCol1 || nDataCol2 < rCell.Col() || rCell.Row() < nDataRow1 || nDataRow2 < rCell.Row())
2040     {
2041         // requested cell is outside the data area.  Don't even bother caching
2042         // this data, but add it to the cached range to prevent accessing the
2043         // source document time and time again.
2044         ScExternalRefCache::TableTypeRef pCacheTab =
2045             maRefCache.getCacheTable(nFileId, rTabName, true, nullptr, nullptr);
2046         if (pCacheTab)
2047             pCacheTab->setCachedCell(rCell.Col(), rCell.Row());
2048
2049         pToken.reset(new ScEmptyCellToken(false, false));
2050         return pToken;
2051     }
2052
2053     pToken = getSingleRefTokenFromSrcDoc(
2054         nFileId, *pSrcDoc, ScAddress(rCell.Col(),rCell.Row(),nTab), pFmt);
2055
2056     putCellDataIntoCache(maRefCache, pToken, nFileId, rTabName, rCell, pFmt);
2057     return pToken;
2058 }
2059
2060 ScExternalRefCache::TokenArrayRef ScExternalRefManager::getDoubleRefTokens(
2061     sal_uInt16 nFileId, const OUString& rTabName, const ScRange& rRange, const ScAddress* pCurPos)
2062 {
2063     if (pCurPos)
2064         insertRefCell(nFileId, *pCurPos);
2065
2066     maybeLinkExternalFile(nFileId);
2067
2068     ScRange aDataRange(rRange);
2069     ScDocument* pSrcDoc = getInMemorySrcDocument(nFileId);
2070     if (pSrcDoc)
2071     {
2072         // Document already loaded in memory.
2073         vector<ScExternalRefCache::SingleRangeData> aCacheData;
2074         ScExternalRefCache::TokenArrayRef pArray =
2075             getDoubleRefTokensFromSrcDoc(*pSrcDoc, rTabName, aDataRange, aCacheData);
2076
2077         // Put the data into cache.
2078         putRangeDataIntoCache(maRefCache, pArray, nFileId, rTabName, aCacheData, rRange, aDataRange);
2079         return pArray;
2080     }
2081
2082     // Check if the given table name and the cell position is cached.
2083     ScExternalRefCache::TokenArrayRef pArray =
2084         maRefCache.getCellRangeData(nFileId, rTabName, rRange);
2085     if (pArray)
2086         // Cache hit !
2087         return pArray;
2088
2089     pSrcDoc = getSrcDocument(nFileId);
2090     if (!pSrcDoc)
2091     {
2092         // Source document is not reachable.  Throw a reference error.
2093         pArray = std::make_shared<ScTokenArray>(maRefCache.getDoc());
2094         pArray->AddToken(FormulaErrorToken(FormulaError::NoRef));
2095         return pArray;
2096     }
2097
2098     vector<ScExternalRefCache::SingleRangeData> aCacheData;
2099     pArray = getDoubleRefTokensFromSrcDoc(*pSrcDoc, rTabName, aDataRange, aCacheData);
2100
2101     // Put the data into cache.
2102     putRangeDataIntoCache(maRefCache, pArray, nFileId, rTabName, aCacheData, rRange, aDataRange);
2103     return pArray;
2104 }
2105
2106 ScExternalRefCache::TokenArrayRef ScExternalRefManager::getRangeNameTokens(
2107     sal_uInt16 nFileId, const OUString& rName, const ScAddress* pCurPos)
2108 {
2109     if (pCurPos)
2110         insertRefCell(nFileId, *pCurPos);
2111
2112     maybeLinkExternalFile(nFileId);
2113
2114     OUString aName = rName; // make a copy to have the casing corrected.
2115     ScDocument* pSrcDoc = getInMemorySrcDocument(nFileId);
2116     if (pSrcDoc)
2117     {
2118         // Document already loaded in memory.
2119         ScExternalRefCache::TokenArrayRef pArray =
2120             getRangeNameTokensFromSrcDoc(nFileId, *pSrcDoc, aName);
2121
2122         if (pArray)
2123             // Cache this range name array.
2124             maRefCache.setRangeNameTokens(nFileId, aName, pArray);
2125
2126         return pArray;
2127     }
2128
2129     ScExternalRefCache::TokenArrayRef pArray = maRefCache.getRangeNameTokens(nFileId, rName);
2130     if (pArray)
2131         // This range name is cached.
2132         return pArray;
2133
2134     pSrcDoc = getSrcDocument(nFileId);
2135     if (!pSrcDoc)
2136         // failed to load document from disk.
2137         return ScExternalRefCache::TokenArrayRef();
2138
2139     pArray = getRangeNameTokensFromSrcDoc(nFileId, *pSrcDoc, aName);
2140
2141     if (pArray)
2142         // Cache this range name array.
2143         maRefCache.setRangeNameTokens(nFileId, aName, pArray);
2144
2145     return pArray;
2146 }
2147
2148 namespace {
2149
2150 bool hasRangeName(const ScDocument& rDoc, const OUString& rName)
2151 {
2152     ScRangeName* pExtNames = rDoc.GetRangeName();
2153     OUString aUpperName = ScGlobal::getCharClass().uppercase(rName);
2154     const ScRangeData* pRangeData = pExtNames->findByUpperName(aUpperName);
2155     return pRangeData != nullptr;
2156 }
2157
2158 }
2159
2160 bool ScExternalRefManager::isValidRangeName(sal_uInt16 nFileId, const OUString& rName)
2161 {
2162     maybeLinkExternalFile(nFileId);
2163     ScDocument* pSrcDoc = getInMemorySrcDocument(nFileId);
2164     if (pSrcDoc)
2165     {
2166         // Only check the presence of the name.
2167         if (hasRangeName(*pSrcDoc, rName))
2168         {
2169             maRefCache.setRangeName(nFileId, rName);
2170             return true;
2171         }
2172         return false;
2173     }
2174
2175     if (maRefCache.isValidRangeName(nFileId, rName))
2176         // Range name is cached.
2177         return true;
2178
2179     pSrcDoc = getSrcDocument(nFileId);
2180     if (!pSrcDoc)
2181         // failed to load document from disk.
2182         return false;
2183
2184     if (hasRangeName(*pSrcDoc, rName))
2185     {
2186         maRefCache.setRangeName(nFileId, rName);
2187         return true;
2188     }
2189
2190     return false;
2191 }
2192
2193 void ScExternalRefManager::refreshAllRefCells(sal_uInt16 nFileId)
2194 {
2195     RefCellMap::iterator itrFile = maRefCells.find(nFileId);
2196     if (itrFile == maRefCells.end())
2197         return;
2198
2199     RefCellSet& rRefCells = itrFile->second;
2200     for_each(rRefCells.begin(), rRefCells.end(), UpdateFormulaCell());
2201
2202     ScViewData* pViewData = ScDocShell::GetViewData();
2203     if (!pViewData)
2204         return;
2205
2206     ScTabViewShell* pVShell = pViewData->GetViewShell();
2207     if (!pVShell)
2208         return;
2209
2210     // Repainting the grid also repaints the texts, but is there a better way
2211     // to refresh texts?
2212     pVShell->Invalidate(FID_REPAINT);
2213     pVShell->PaintGrid();
2214 }
2215
2216 namespace {
2217
2218 void insertRefCellByIterator(
2219     const ScExternalRefManager::RefCellMap::iterator& itr, ScFormulaCell* pCell)
2220 {
2221     if (pCell)
2222     {
2223         itr->second.insert(pCell);
2224         pCell->SetIsExtRef();
2225     }
2226 }
2227
2228 }
2229
2230 void ScExternalRefManager::insertRefCell(sal_uInt16 nFileId, const ScAddress& rCell)
2231 {
2232     RefCellMap::iterator itr = maRefCells.find(nFileId);
2233     if (itr == maRefCells.end())
2234     {
2235         RefCellSet aRefCells;
2236         pair<RefCellMap::iterator, bool> r = maRefCells.emplace(
2237             nFileId, aRefCells);
2238         if (!r.second)
2239             // insertion failed.
2240             return;
2241
2242         itr = r.first;
2243     }
2244
2245     insertRefCellByIterator(itr, mrDoc.GetFormulaCell(rCell));
2246 }
2247
2248 void ScExternalRefManager::insertRefCellFromTemplate( ScFormulaCell* pTemplateCell, ScFormulaCell* pCell )
2249 {
2250     if (!pTemplateCell || !pCell)
2251         return;
2252
2253     for (RefCellMap::iterator itr = maRefCells.begin(); itr != maRefCells.end(); ++itr)
2254     {
2255         if (itr->second.find(pTemplateCell) != itr->second.end())
2256             insertRefCellByIterator(itr, pCell);
2257     }
2258 }
2259
2260 bool ScExternalRefManager::hasCellExternalReference(const ScAddress& rCell)
2261 {
2262     ScFormulaCell* pCell = mrDoc.GetFormulaCell(rCell);
2263
2264     if (pCell)
2265         return std::any_of(maRefCells.begin(), maRefCells.end(),
2266             [&pCell](const RefCellMap::value_type& rEntry) { return rEntry.second.find(pCell) != rEntry.second.end(); });
2267
2268     return false;
2269 }
2270
2271 void ScExternalRefManager::enableDocTimer( bool bEnable )
2272 {
2273     if (mbDocTimerEnabled == bEnable)
2274         return;
2275
2276     mbDocTimerEnabled = bEnable;
2277     if (mbDocTimerEnabled)
2278     {
2279         if (!maDocShells.empty())
2280         {
2281             for (auto& rEntry : maDocShells)
2282                 rEntry.second.maLastAccess = tools::Time(tools::Time::SYSTEM);
2283
2284             maSrcDocTimer.Start();
2285         }
2286     }
2287     else
2288         maSrcDocTimer.Stop();
2289 }
2290
2291 void ScExternalRefManager::fillCellFormat(sal_uLong nFmtIndex, ScExternalRefCache::CellFormat* pFmt) const
2292 {
2293     if (!pFmt)
2294         return;
2295
2296     SvNumFormatType nFmtType = mrDoc.GetFormatTable()->GetType(nFmtIndex);
2297     if (nFmtType != SvNumFormatType::UNDEFINED)
2298     {
2299         pFmt->mbIsSet = true;
2300         pFmt->mnIndex = nFmtIndex;
2301         pFmt->mnType = nFmtType;
2302     }
2303 }
2304
2305 ScExternalRefCache::TokenRef ScExternalRefManager::getSingleRefTokenFromSrcDoc(
2306     sal_uInt16 nFileId, ScDocument& rSrcDoc, const ScAddress& rPos,
2307     ScExternalRefCache::CellFormat* pFmt)
2308 {
2309     // Get the cell from src doc, and convert it into a token.
2310     ScRefCellValue aCell(rSrcDoc, rPos);
2311     ScExternalRefCache::TokenRef pToken(convertToToken(mrDoc, rSrcDoc, aCell));
2312
2313     if (!pToken)
2314     {
2315         // Generate an error for unresolvable cells.
2316         pToken.reset( new FormulaErrorToken( FormulaError::NoValue));
2317     }
2318
2319     // Get number format information.
2320     sal_uInt32 nFmtIndex = rSrcDoc.GetNumberFormat(rPos.Col(), rPos.Row(), rPos.Tab());
2321     nFmtIndex = getMappedNumberFormat(nFileId, nFmtIndex, rSrcDoc);
2322     fillCellFormat(nFmtIndex, pFmt);
2323     return pToken;
2324 }
2325
2326 ScExternalRefCache::TokenArrayRef ScExternalRefManager::getDoubleRefTokensFromSrcDoc(
2327     const ScDocument& rSrcDoc, const OUString& rTabName, ScRange& rRange,
2328     vector<ScExternalRefCache::SingleRangeData>& rCacheData)
2329 {
2330     ScExternalRefCache::TokenArrayRef pArray;
2331     SCTAB nTab1;
2332
2333     if (!rSrcDoc.GetTable(rTabName, nTab1))
2334     {
2335         // specified table name doesn't exist in the source document.
2336         pArray = std::make_shared<ScTokenArray>(rSrcDoc);
2337         pArray->AddToken(FormulaErrorToken(FormulaError::NoRef));
2338         return pArray;
2339     }
2340
2341     ScRange aRange(rRange);
2342     aRange.PutInOrder();
2343     SCTAB nTabSpan = aRange.aEnd.Tab() - aRange.aStart.Tab();
2344
2345     vector<ScExternalRefCache::SingleRangeData> aCacheData;
2346     aCacheData.reserve(nTabSpan+1);
2347     aCacheData.emplace_back();
2348     aCacheData.back().maTableName = ScGlobal::getCharClass().uppercase(rTabName);
2349
2350     for (SCTAB i = 1; i < nTabSpan + 1; ++i)
2351     {
2352         OUString aTabName;
2353         if (!rSrcDoc.GetName(nTab1 + 1, aTabName))
2354             // source document doesn't have any table by the specified name.
2355             break;
2356
2357         aCacheData.emplace_back();
2358         aCacheData.back().maTableName = ScGlobal::getCharClass().uppercase(aTabName);
2359     }
2360
2361     aRange.aStart.SetTab(nTab1);
2362     aRange.aEnd.SetTab(nTab1 + nTabSpan);
2363
2364     pArray = convertToTokenArray(mrDoc, rSrcDoc, aRange, aCacheData);
2365     rRange = aRange;
2366     rCacheData.swap(aCacheData);
2367     return pArray;
2368 }
2369
2370 ScExternalRefCache::TokenArrayRef ScExternalRefManager::getRangeNameTokensFromSrcDoc(
2371     sal_uInt16 nFileId, const ScDocument& rSrcDoc, OUString& rName)
2372 {
2373     ScRangeName* pExtNames = rSrcDoc.GetRangeName();
2374     OUString aUpperName = ScGlobal::getCharClass().uppercase(rName);
2375     const ScRangeData* pRangeData = pExtNames->findByUpperName(aUpperName);
2376     if (!pRangeData)
2377         return ScExternalRefCache::TokenArrayRef();
2378
2379     // Parse all tokens in this external range data, and replace each absolute
2380     // reference token with an external reference token, and cache them.  Also
2381     // register the source document with the link manager if it's a new
2382     // source.
2383
2384     ScExternalRefCache::TokenArrayRef pNew = std::make_shared<ScTokenArray>(rSrcDoc);
2385
2386     ScTokenArray aCode(*pRangeData->GetCode());
2387     FormulaTokenArrayPlainIterator aIter(aCode);
2388     for (const FormulaToken* pToken = aIter.First(); pToken; pToken = aIter.Next())
2389     {
2390         bool bTokenAdded = false;
2391         switch (pToken->GetType())
2392         {
2393             case svSingleRef:
2394             {
2395                 const ScSingleRefData& rRef = *pToken->GetSingleRef();
2396                 OUString aTabName;
2397                 rSrcDoc.GetName(rRef.Tab(), aTabName);
2398                 ScExternalSingleRefToken aNewToken(nFileId, svl::SharedString( aTabName),   // string not interned
2399                         *pToken->GetSingleRef());
2400                 pNew->AddToken(aNewToken);
2401                 bTokenAdded = true;
2402             }
2403             break;
2404             case svDoubleRef:
2405             {
2406                 const ScSingleRefData& rRef = *pToken->GetSingleRef();
2407                 OUString aTabName;
2408                 rSrcDoc.GetName(rRef.Tab(), aTabName);
2409                 ScExternalDoubleRefToken aNewToken(nFileId, svl::SharedString( aTabName),   // string not interned
2410                         *pToken->GetDoubleRef());
2411                 pNew->AddToken(aNewToken);
2412                 bTokenAdded = true;
2413             }
2414             break;
2415             default:
2416                 ;   // nothing
2417         }
2418
2419         if (!bTokenAdded)
2420             pNew->AddToken(*pToken);
2421     }
2422
2423     rName = pRangeData->GetName(); // Get the correctly-cased name.
2424     return pNew;
2425 }
2426
2427 ScDocument* ScExternalRefManager::getInMemorySrcDocument(sal_uInt16 nFileId)
2428 {
2429     const OUString* pFileName = getExternalFileName(nFileId);
2430     if (!pFileName)
2431         return nullptr;
2432
2433     // Do not load document until it was allowed.
2434     if (!isLinkUpdateAllowedInDoc(mrDoc))
2435         return nullptr;
2436
2437     ScDocument* pSrcDoc = nullptr;
2438     ScDocShell* pShell = static_cast<ScDocShell*>(SfxObjectShell::GetFirst(checkSfxObjectShell<ScDocShell>, false));
2439     while (pShell)
2440     {
2441         SfxMedium* pMedium = pShell->GetMedium();
2442         if (pMedium && !pMedium->GetName().isEmpty())
2443         {
2444             // TODO: We should make the case sensitivity platform dependent.
2445             if (pFileName->equalsIgnoreAsciiCase(pMedium->GetName()))
2446             {
2447                 // Found !
2448                 pSrcDoc = &pShell->GetDocument();
2449                 break;
2450             }
2451         }
2452         else
2453         {
2454             // handle unsaved documents here
2455             OUString aName = pShell->GetName();
2456             if (pFileName->equalsIgnoreAsciiCase(aName))
2457             {
2458                 // Found !
2459                 SrcShell aSrcDoc;
2460                 aSrcDoc.maShell = pShell;
2461                 maUnsavedDocShells.emplace(nFileId, aSrcDoc);
2462                 StartListening(*pShell);
2463                 pSrcDoc = &pShell->GetDocument();
2464                 break;
2465             }
2466         }
2467         pShell = static_cast<ScDocShell*>(SfxObjectShell::GetNext(*pShell, checkSfxObjectShell<ScDocShell>, false));
2468     }
2469
2470     initDocInCache(maRefCache, pSrcDoc, nFileId);
2471     return pSrcDoc;
2472 }
2473
2474 ScDocument* ScExternalRefManager::getSrcDocument(sal_uInt16 nFileId)
2475 {
2476     if (!mrDoc.IsExecuteLinkEnabled())
2477         return nullptr;
2478
2479     DocShellMap::iterator itrEnd = maDocShells.end();
2480     DocShellMap::iterator itr = maDocShells.find(nFileId);
2481
2482     if (itr != itrEnd)
2483     {
2484         // document already loaded.
2485
2486         SfxObjectShell* p = itr->second.maShell.get();
2487         itr->second.maLastAccess = tools::Time( tools::Time::SYSTEM );
2488         return &static_cast<ScDocShell*>(p)->GetDocument();
2489     }
2490
2491     itrEnd = maUnsavedDocShells.end();
2492     itr = maUnsavedDocShells.find(nFileId);
2493     if (itr != itrEnd)
2494     {
2495         //document is unsaved document
2496
2497         SfxObjectShell* p = itr->second.maShell.get();
2498         itr->second.maLastAccess = tools::Time( tools::Time::SYSTEM );
2499         return &static_cast<ScDocShell*>(p)->GetDocument();
2500     }
2501
2502     const OUString* pFile = getExternalFileName(nFileId);
2503     if (!pFile)
2504         // no file name associated with this ID.
2505         return nullptr;
2506
2507     SrcShell aSrcDoc;
2508     try
2509     {
2510         OUString aFilter;
2511         aSrcDoc.maShell = loadSrcDocument(nFileId, aFilter);
2512     }
2513     catch (const css::uno::Exception&)
2514     {
2515     }
2516     if (!aSrcDoc.maShell.is())
2517     {
2518         // source document could not be loaded.
2519         return nullptr;
2520     }
2521
2522     return &cacheNewDocShell(nFileId, aSrcDoc);
2523 }
2524
2525 SfxObjectShellRef ScExternalRefManager::loadSrcDocument(sal_uInt16 nFileId, OUString& rFilter)
2526 {
2527     // Do not load document until it was allowed.
2528     if (!isLinkUpdateAllowedInDoc(mrDoc))
2529         return nullptr;
2530
2531     const SrcFileData* pFileData = getExternalFileData(nFileId);
2532     if (!pFileData)
2533         return nullptr;
2534
2535     // Always load the document by using the path created from the relative
2536     // path.  If the referenced document is not there, simply exit.  The
2537     // original file name should be used only when the relative path is not
2538     // given.
2539     OUString aFile = pFileData->maFileName;
2540     maybeCreateRealFileName(nFileId);
2541     if (!pFileData->maRealFileName.isEmpty())
2542         aFile = pFileData->maRealFileName;
2543
2544     if (!isFileLoadable(aFile))
2545         return nullptr;
2546
2547     INetURLObject aURLObject(aFile);
2548     const OUString sHost = aURLObject.GetHost();
2549     if (HostFilter::isForbidden(sHost))
2550     {
2551         SAL_WARN( "sc.ui", "ScExternalRefManager::loadSrcDocument: blocked access to external file: \"" << aFile << "\"");
2552         return nullptr;
2553     }
2554
2555     OUString aOptions = pFileData->maFilterOptions;
2556     if ( !pFileData->maFilterName.isEmpty() )
2557         rFilter = pFileData->maFilterName;      // don't overwrite stored filter with guessed filter
2558     else
2559         ScDocumentLoader::GetFilterName(aFile, rFilter, aOptions, true, false);
2560     std::shared_ptr<const SfxFilter> pFilter = ScDocShell::Factory().GetFilterContainer()->GetFilter4FilterName(rFilter);
2561
2562     if (pFileData->maRelativeName.isEmpty())
2563     {
2564         // Generate a relative file path.
2565         INetURLObject aBaseURL(getOwnDocumentName());
2566         aBaseURL.insertName(u"content.xml");
2567
2568         OUString aStr = URIHelper::simpleNormalizedMakeRelative(
2569             aBaseURL.GetMainURL(INetURLObject::DecodeMechanism::NONE), aFile);
2570
2571         setRelativeFileName(nFileId, aStr);
2572     }
2573
2574     std::unique_ptr<SfxItemSet> pSet(new SfxAllItemSet(SfxGetpApp()->GetPool()));
2575     if (!aOptions.isEmpty())
2576         pSet->Put(SfxStringItem(SID_FILE_FILTEROPTIONS, aOptions));
2577
2578     // make medium hidden to prevent assertion from progress bar
2579     pSet->Put( SfxBoolItem(SID_HIDDEN, true) );
2580
2581     // If the current document is allowed to execute macros then the referenced
2582     // document may execute macros according to the security configuration.
2583     // Similar for UpdateDocMode to update links, just that if we reach here
2584     // the user already allowed updates and intermediate documents are expected
2585     // to update as well. When loading the document ScDocShell::Load() will
2586     // check through ScDocShell::GetLinkUpdateModeState() if its location is
2587     // trusted.
2588     ScDocShell* pShell = mrDoc.GetDocumentShell();
2589     if (pShell)
2590     {
2591         SfxMedium* pMedium = pShell->GetMedium();
2592         if (pMedium)
2593         {
2594             const SfxUInt16Item* pItem = pMedium->GetItemSet().GetItemIfSet( SID_MACROEXECMODE, false );
2595             if (pItem &&
2596                     pItem->GetValue() != css::document::MacroExecMode::NEVER_EXECUTE)
2597                 pSet->Put( SfxUInt16Item( SID_MACROEXECMODE, css::document::MacroExecMode::USE_CONFIG));
2598         }
2599
2600         pSet->Put( SfxUInt16Item( SID_UPDATEDOCMODE, css::document::UpdateDocMode::FULL_UPDATE));
2601     }
2602
2603     unique_ptr<SfxMedium> pMedium(new SfxMedium(aFile, StreamMode::STD_READ,
2604                                                 std::move(pFilter), std::move(pSet)));
2605     if (pMedium->GetErrorIgnoreWarning() != ERRCODE_NONE)
2606         return nullptr;
2607
2608     // To load encrypted documents with password, user interaction needs to be enabled.
2609     pMedium->UseInteractionHandler(mbUserInteractionEnabled);
2610
2611     rtl::Reference<ScDocShell> pNewShell = new ScDocShell(SfxModelFlags::EXTERNAL_LINK);
2612
2613     // increment the recursive link count of the source document.
2614     ScExtDocOptions* pExtOpt = mrDoc.GetExtDocOptions();
2615     sal_uInt32 nLinkCount = pExtOpt ? pExtOpt->GetDocSettings().mnLinkCnt : 0;
2616     ScDocument& rSrcDoc = pNewShell->GetDocument();
2617     rSrcDoc.EnableExecuteLink(false); // to prevent circular access of external references.
2618     rSrcDoc.EnableUndo(false);
2619     rSrcDoc.LockAdjustHeight();
2620     rSrcDoc.EnableUserInteraction(false);
2621
2622     ScExtDocOptions* pExtOptNew = rSrcDoc.GetExtDocOptions();
2623     if (!pExtOptNew)
2624     {
2625         rSrcDoc.SetExtDocOptions(std::make_unique<ScExtDocOptions>());
2626         pExtOptNew = rSrcDoc.GetExtDocOptions();
2627     }
2628     pExtOptNew->GetDocSettings().mnLinkCnt = nLinkCount + 1;
2629
2630     if (!pNewShell->DoLoad(pMedium.release()))
2631     {
2632         pNewShell->DoClose();
2633         pNewShell.clear();
2634         return pNewShell;
2635     }
2636
2637     // with UseInteractionHandler, options may be set by dialog during DoLoad
2638     OUString aNew = ScDocumentLoader::GetOptions(*pNewShell->GetMedium());
2639     if (!aNew.isEmpty() && aNew != aOptions)
2640         aOptions = aNew;
2641     setFilterData(nFileId, rFilter, aOptions);    // update the filter data, including the new options
2642
2643     return pNewShell;
2644 }
2645
2646 ScDocument& ScExternalRefManager::cacheNewDocShell( sal_uInt16 nFileId, SrcShell& rSrcShell )
2647 {
2648     if (mbDocTimerEnabled && maDocShells.empty())
2649         // If this is the first source document insertion, start up the timer.
2650         maSrcDocTimer.Start();
2651
2652     maDocShells.emplace(nFileId, rSrcShell);
2653     SfxObjectShell& rShell = *rSrcShell.maShell;
2654     ScDocument& rSrcDoc = static_cast<ScDocShell&>(rShell).GetDocument();
2655     initDocInCache(maRefCache, &rSrcDoc, nFileId);
2656     return rSrcDoc;
2657 }
2658
2659 bool ScExternalRefManager::isFileLoadable(const OUString& rFile) const
2660 {
2661     if (rFile.isEmpty())
2662         return false;
2663
2664     if (isOwnDocument(rFile))
2665         return false;
2666     OUString aPhysical;
2667     if (osl::FileBase::getSystemPathFromFileURL(rFile, aPhysical)
2668         == osl::FileBase::E_None)
2669     {
2670         // #i114504# try IsFolder/Exists only for file URLs
2671
2672         if (utl::UCBContentHelper::IsFolder(rFile))
2673             return false;
2674
2675         return utl::UCBContentHelper::Exists(rFile);
2676     }
2677     else
2678         return true;    // for http and others, Exists doesn't work, but the URL can still be opened
2679 }
2680
2681 void ScExternalRefManager::maybeLinkExternalFile( sal_uInt16 nFileId, bool bDeferFilterDetection )
2682 {
2683     if (maLinkedDocs.count(nFileId))
2684         // file already linked, or the link has been broken.
2685         return;
2686
2687     // Source document not linked yet.  Link it now.
2688     const OUString* pFileName = getExternalFileName(nFileId);
2689     if (!pFileName)
2690         return;
2691
2692     OUString aFilter, aOptions;
2693     const SrcFileData* pFileData = getExternalFileData(nFileId);
2694     if (pFileData)
2695     {
2696         aFilter = pFileData->maFilterName;
2697         aOptions = pFileData->maFilterOptions;
2698     }
2699
2700     // Filter detection may access external links; defer it until we are allowed.
2701     if (!bDeferFilterDetection)
2702         bDeferFilterDetection = !isLinkUpdateAllowedInDoc(mrDoc);
2703
2704     // If a filter was already set (for example, loading the cached table),
2705     // don't call GetFilterName which has to access the source file.
2706     // If filter detection is deferred, the next successful loadSrcDocument()
2707     // will update SrcFileData filter name.
2708     if (aFilter.isEmpty() && !bDeferFilterDetection)
2709         ScDocumentLoader::GetFilterName(*pFileName, aFilter, aOptions, true, false);
2710     sfx2::LinkManager* pLinkMgr = mrDoc.GetLinkManager();
2711     if (!pLinkMgr)
2712     {
2713         SAL_WARN( "sc.ui", "ScExternalRefManager::maybeLinkExternalFile: pLinkMgr==NULL");
2714         return;
2715     }
2716     ScExternalRefLink* pLink = new ScExternalRefLink(mrDoc, nFileId);
2717     OSL_ENSURE(pFileName, "ScExternalRefManager::maybeLinkExternalFile: file name pointer is NULL");
2718     pLinkMgr->InsertFileLink(*pLink, sfx2::SvBaseLinkObjectType::ClientFile, *pFileName,
2719             (aFilter.isEmpty() && bDeferFilterDetection ? nullptr : &aFilter));
2720
2721     pLink->SetDoRefresh(false);
2722     pLink->Update();
2723     pLink->SetDoRefresh(true);
2724
2725     maLinkedDocs.emplace(nFileId, true);
2726 }
2727
2728 void ScExternalRefManager::addFilesToLinkManager()
2729 {
2730     if (maSrcFiles.empty())
2731         return;
2732
2733     SAL_WARN_IF( maSrcFiles.size() >= SAL_MAX_UINT16,
2734             "sc.ui", "ScExternalRefManager::addFilesToLinkManager: files overflow");
2735     const sal_uInt16 nSize = static_cast<sal_uInt16>( std::min<size_t>( maSrcFiles.size(), SAL_MAX_UINT16));
2736     for (sal_uInt16 nFileId = 0; nFileId < nSize; ++nFileId)
2737         maybeLinkExternalFile( nFileId, true);
2738 }
2739
2740 void ScExternalRefManager::SrcFileData::maybeCreateRealFileName(std::u16string_view rOwnDocName)
2741 {
2742     if (maRelativeName.isEmpty())
2743         // No relative path given.  Nothing to do.
2744         return;
2745
2746     if (!maRealFileName.isEmpty())
2747         // Real file name already created.  Nothing to do.
2748         return;
2749
2750     // Formulate the absolute file path from the relative path.
2751     const OUString& rRelPath = maRelativeName;
2752     INetURLObject aBaseURL(rOwnDocName);
2753     aBaseURL.insertName(u"content.xml");
2754     bool bWasAbs = false;
2755     maRealFileName = aBaseURL.smartRel2Abs(rRelPath, bWasAbs).GetMainURL(INetURLObject::DecodeMechanism::NONE);
2756 }
2757
2758 void ScExternalRefManager::maybeCreateRealFileName(sal_uInt16 nFileId)
2759 {
2760     if (nFileId >= maSrcFiles.size())
2761         return;
2762
2763     maSrcFiles[nFileId].maybeCreateRealFileName(getOwnDocumentName());
2764 }
2765
2766 OUString ScExternalRefManager::getOwnDocumentName() const
2767 {
2768     if (comphelper::IsFuzzing())
2769         return u"file:///tmp/document"_ustr;
2770
2771     ScDocShell* pShell = mrDoc.GetDocumentShell();
2772     if (!pShell)
2773         // This should not happen!
2774         return OUString();
2775
2776     SfxMedium* pMed = pShell->GetMedium();
2777     if (!pMed)
2778         return OUString();
2779
2780     return pMed->GetName();
2781 }
2782
2783 bool ScExternalRefManager::isOwnDocument(std::u16string_view rFile) const
2784 {
2785     return getOwnDocumentName() == rFile;
2786 }
2787
2788 void ScExternalRefManager::convertToAbsName(OUString& rFile) const
2789 {
2790     // unsaved documents have no AbsName
2791     ScDocShell* pShell = static_cast<ScDocShell*>(SfxObjectShell::GetFirst(checkSfxObjectShell<ScDocShell>, false));
2792     while (pShell)
2793     {
2794         if (rFile == pShell->GetName())
2795             return;
2796
2797         pShell = static_cast<ScDocShell*>(SfxObjectShell::GetNext(*pShell, checkSfxObjectShell<ScDocShell>, false));
2798     }
2799
2800     ScDocShell* pDocShell = mrDoc.GetDocumentShell();
2801     rFile = ScGlobal::GetAbsDocName(rFile, pDocShell);
2802 }
2803
2804 sal_uInt16 ScExternalRefManager::getExternalFileId(const OUString& rFile)
2805 {
2806     vector<SrcFileData>::const_iterator itrBeg = maSrcFiles.begin(), itrEnd = maSrcFiles.end();
2807     vector<SrcFileData>::const_iterator itr = find_if(itrBeg, itrEnd, FindSrcFileByName(rFile));
2808     if (itr != itrEnd)
2809     {
2810         size_t nId = distance(itrBeg, itr);
2811         return static_cast<sal_uInt16>(nId);
2812     }
2813
2814     SrcFileData aData;
2815     aData.maFileName = rFile;
2816     maSrcFiles.push_back(aData);
2817     return static_cast<sal_uInt16>(maSrcFiles.size() - 1);
2818 }
2819
2820 const OUString* ScExternalRefManager::getExternalFileName(sal_uInt16 nFileId, bool bForceOriginal)
2821 {
2822     if (nFileId >= maSrcFiles.size())
2823         return nullptr;
2824
2825     if (bForceOriginal)
2826         return &maSrcFiles[nFileId].maFileName;
2827
2828     maybeCreateRealFileName(nFileId);
2829
2830     if (!maSrcFiles[nFileId].maRealFileName.isEmpty())
2831         return &maSrcFiles[nFileId].maRealFileName;
2832
2833     return &maSrcFiles[nFileId].maFileName;
2834 }
2835
2836 sal_uInt16 ScExternalRefManager::convertFileIdToUsedFileId(sal_uInt16 nFileId)
2837 {
2838     if (!mbSkipUnusedFileIds)
2839         return nFileId;
2840     else
2841         return maConvertFileIdToUsedFileId[nFileId];
2842 }
2843
2844 void ScExternalRefManager::setSkipUnusedFileIds(std::vector<sal_uInt16>& rExternFileIds)
2845 {
2846     mbSkipUnusedFileIds = true;
2847     maConvertFileIdToUsedFileId.resize(maSrcFiles.size());
2848     std::fill(maConvertFileIdToUsedFileId.begin(), maConvertFileIdToUsedFileId.end(), 0);
2849     int nUsedCount = 0;
2850     for (auto nEntry : rExternFileIds)
2851     {
2852         maConvertFileIdToUsedFileId[nEntry] = nUsedCount++;
2853     }
2854 }
2855
2856 void ScExternalRefManager::disableSkipUnusedFileIds()
2857 {
2858     mbSkipUnusedFileIds = false;
2859 }
2860
2861 std::vector<OUString> ScExternalRefManager::getAllCachedExternalFileNames() const
2862 {
2863     std::vector<OUString> aNames;
2864     aNames.reserve(maSrcFiles.size());
2865     for (const SrcFileData& rData : maSrcFiles)
2866     {
2867         aNames.push_back(rData.maFileName);
2868     }
2869
2870     return aNames;
2871 }
2872
2873 bool ScExternalRefManager::hasExternalFile(sal_uInt16 nFileId) const
2874 {
2875     return nFileId < maSrcFiles.size();
2876 }
2877
2878 bool ScExternalRefManager::hasExternalFile(const OUString& rFile) const
2879 {
2880     return ::std::any_of(maSrcFiles.begin(), maSrcFiles.end(), FindSrcFileByName(rFile));
2881 }
2882
2883 const ScExternalRefManager::SrcFileData* ScExternalRefManager::getExternalFileData(sal_uInt16 nFileId) const
2884 {
2885     if (nFileId >= maSrcFiles.size())
2886         return nullptr;
2887
2888     return &maSrcFiles[nFileId];
2889 }
2890
2891 const OUString* ScExternalRefManager::getRealTableName(sal_uInt16 nFileId, const OUString& rTabName) const
2892 {
2893     return maRefCache.getRealTableName(nFileId, rTabName);
2894 }
2895
2896 const OUString* ScExternalRefManager::getRealRangeName(sal_uInt16 nFileId, const OUString& rRangeName) const
2897 {
2898     return maRefCache.getRealRangeName(nFileId, rRangeName);
2899 }
2900
2901 template<typename MapContainer>
2902 static void lcl_removeByFileId(sal_uInt16 nFileId, MapContainer& rMap)
2903 {
2904     typename MapContainer::iterator itr = rMap.find(nFileId);
2905     if (itr != rMap.end())
2906     {
2907         // Close this document shell.
2908         itr->second.maShell->DoClose();
2909         rMap.erase(itr);
2910     }
2911 }
2912
2913 void ScExternalRefManager::clearCache(sal_uInt16 nFileId)
2914 {
2915     maRefCache.clearCache(nFileId);
2916 }
2917
2918 namespace {
2919
2920 class RefCacheFiller : public sc::ColumnSpanSet::ColumnAction
2921 {
2922     svl::SharedStringPool& mrStrPool;
2923
2924     ScExternalRefCache& mrRefCache;
2925     ScExternalRefCache::TableTypeRef mpRefTab;
2926     sal_uInt16 mnFileId;
2927     ScColumn* mpCurCol;
2928     sc::ColumnBlockConstPosition maBlockPos;
2929
2930 public:
2931     RefCacheFiller( svl::SharedStringPool& rStrPool, ScExternalRefCache& rRefCache, sal_uInt16 nFileId ) :
2932         mrStrPool(rStrPool), mrRefCache(rRefCache), mnFileId(nFileId), mpCurCol(nullptr) {}
2933
2934     virtual void startColumn( ScColumn* pCol ) override
2935     {
2936         mpCurCol = pCol;
2937         if (!mpCurCol)
2938             return;
2939
2940         mpCurCol->InitBlockPosition(maBlockPos);
2941         mpRefTab = mrRefCache.getCacheTable(mnFileId, mpCurCol->GetTab());
2942     }
2943
2944     virtual void execute( SCROW nRow1, SCROW nRow2, bool bVal ) override
2945     {
2946         if (!mpCurCol || !bVal)
2947             return;
2948
2949         if (!mpRefTab)
2950             return;
2951
2952         for (SCROW nRow = nRow1; nRow <= nRow2; ++nRow)
2953         {
2954             ScExternalRefCache::TokenRef pTok;
2955             ScRefCellValue aCell = mpCurCol->GetCellValue(maBlockPos, nRow);
2956             switch (aCell.getType())
2957             {
2958                 case CELLTYPE_STRING:
2959                 case CELLTYPE_EDIT:
2960                 {
2961                     OUString aStr = aCell.getString(&mpCurCol->GetDoc());
2962                     svl::SharedString aSS = mrStrPool.intern(aStr);
2963                     pTok.reset(new formula::FormulaStringToken(std::move(aSS)));
2964                 }
2965                 break;
2966                 case CELLTYPE_VALUE:
2967                     pTok.reset(new formula::FormulaDoubleToken(aCell.getDouble()));
2968                 break;
2969                 case CELLTYPE_FORMULA:
2970                 {
2971                     sc::FormulaResultValue aRes = aCell.getFormula()->GetResult();
2972                     switch (aRes.meType)
2973                     {
2974                         case sc::FormulaResultValue::Value:
2975                             pTok.reset(new formula::FormulaDoubleToken(aRes.mfValue));
2976                         break;
2977                         case sc::FormulaResultValue::String:
2978                         {
2979                             // Re-intern the string to the host document pool.
2980                             svl::SharedString aInterned = mrStrPool.intern(aRes.maString.getString());
2981                             pTok.reset(new formula::FormulaStringToken(std::move(aInterned)));
2982                         }
2983                         break;
2984                         case sc::FormulaResultValue::Error:
2985                         case sc::FormulaResultValue::Invalid:
2986                         default:
2987                             pTok.reset(new FormulaErrorToken(FormulaError::NoValue));
2988                     }
2989                 }
2990                 break;
2991                 default:
2992                     pTok.reset(new FormulaErrorToken(FormulaError::NoValue));
2993             }
2994
2995             if (pTok)
2996             {
2997                 // Cache this cell.
2998                 mpRefTab->setCell(mpCurCol->GetCol(), nRow, pTok, mpCurCol->GetNumberFormat(mpCurCol->GetDoc().GetNonThreadedContext(), nRow));
2999                 mpRefTab->setCachedCell(mpCurCol->GetCol(), nRow);
3000             }
3001         }
3002     };
3003 };
3004
3005 }
3006
3007 bool ScExternalRefManager::refreshSrcDocument(sal_uInt16 nFileId)
3008 {
3009     SfxObjectShellRef xDocShell;
3010     try
3011     {
3012         OUString aFilter;
3013         xDocShell = loadSrcDocument(nFileId, aFilter);
3014     }
3015     catch ( const css::uno::Exception& ) {}
3016
3017     if (!xDocShell.is())
3018         // Failed to load the document.  Bail out.
3019         return false;
3020
3021     ScDocShell& rDocSh = static_cast<ScDocShell&>(*xDocShell);
3022     ScDocument& rSrcDoc = rDocSh.GetDocument();
3023
3024     sc::ColumnSpanSet aCachedArea;
3025     maRefCache.getAllCachedDataSpans(rSrcDoc, nFileId, aCachedArea);
3026
3027     // Clear the existing cache, and refill it.  Make sure we keep the
3028     // existing cache table instances here.
3029     maRefCache.clearCacheTables(nFileId);
3030     RefCacheFiller aAction(mrDoc.GetSharedStringPool(), maRefCache, nFileId);
3031     aCachedArea.executeColumnAction(rSrcDoc, aAction);
3032
3033     DocShellMap::iterator it = maDocShells.find(nFileId);
3034     if (it != maDocShells.end())
3035     {
3036         it->second.maShell->DoClose();
3037         it->second.maShell = std::move(xDocShell);
3038         it->second.maLastAccess = tools::Time(tools::Time::SYSTEM);
3039     }
3040     else
3041     {
3042         SrcShell aSrcDoc;
3043         aSrcDoc.maShell = std::move(xDocShell);
3044         aSrcDoc.maLastAccess = tools::Time(tools::Time::SYSTEM);
3045         cacheNewDocShell(nFileId, aSrcDoc);
3046     }
3047
3048     // Update all cells containing names from this source document.
3049     refreshAllRefCells(nFileId);
3050
3051     notifyAllLinkListeners(nFileId, LINK_MODIFIED);
3052
3053     return true;
3054 }
3055
3056 void ScExternalRefManager::breakLink(sal_uInt16 nFileId)
3057 {
3058     // Turn all formula cells referencing this external document into static
3059     // cells.
3060     RefCellMap::iterator itrRefs = maRefCells.find(nFileId);
3061     if (itrRefs != maRefCells.end())
3062     {
3063         // Make a copy because removing the formula cells below will modify
3064         // the original container.
3065         RefCellSet aSet = itrRefs->second;
3066         for_each(aSet.begin(), aSet.end(), ConvertFormulaToStatic(&mrDoc));
3067         maRefCells.erase(nFileId);
3068     }
3069
3070     // Remove all named ranges that reference this document.
3071
3072     // Global named ranges.
3073     ScRangeName* pRanges = mrDoc.GetRangeName();
3074     if (pRanges)
3075         removeRangeNamesBySrcDoc(*pRanges, nFileId);
3076
3077     // Sheet-local named ranges.
3078     for (SCTAB i = 0, n = mrDoc.GetTableCount(); i < n; ++i)
3079     {
3080         pRanges = mrDoc.GetRangeName(i);
3081         if (pRanges)
3082             removeRangeNamesBySrcDoc(*pRanges, nFileId);
3083     }
3084
3085     clearCache(nFileId);
3086     lcl_removeByFileId(nFileId, maDocShells);
3087
3088     if (maDocShells.empty())
3089         maSrcDocTimer.Stop();
3090
3091     LinkedDocMap::iterator itr = maLinkedDocs.find(nFileId);
3092     if (itr != maLinkedDocs.end())
3093         itr->second = false;
3094
3095     notifyAllLinkListeners(nFileId, LINK_BROKEN);
3096 }
3097
3098 void ScExternalRefManager::switchSrcFile(sal_uInt16 nFileId, const OUString& rNewFile, const OUString& rNewFilter)
3099 {
3100     maSrcFiles[nFileId].maFileName = rNewFile;
3101     maSrcFiles[nFileId].maRelativeName.clear();
3102     maSrcFiles[nFileId].maRealFileName.clear();
3103     if (maSrcFiles[nFileId].maFilterName != rNewFilter)
3104     {
3105         // Filter type has changed.
3106         maSrcFiles[nFileId].maFilterName = rNewFilter;
3107         maSrcFiles[nFileId].maFilterOptions.clear();
3108     }
3109     refreshSrcDocument(nFileId);
3110 }
3111
3112 void ScExternalRefManager::setRelativeFileName(sal_uInt16 nFileId, const OUString& rRelUrl)
3113 {
3114     if (nFileId >= maSrcFiles.size())
3115         return;
3116     maSrcFiles[nFileId].maRelativeName = rRelUrl;
3117 }
3118
3119 void ScExternalRefManager::setFilterData(sal_uInt16 nFileId, const OUString& rFilterName, const OUString& rOptions)
3120 {
3121     if (nFileId >= maSrcFiles.size())
3122         return;
3123     maSrcFiles[nFileId].maFilterName = rFilterName;
3124     maSrcFiles[nFileId].maFilterOptions = rOptions;
3125 }
3126
3127 void ScExternalRefManager::clear()
3128 {
3129     for (auto& rEntry : maLinkListeners)
3130     {
3131         for (auto& it : rEntry.second)
3132         {
3133             it->notify(0, OH_NO_WE_ARE_GOING_TO_DIE);
3134         }
3135     }
3136
3137     for (auto& rEntry : maDocShells)
3138         rEntry.second.maShell->DoClose();
3139
3140     maDocShells.clear();
3141     maSrcDocTimer.Stop();
3142 }
3143
3144 bool ScExternalRefManager::hasExternalData() const
3145 {
3146     return !maSrcFiles.empty();
3147 }
3148
3149 void ScExternalRefManager::resetSrcFileData(const OUString& rBaseFileUrl)
3150 {
3151     for (auto& rSrcFile : maSrcFiles)
3152     {
3153         // Re-generate relative file name from the absolute file name.
3154         OUString aAbsName = rSrcFile.maRealFileName;
3155         if (aAbsName.isEmpty())
3156             aAbsName = rSrcFile.maFileName;
3157
3158         rSrcFile.maRelativeName = URIHelper::simpleNormalizedMakeRelative(
3159             rBaseFileUrl, aAbsName);
3160     }
3161 }
3162
3163 void ScExternalRefManager::updateAbsAfterLoad()
3164 {
3165     OUString aOwn( getOwnDocumentName() );
3166     for (auto& rSrcFile : maSrcFiles)
3167     {
3168         // update maFileName to the real file name,
3169         // to be called when the original name is no longer needed (after CompileXML)
3170
3171         rSrcFile.maybeCreateRealFileName( aOwn );
3172         OUString aReal = rSrcFile.maRealFileName;
3173         if (!aReal.isEmpty())
3174             rSrcFile.maFileName = aReal;
3175     }
3176 }
3177
3178 void ScExternalRefManager::removeRefCell(ScFormulaCell* pCell)
3179 {
3180     for_each(maRefCells.begin(), maRefCells.end(), RemoveFormulaCell(pCell));
3181 }
3182
3183 void ScExternalRefManager::addLinkListener(sal_uInt16 nFileId, LinkListener* pListener)
3184 {
3185     LinkListenerMap::iterator itr = maLinkListeners.find(nFileId);
3186     if (itr == maLinkListeners.end())
3187     {
3188         pair<LinkListenerMap::iterator, bool> r = maLinkListeners.emplace(
3189             nFileId, LinkListeners());
3190         if (!r.second)
3191         {
3192             OSL_FAIL("insertion of new link listener list failed");
3193             return;
3194         }
3195
3196         itr = r.first;
3197     }
3198
3199     LinkListeners& rList = itr->second;
3200     rList.insert(pListener);
3201 }
3202
3203 void ScExternalRefManager::removeLinkListener(sal_uInt16 nFileId, LinkListener* pListener)
3204 {
3205     LinkListenerMap::iterator itr = maLinkListeners.find(nFileId);
3206     if (itr == maLinkListeners.end())
3207         // no listeners for a specified file.
3208         return;
3209
3210     LinkListeners& rList = itr->second;
3211     rList.erase(pListener);
3212
3213     if (rList.empty())
3214         // No more listeners for this file.  Remove its entry.
3215         maLinkListeners.erase(itr);
3216 }
3217
3218 void ScExternalRefManager::removeLinkListener(LinkListener* pListener)
3219 {
3220     for (auto& rEntry : maLinkListeners)
3221         rEntry.second.erase(pListener);
3222 }
3223
3224 void ScExternalRefManager::notifyAllLinkListeners(sal_uInt16 nFileId, LinkUpdateType eType)
3225 {
3226     LinkListenerMap::iterator itr = maLinkListeners.find(nFileId);
3227     if (itr == maLinkListeners.end())
3228         // no listeners for a specified file.
3229         return;
3230
3231     LinkListeners& rList = itr->second;
3232     for_each(rList.begin(), rList.end(), NotifyLinkListener(nFileId, eType));
3233 }
3234
3235 void ScExternalRefManager::purgeStaleSrcDocument(sal_Int32 nTimeOut)
3236 {
3237     // To avoid potentially freezing Calc, we close one stale document at a time.
3238     DocShellMap::iterator itr = std::find_if(maDocShells.begin(), maDocShells.end(),
3239         [nTimeOut](const DocShellMap::value_type& rEntry) {
3240             // in 100th of a second.
3241             sal_Int32 nSinceLastAccess = (tools::Time( tools::Time::SYSTEM ) - rEntry.second.maLastAccess).GetTime();
3242             return nSinceLastAccess >= nTimeOut;
3243         });
3244     if (itr != maDocShells.end())
3245     {
3246         // Timed out.  Let's close this.
3247         itr->second.maShell->DoClose();
3248         maDocShells.erase(itr);
3249     }
3250
3251     if (maDocShells.empty())
3252         maSrcDocTimer.Stop();
3253 }
3254
3255 sal_uInt32 ScExternalRefManager::getMappedNumberFormat(sal_uInt16 nFileId, sal_uInt32 nNumFmt, const ScDocument& rSrcDoc)
3256 {
3257     NumFmtMap::iterator itr = maNumFormatMap.find(nFileId);
3258     if (itr == maNumFormatMap.end())
3259     {
3260         // Number formatter map is not initialized for this external document.
3261         pair<NumFmtMap::iterator, bool> r = maNumFormatMap.emplace(
3262             nFileId, SvNumberFormatterMergeMap());
3263
3264         if (!r.second)
3265             // insertion failed.
3266             return nNumFmt;
3267
3268         itr = r.first;
3269         mrDoc.GetFormatTable()->MergeFormatter(*rSrcDoc.GetFormatTable());
3270         SvNumberFormatterMergeMap aMap = mrDoc.GetFormatTable()->ConvertMergeTableToMap();
3271         itr->second.swap(aMap);
3272     }
3273     const SvNumberFormatterMergeMap& rMap = itr->second;
3274     SvNumberFormatterMergeMap::const_iterator itrNumFmt = rMap.find(nNumFmt);
3275     if (itrNumFmt != rMap.end())
3276         // mapped value found.
3277         return itrNumFmt->second;
3278
3279     return nNumFmt;
3280 }
3281
3282 void ScExternalRefManager::transformUnsavedRefToSavedRef( SfxObjectShell* pShell )
3283 {
3284     DocShellMap::iterator itr = maUnsavedDocShells.begin();
3285     while( itr != maUnsavedDocShells.end() )
3286     {
3287         if ( itr->second.maShell.get() == pShell )
3288         {
3289             // found that the shell is marked as unsaved
3290             OUString aFileURL = pShell->GetMedium()->GetURLObject().GetMainURL(INetURLObject::DecodeMechanism::ToIUri);
3291             switchSrcFile(itr->first, aFileURL, OUString());
3292             EndListening(*pShell);
3293             itr = maUnsavedDocShells.erase(itr);
3294         }
3295         else
3296             ++itr;
3297     }
3298 }
3299
3300 void ScExternalRefManager::Notify( SfxBroadcaster&, const SfxHint& rHint )
3301 {
3302     if (rHint.GetId() != SfxHintId::ThisIsAnSfxEventHint)
3303         return;
3304
3305     switch (static_cast<const SfxEventHint&>(rHint).GetEventId())
3306     {
3307         case SfxEventHintId::PrepareCloseDoc:
3308             {
3309                 std::unique_ptr<weld::MessageDialog> xWarn(Application::CreateMessageDialog(ScDocShell::GetActiveDialogParent(),
3310                                                            VclMessageType::Warning, VclButtonsType::Ok,
3311                                                            ScResId(STR_CLOSE_WITH_UNSAVED_REFS)));
3312                 xWarn->run();
3313             }
3314             break;
3315         case SfxEventHintId::SaveDocDone:
3316         case SfxEventHintId::SaveAsDocDone:
3317             {
3318                 rtl::Reference<SfxObjectShell> pObjShell = static_cast<const SfxEventHint&>( rHint ).GetObjShell();
3319                 transformUnsavedRefToSavedRef(pObjShell.get());
3320             }
3321             break;
3322         default:
3323             break;
3324     }
3325 }
3326
3327 IMPL_LINK(ScExternalRefManager, TimeOutHdl, Timer*, pTimer, void)
3328 {
3329     if (pTimer == &maSrcDocTimer)
3330         purgeStaleSrcDocument(SRCDOC_LIFE_SPAN);
3331 }
3332
3333 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */