package/source/zipapi/ZipOutputStream.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  *
   9  * This file incorporates work covered by the following license notice:
  10  *
  11  *   Licensed to the Apache Software Foundation (ASF) under one or more
  12  *   contributor license agreements. See the NOTICE file distributed
  13  *   with this work for additional information regarding copyright
  14  *   ownership. The ASF licenses this file to you under the Apache
  15  *   License, Version 2.0 (the "License"); you may not use this file
  16  *   except in compliance with the License. You may obtain a copy of
  17  *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
  18  */
  19
  20 #include <ZipOutputStream.hxx>
  21
  22 #include <com/sun/star/packages/zip/ZipConstants.hpp>
  23 #include <com/sun/star/io/IOException.hpp>
  24 #include <com/sun/star/io/XInputStream.hpp>
  25 #include <comphelper/storagehelper.hxx>
  26
  27 #include <osl/time.h>
  28 #include <osl/thread.hxx>
  29
  30 #include <PackageConstants.hxx>
  31 #include <ZipEntry.hxx>
  32 #include <ZipOutputEntry.hxx>
  33 #include <ZipPackageStream.hxx>
  34
  35 #include <thread>
  36
  37 using namespace com::sun::star;
  38 using namespace com::sun::star::io;
  39 using namespace com::sun::star::uno;
  40 using namespace com::sun::star::packages::zip::ZipConstants;
  41
  42 /** This class is used to write Zip files
  43  */
  44 ZipOutputStream::ZipOutputStream( const uno::Reference < io::XOutputStream > &xOStream )
  45 : m_xStream(xOStream)
  46 , mpThreadTaskTag( comphelper::ThreadPool::createThreadTaskTag() )
  47 , m_aChucker(xOStream)
  48 , m_pCurrentEntry(nullptr)
  49 {
  50 }
  51
  52 ZipOutputStream::~ZipOutputStream()
  53 {
  54 }
  55
  56 void ZipOutputStream::setEntry(ZipEntry& rEntry)
  57 {
  58     if (rEntry.nTime == -1)
  59         rEntry.nTime = getCurrentDosTime();
  60     if (rEntry.nMethod == -1)
  61         rEntry.nMethod = DEFLATED;
  62     rEntry.nVersion = 20;
  63     rEntry.nFlag = 1 << 11;
  64     if (rEntry.nSize == -1 || rEntry.nCompressedSize == -1 ||
  65         rEntry.nCrc == -1)
  66     {
  67         rEntry.nSize = rEntry.nCompressedSize = 0;
  68         rEntry.nFlag |= 8;
  69     }
  70 }
  71
  72 void ZipOutputStream::addDeflatingThreadTask( ZipOutputEntryInThread *pEntry, std::unique_ptr<comphelper::ThreadTask> pTask )
  73 {
  74     comphelper::ThreadPool::getSharedOptimalPool().pushTask(std::move(pTask));
  75     m_aEntries.push_back(pEntry);
  76 }
  77
  78 void ZipOutputStream::rawWrite( const Sequence< sal_Int8 >& rBuffer )
  79 {
  80     m_aChucker.WriteBytes( rBuffer );
  81 }
  82
  83 void ZipOutputStream::rawCloseEntry( bool bEncrypt )
  84 {
  85     assert(m_pCurrentEntry && "Forgot to call writeLOC()?");
  86     if ( m_pCurrentEntry->nMethod == DEFLATED && ( m_pCurrentEntry->nFlag & 8 ) )
  87         writeDataDescriptor(*m_pCurrentEntry);
  88
  89     if (bEncrypt)
  90     {
  91         m_pCurrentEntry->nMethod = STORED;
  92         assert(m_pCurrentEntry->nSize == m_pCurrentEntry->nCompressedSize);
  93     }
  94
  95     m_pCurrentEntry = nullptr;
  96 }
  97
  98 void ZipOutputStream::consumeScheduledThreadTaskEntry(std::unique_ptr<ZipOutputEntryInThread> pCandidate)
  99 {
 100     //Any exceptions thrown in the threads were caught and stored for now
 101     const std::exception_ptr& rCaughtException(pCandidate->getParallelDeflateException());
 102     if (rCaughtException)
 103     {
 104         m_aDeflateException = rCaughtException; // store it for later throwing
 105         // the exception handler in DeflateThreadTask should have cleaned temp file
 106         return;
 107     }
 108
 109     writeLOC(pCandidate->moveZipEntry(), pCandidate->isEncrypt());
 110
 111     sal_Int32 nRead;
 112     uno::Sequence< sal_Int8 > aSequence(n_ConstBufferSize);
 113     uno::Reference< io::XInputStream > xInput = pCandidate->getData();
 114     do
 115     {
 116         nRead = xInput->readBytes(aSequence, n_ConstBufferSize);
 117         if (nRead < n_ConstBufferSize)
 118             aSequence.realloc(nRead);
 119
 120         rawWrite(aSequence);
 121     }
 122     while (nRead == n_ConstBufferSize);
 123     xInput.clear();
 124
 125     rawCloseEntry(pCandidate->isEncrypt());
 126
 127     pCandidate->getZipPackageStream()->successfullyWritten(pCandidate->getZipEntry());
 128     pCandidate->deleteBufferFile();
 129 }
 130
 131 void ZipOutputStream::consumeFinishedScheduledThreadTaskEntries()
 132 {
 133     std::vector< ZipOutputEntryInThread* > aNonFinishedEntries;
 134
 135     for(ZipOutputEntryInThread* pEntry : m_aEntries)
 136     {
 137         if(pEntry->isFinished())
 138         {
 139             consumeScheduledThreadTaskEntry(std::unique_ptr<ZipOutputEntryInThread>(pEntry));
 140         }
 141         else
 142         {
 143             aNonFinishedEntries.push_back(pEntry);
 144         }
 145     }
 146
 147     // always reset to non-consumed entries
 148     m_aEntries = std::move(aNonFinishedEntries);
 149 }
 150
 151 void ZipOutputStream::reduceScheduledThreadTasksToGivenNumberOrLess(std::size_t nThreadTasks)
 152 {
 153     while(m_aEntries.size() > nThreadTasks)
 154     {
 155         consumeFinishedScheduledThreadTaskEntries();
 156
 157         if(m_aEntries.size() > nThreadTasks)
 158         {
 159             std::this_thread::sleep_for(std::chrono::microseconds(100));
 160         }
 161     }
 162 }
 163
 164 void ZipOutputStream::finish()
 165 {
 166     assert(!m_aZipList.empty() && "Zip file must have at least one entry!");
 167
 168     // Wait for all thread tasks to finish & write
 169     comphelper::ThreadPool::getSharedOptimalPool().waitUntilDone(mpThreadTaskTag);
 170
 171     // consume all processed entries
 172     while(!m_aEntries.empty())
 173     {
 174         ZipOutputEntryInThread* pCandidate = m_aEntries.back();
 175         m_aEntries.pop_back();
 176         consumeScheduledThreadTaskEntry(std::unique_ptr<ZipOutputEntryInThread>(pCandidate));
 177     }
 178
 179     sal_Int32 nOffset= static_cast < sal_Int32 > (m_aChucker.GetPosition());
 180     for (auto& p : m_aZipList)
 181     {
 182         writeCEN( *p );
 183     }
 184     writeEND( nOffset, static_cast < sal_Int32 > (m_aChucker.GetPosition()) - nOffset);
 185     m_aZipList.clear();
 186
 187     if (m_aDeflateException)
 188     {   // throw once all thread tasks are finished and m_aEntries can be released
 189         std::rethrow_exception(m_aDeflateException);
 190     }
 191 }
 192
 193 const css::uno::Reference< css::io::XOutputStream >& ZipOutputStream::getStream() const
 194 {
 195     return m_xStream;
 196 }
 197
 198 void ZipOutputStream::writeEND(sal_uInt32 nOffset, sal_uInt32 nLength)
 199 {
 200     m_aChucker.WriteInt32( ENDSIG );
 201     m_aChucker.WriteInt16( 0 );
 202     m_aChucker.WriteInt16( 0 );
 203     m_aChucker.WriteInt16( m_aZipList.size() );
 204     m_aChucker.WriteInt16( m_aZipList.size() );
 205     m_aChucker.WriteUInt32( nLength );
 206     m_aChucker.WriteUInt32( nOffset );
 207     m_aChucker.WriteInt16( 0 );
 208 }
 209
 210 static sal_uInt32 getTruncated( sal_Int64 nNum, bool *pIsTruncated )
 211 {
 212     if( nNum >= 0xffffffff )
 213     {
 214         *pIsTruncated = true;
 215         return 0xffffffff;
 216     }
 217     else
 218         return static_cast< sal_uInt32 >( nNum );
 219 }
 220
 221 void ZipOutputStream::writeCEN( const ZipEntry &rEntry )
 222 {
 223     if ( !::comphelper::OStorageHelper::IsValidZipEntryFileName( rEntry.sPath, true ) )
 224         throw IOException(u"Unexpected character is used in file name."_ustr );
 225
 226     OString sUTF8Name = OUStringToOString( rEntry.sPath, RTL_TEXTENCODING_UTF8 );
 227     sal_Int16 nNameLength       = static_cast < sal_Int16 > ( sUTF8Name.getLength() );
 228
 229     m_aChucker.WriteInt32( CENSIG );
 230     m_aChucker.WriteInt16( rEntry.nVersion );
 231     m_aChucker.WriteInt16( rEntry.nVersion );
 232     m_aChucker.WriteInt16( rEntry.nFlag );
 233     m_aChucker.WriteInt16( rEntry.nMethod );
 234     bool bWrite64Header = false;
 235
 236     m_aChucker.WriteUInt32( rEntry.nTime );
 237     m_aChucker.WriteUInt32( rEntry.nCrc );
 238     m_aChucker.WriteUInt32( getTruncated( rEntry.nCompressedSize, &bWrite64Header ) );
 239     m_aChucker.WriteUInt32( getTruncated( rEntry.nSize, &bWrite64Header ) );
 240     sal_uInt32 nOffset32bit = getTruncated( rEntry.nOffset, &bWrite64Header );
 241     m_aChucker.WriteInt16(nNameLength);
 242     m_aChucker.WriteInt16( bWrite64Header? 32 : 0 );    //in ZIP64 case extra field is 32byte
 243     m_aChucker.WriteInt16( 0 );
 244     m_aChucker.WriteInt16( 0 );
 245     m_aChucker.WriteInt16( 0 );
 246     m_aChucker.WriteInt32( 0 );
 247     m_aChucker.WriteUInt32( nOffset32bit );
 248
 249     Sequence < sal_Int8 > aSequence( reinterpret_cast<sal_Int8 const *>(sUTF8Name.getStr()), sUTF8Name.getLength() );
 250     m_aChucker.WriteBytes( aSequence );
 251
 252     if (bWrite64Header)
 253     {
 254         writeExtraFields( rEntry );
 255     }
 256 }
 257
 258 void ZipOutputStream::writeDataDescriptor(const ZipEntry& rEntry)
 259 {
 260     bool bWrite64Header = false;
 261
 262     m_aChucker.WriteInt32( EXTSIG );
 263     m_aChucker.WriteUInt32( rEntry.nCrc );
 264     // For ZIP64(tm) format archives, the compressed and uncompressed sizes are 8 bytes each.
 265     // TODO: Not sure if this is the "when ZIP64(tm) format is used"
 266     bWrite64Header = rEntry.nCompressedSize >= 0x100000000 || rEntry.nSize >= 0x100000000;
 267     if (!bWrite64Header)
 268     {
 269         m_aChucker.WriteUInt32( static_cast<sal_uInt32>(rEntry.nCompressedSize) );
 270         m_aChucker.WriteUInt32( static_cast<sal_uInt32>(rEntry.nSize) );
 271     }
 272     else
 273     {
 274         m_aChucker.WriteUInt64( rEntry.nCompressedSize );
 275         m_aChucker.WriteUInt64( rEntry.nSize );
 276     }
 277 }
 278
 279 void ZipOutputStream::writeExtraFields(const ZipEntry& rEntry)
 280 {
 281     //Could contain more fields, now we only save Zip64 extended information
 282     m_aChucker.WriteInt16( 1 );  //id of Zip64 extended information extra field
 283     m_aChucker.WriteInt16( 28 ); //data size of this field = 3*8+4 byte
 284     m_aChucker.WriteUInt64( rEntry.nSize );
 285     m_aChucker.WriteUInt64( rEntry.nCompressedSize );
 286     m_aChucker.WriteUInt64( rEntry.nOffset );
 287     m_aChucker.WriteInt32( 0 );  //Number of the disk on which this file starts
 288 }
 289
 290 void ZipOutputStream::writeLOC(std::unique_ptr<ZipEntry>&& pEntry, bool bEncrypt)
 291 {
 292     assert(!m_pCurrentEntry && "Forgot to close an entry with rawCloseEntry()?");
 293     m_aZipList.push_back(std::move(pEntry));
 294     m_pCurrentEntry = m_aZipList.back().get();
 295     const ZipEntry &rEntry = *m_pCurrentEntry;
 296
 297     if ( !::comphelper::OStorageHelper::IsValidZipEntryFileName( rEntry.sPath, true ) )
 298         throw IOException(u"Unexpected character is used in file name."_ustr );
 299
 300     OString sUTF8Name = OUStringToOString( rEntry.sPath, RTL_TEXTENCODING_UTF8 );
 301     sal_Int16 nNameLength       = static_cast < sal_Int16 > ( sUTF8Name.getLength() );
 302
 303     m_aChucker.WriteInt32( LOCSIG );
 304     m_aChucker.WriteInt16( rEntry.nVersion );
 305
 306     m_aChucker.WriteInt16( rEntry.nFlag );
 307     // If it's an encrypted entry, we pretend its stored plain text
 308     if (bEncrypt)
 309         m_aChucker.WriteInt16( STORED );
 310     else
 311         m_aChucker.WriteInt16( rEntry.nMethod );
 312
 313     bool bWrite64Header = false;
 314
 315     m_aChucker.WriteUInt32( rEntry.nTime );
 316     if ((rEntry.nFlag & 8) == 8 )
 317     {
 318         m_aChucker.WriteInt32( 0 );
 319         m_aChucker.WriteInt32( 0 );
 320         m_aChucker.WriteInt32( 0 );
 321     }
 322     else
 323     {
 324         m_aChucker.WriteUInt32( rEntry.nCrc );
 325         m_aChucker.WriteUInt32( getTruncated( rEntry.nCompressedSize, &bWrite64Header ) );
 326         m_aChucker.WriteUInt32( getTruncated( rEntry.nSize, &bWrite64Header ) );
 327     }
 328     m_aChucker.WriteInt16( nNameLength );
 329     m_aChucker.WriteInt16( bWrite64Header ? 32 : 0 );
 330
 331     Sequence < sal_Int8 > aSequence( reinterpret_cast<sal_Int8 const *>(sUTF8Name.getStr()), sUTF8Name.getLength() );
 332     m_aChucker.WriteBytes( aSequence );
 333
 334     m_pCurrentEntry->nOffset = m_aChucker.GetPosition() - (LOCHDR + nNameLength);
 335
 336     if (bWrite64Header)
 337     {
 338         writeExtraFields(rEntry);
 339     }
 340 }
 341
 342 sal_uInt32 ZipOutputStream::getCurrentDosTime()
 343 {
 344     oslDateTime aDateTime;
 345     TimeValue aTimeValue;
 346     osl_getSystemTime ( &aTimeValue );
 347     osl_getDateTimeFromTimeValue( &aTimeValue, &aDateTime);
 348
 349     // at year 2108, there is an overflow
 350     // -> some decision needs to be made
 351     // how to handle the ZIP file format (just overflow?)
 352
 353     // if the current system time is before 1980,
 354     // then the time traveller will have to make a decision
 355     // how to handle the ZIP file format before it is invented
 356     // (just underflow?)
 357
 358     assert(aDateTime.Year > 1980 && aDateTime.Year < 2108);
 359
 360     sal_uInt32 nYear = static_cast <sal_uInt32> (aDateTime.Year);
 361
 362     if (nYear>=1980)
 363         nYear-=1980;
 364     else if (nYear>=80)
 365     {
 366         nYear-=80;
 367     }
 368     sal_uInt32 nResult = static_cast < sal_uInt32>( ( ( ( aDateTime.Day) +
 369                                           ( 32 * (aDateTime.Month)) +
 370                                           ( 512 * nYear ) ) << 16) |
 371                                         ( ( aDateTime.Seconds/2) +
 372                                             ( 32 * aDateTime.Minutes) +
 373                                           ( 2048 * static_cast <sal_uInt32 > (aDateTime.Hours) ) ) );
 374     return nResult;
 375 }
 376
 377 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */