oox/source/core/binarycodec.cxx

   1 /*************************************************************************
   2  *
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * Copyright 2008 by Sun Microsystems, Inc.
   6  *
   7  * OpenOffice.org - a multi-platform office productivity suite
   8  *
   9  * $RCSfile: binarycodec.cxx,v $
  10  * $Revision: 1.4 $
  11  *
  12  * This file is part of OpenOffice.org.
  13  *
  14  * OpenOffice.org is free software: you can redistribute it and/or modify
  15  * it under the terms of the GNU Lesser General Public License version 3
  16  * only, as published by the Free Software Foundation.
  17  *
  18  * OpenOffice.org is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU Lesser General Public License version 3 for more details
  22  * (a copy is included in the LICENSE file that accompanied this code).
  23  *
  24  * You should have received a copy of the GNU Lesser General Public License
  25  * version 3 along with OpenOffice.org.  If not, see
  26  * <http://www.openoffice.org/license.html>
  27  * for a copy of the LGPLv3 License.
  28  *
  29  ************************************************************************/
  30
  31 #include "oox/core/binarycodec.hxx"
  32 #include <algorithm>
  33 #include <string.h>
  34 #include <osl/diagnose.h>
  35
  36 namespace oox {
  37 namespace core {
  38
  39 // ============================================================================
  40
  41 namespace {
  42
  43 /** Rotates rnValue left by nBits bits. */
  44 template< typename Type >
  45 inline void lclRotateLeft( Type& rnValue, size_t nBits )
  46 {
  47     OSL_ENSURE( nBits < sizeof( Type ) * 8, "lclRotateLeft - rotation count overflow" );
  48     rnValue = static_cast< Type >( (rnValue << nBits) | (rnValue >> (sizeof( Type ) * 8 - nBits)) );
  49 }
  50
  51 /** Rotates the lower nWidth bits of rnValue left by nBits bits. */
  52 template< typename Type >
  53 inline void lclRotateLeft( Type& rnValue, size_t nBits, size_t nWidth )
  54 {
  55     OSL_ENSURE( (nBits < nWidth) && (nWidth < sizeof( Type ) * 8), "lclRotateLeft - rotation count overflow" );
  56     Type nMask = static_cast< Type >( (1UL << nWidth) - 1 );
  57     rnValue = static_cast< Type >(
  58         ((rnValue << nBits) | ((rnValue & nMask) >> (nWidth - nBits))) & nMask );
  59 }
  60
  61 sal_Int32 lclGetLen( const sal_uInt8* pnPassData, sal_Int32 nBufferSize )
  62 {
  63     sal_Int32 nLen = 0;
  64     while( (nLen < nBufferSize) && pnPassData[ nLen ] ) ++nLen;
  65     return nLen;
  66 }
  67
  68 sal_uInt16 lclGetKey( const sal_uInt8* pnPassData, sal_Int32 nBufferSize )
  69 {
  70     sal_Int32 nLen = lclGetLen( pnPassData, nBufferSize );
  71     if( nLen <= 0 ) return 0;
  72
  73     sal_uInt16 nKey = 0;
  74     sal_uInt16 nKeyBase = 0x8000;
  75     sal_uInt16 nKeyEnd = 0xFFFF;
  76     const sal_uInt8* pnChar = pnPassData + nLen - 1;
  77     for( sal_Int32 nIndex = 0; nIndex < nLen; ++nIndex, --pnChar )
  78     {
  79         sal_uInt8 cChar = *pnChar & 0x7F;
  80         for( size_t nBit = 0; nBit < 8; ++nBit )
  81         {
  82             lclRotateLeft( nKeyBase, 1 );
  83             if( nKeyBase & 1 ) nKeyBase ^= 0x1020;
  84             if( cChar & 1 ) nKey ^= nKeyBase;
  85             cChar >>= 1;
  86             lclRotateLeft( nKeyEnd, 1 );
  87             if( nKeyEnd & 1 ) nKeyEnd ^= 0x1020;
  88         }
  89     }
  90     return nKey ^ nKeyEnd;
  91 }
  92
  93 sal_uInt16 lclGetHash( const sal_uInt8* pnPassData, sal_Int32 nBufferSize )
  94 {
  95     sal_Int32 nLen = lclGetLen( pnPassData, nBufferSize );
  96
  97     sal_uInt16 nHash = static_cast< sal_uInt16 >( nLen );
  98     if( nLen > 0 )
  99         nHash ^= 0xCE4B;
 100
 101     const sal_uInt8* pnChar = pnPassData;
 102     for( sal_Int32 nIndex = 0; nIndex < nLen; ++nIndex, ++pnChar )
 103     {
 104         sal_uInt16 cChar = *pnChar;
 105         size_t nRot = static_cast< size_t >( (nIndex + 1) % 15 );
 106         lclRotateLeft( cChar, nRot, 15 );
 107         nHash ^= cChar;
 108     }
 109     return nHash;
 110 }
 111
 112 } // namespace
 113
 114 // ============================================================================
 115
 116 BinaryCodec_XOR::BinaryCodec_XOR( CodecType eCodecType ) :
 117     meCodecType( eCodecType ),
 118     mnOffset( 0 ),
 119     mnBaseKey( 0 ),
 120     mnHash( 0 )
 121 {
 122     (void)memset( mpnKey, 0, sizeof( mpnKey ) );
 123 }
 124
 125 BinaryCodec_XOR::~BinaryCodec_XOR()
 126 {
 127     (void)memset( mpnKey, 0, sizeof( mpnKey ) );
 128     mnBaseKey = mnHash = 0;
 129 }
 130
 131 void BinaryCodec_XOR::initKey( const sal_uInt8 pnPassData[ 16 ] )
 132 {
 133     // calculate base key and hash from passed password
 134     mnBaseKey = lclGetKey( pnPassData, 16 );
 135     mnHash = lclGetHash( pnPassData, 16 );
 136
 137      static const sal_uInt8 spnFillChars[] =
 138     {
 139         0xBB, 0xFF, 0xFF, 0xBA,
 140         0xFF, 0xFF, 0xB9, 0x80,
 141         0x00, 0xBE, 0x0F, 0x00,
 142         0xBF, 0x0F, 0x00
 143     };
 144
 145     (void)memcpy( mpnKey, pnPassData, 16 );
 146     sal_Int32 nIndex;
 147     sal_Int32 nLen = lclGetLen( pnPassData, 16 );
 148     const sal_uInt8* pnFillChar = spnFillChars;
 149     for( nIndex = nLen; nIndex < static_cast< sal_Int32 >( sizeof( mpnKey ) ); ++nIndex, ++pnFillChar )
 150         mpnKey[ nIndex ] = *pnFillChar;
 151
 152     // rotation of key values is application dependent
 153     size_t nRotateSize = 0;
 154     switch( meCodecType )
 155     {
 156         case CODEC_WORD:    nRotateSize = 7;    break;
 157         case CODEC_EXCEL:   nRotateSize = 2;    break;
 158         // compiler will warn, if new codec type is introduced and not handled here
 159     }
 160
 161     // use little-endian base key to create key array
 162     sal_uInt8 pnBaseKeyLE[ 2 ];
 163     pnBaseKeyLE[ 0 ] = static_cast< sal_uInt8 >( mnBaseKey );
 164     pnBaseKeyLE[ 1 ] = static_cast< sal_uInt8 >( mnBaseKey >> 8 );
 165     sal_uInt8* pnKeyChar = mpnKey;
 166     for( nIndex = 0; nIndex < static_cast< sal_Int32 >( sizeof( mpnKey ) ); ++nIndex, ++pnKeyChar )
 167     {
 168         *pnKeyChar ^= pnBaseKeyLE[ nIndex & 1 ];
 169         lclRotateLeft( *pnKeyChar, nRotateSize );
 170     }
 171 }
 172
 173 bool BinaryCodec_XOR::verifyKey( sal_uInt16 nKey, sal_uInt16 nHash ) const
 174 {
 175     return (nKey == mnBaseKey) && (nHash == mnHash);
 176 }
 177
 178 void BinaryCodec_XOR::startBlock()
 179 {
 180     mnOffset = 0;
 181 }
 182
 183 bool BinaryCodec_XOR::decode( sal_uInt8* pnDestData, const sal_uInt8* pnSrcData, sal_Int32 nBytes )
 184 {
 185     const sal_uInt8* pnCurrKey = mpnKey + mnOffset;
 186     const sal_uInt8* pnKeyLast = mpnKey + 0x0F;
 187
 188     // switch/case outside of the for loop (performance)
 189     const sal_uInt8* pnSrcDataEnd = pnSrcData + nBytes;
 190     switch( meCodecType )
 191     {
 192         case CODEC_WORD:
 193         {
 194             for( ; pnSrcData < pnSrcDataEnd; ++pnSrcData, ++pnDestData )
 195             {
 196                 sal_uInt8 nData = *pnSrcData ^ *pnCurrKey;
 197                 if( (*pnSrcData != 0) && (nData != 0) )
 198                     *pnDestData = nData;
 199                 if( pnCurrKey < pnKeyLast ) ++pnCurrKey; else pnCurrKey = mpnKey;
 200             }
 201         }
 202         break;
 203         case CODEC_EXCEL:
 204         {
 205             for( ; pnSrcData < pnSrcDataEnd; ++pnSrcData, ++pnDestData )
 206             {
 207                 *pnDestData = *pnSrcData;
 208                 lclRotateLeft( *pnDestData, 3 );
 209                 *pnDestData ^= *pnCurrKey;
 210                 if( pnCurrKey < pnKeyLast ) ++pnCurrKey; else pnCurrKey = mpnKey;
 211             }
 212         }
 213         break;
 214         // compiler will warn, if new codec type is introduced and not handled here
 215     }
 216
 217     // update offset and leave
 218     return skip( nBytes );
 219 }
 220
 221 bool BinaryCodec_XOR::skip( sal_Int32 nBytes )
 222 {
 223     mnOffset = static_cast< sal_Int32 >( (mnOffset + nBytes) & 0x0F );
 224     return true;
 225 }
 226
 227 sal_uInt16 BinaryCodec_XOR::getHash( const sal_uInt8* pnPassData, sal_Int32 nSize )
 228 {
 229     return lclGetHash( pnPassData, nSize );
 230 }
 231
 232 // ============================================================================
 233
 234 BinaryCodec_RCF::BinaryCodec_RCF()
 235 {
 236     mhCipher = rtl_cipher_create( rtl_Cipher_AlgorithmARCFOUR, rtl_Cipher_ModeStream );
 237     OSL_ENSURE( mhCipher != 0, "BinaryCodec_RCF::BinaryCodec_RCF - cannot create cipher" );
 238
 239     mhDigest = rtl_digest_create( rtl_Digest_AlgorithmMD5 );
 240     OSL_ENSURE( mhDigest != 0, "BinaryCodec_RCF::BinaryCodec_RCF - cannot create digest" );
 241
 242     (void)memset( mpnDigestValue, 0, sizeof( mpnDigestValue ) );
 243 }
 244
 245 BinaryCodec_RCF::~BinaryCodec_RCF()
 246 {
 247     (void)memset( mpnDigestValue, 0, sizeof( mpnDigestValue ) );
 248     rtl_digest_destroy( mhDigest );
 249     rtl_cipher_destroy( mhCipher );
 250 }
 251
 252 void BinaryCodec_RCF::initKey( const sal_uInt16 pnPassData[ 16 ], const sal_uInt8 pnUnique[ 16 ] )
 253 {
 254     // create little-endian key data array from password data
 255     sal_uInt8 pnKeyData[ 64 ];
 256     (void)memset( pnKeyData, 0, sizeof( pnKeyData ) );
 257
 258     const sal_uInt16* pnCurrPass = pnPassData;
 259     const sal_uInt16* pnPassEnd = pnPassData + 16;
 260     sal_uInt8* pnCurrKey = pnKeyData;
 261     size_t nPassSize = 0;
 262     for( ; (pnCurrPass < pnPassEnd) && (*pnCurrPass != 0); ++pnCurrPass, ++nPassSize )
 263     {
 264         *pnCurrKey++ = static_cast< sal_uInt8 >( *pnCurrPass );
 265         *pnCurrKey++ = static_cast< sal_uInt8 >( *pnCurrPass >> 8 );
 266     }
 267     pnKeyData[ 2 * nPassSize ] = 0x80;
 268     pnKeyData[ 56 ] = static_cast< sal_uInt8 >( nPassSize << 4 );
 269
 270     // fill raw digest of key data into key data
 271     (void)rtl_digest_updateMD5( mhDigest, pnKeyData, sizeof( pnKeyData ) );
 272     (void)rtl_digest_rawMD5( mhDigest, pnKeyData, RTL_DIGEST_LENGTH_MD5 );
 273
 274     // update digest with key data and passed unique data
 275     for( size_t nIndex = 0; nIndex < 16; ++nIndex )
 276     {
 277         rtl_digest_updateMD5( mhDigest, pnKeyData, 5 );
 278         rtl_digest_updateMD5( mhDigest, pnUnique, 16 );
 279     }
 280
 281     // update digest with padding
 282     pnKeyData[ 16 ] = 0x80;
 283     (void)memset( pnKeyData + 17, 0, sizeof( pnKeyData ) - 17 );
 284     pnKeyData[ 56 ] = 0x80;
 285     pnKeyData[ 57 ] = 0x0A;
 286     rtl_digest_updateMD5( mhDigest, pnKeyData + 16, sizeof( pnKeyData ) - 16 );
 287
 288     // fill raw digest of above updates into digest value
 289     rtl_digest_rawMD5( mhDigest, mpnDigestValue, sizeof( mpnDigestValue ) );
 290
 291     // erase key data array and leave
 292     (void)memset( pnKeyData, 0, sizeof( pnKeyData ) );
 293 }
 294
 295 bool BinaryCodec_RCF::verifyKey( const sal_uInt8 pnSaltData[ 16 ], const sal_uInt8 pnSaltDigest[ 16 ] )
 296 {
 297     if( !startBlock( 0 ) )
 298         return false;
 299
 300     sal_uInt8 pnDigest[ RTL_DIGEST_LENGTH_MD5 ];
 301     sal_uInt8 pnBuffer[ 64 ];
 302
 303     // decode salt data into buffer
 304     rtl_cipher_decode( mhCipher, pnSaltData, 16, pnBuffer, sizeof( pnBuffer ) );
 305
 306     pnBuffer[ 16 ] = 0x80;
 307     (void)memset( pnBuffer + 17, 0, sizeof( pnBuffer ) - 17 );
 308     pnBuffer[ 56 ] = 0x80;
 309
 310     // fill raw digest of buffer into digest
 311     rtl_digest_updateMD5( mhDigest, pnBuffer, sizeof( pnBuffer ) );
 312     rtl_digest_rawMD5( mhDigest, pnDigest, sizeof( pnDigest ) );
 313
 314     // decode original salt digest into buffer
 315     rtl_cipher_decode( mhCipher, pnSaltDigest, 16, pnBuffer, sizeof( pnBuffer ) );
 316
 317     // compare buffer with computed digest
 318     bool bResult = memcmp( pnBuffer, pnDigest, sizeof( pnDigest ) ) == 0;
 319
 320     // erase buffer and digest arrays and leave
 321     (void)memset( pnBuffer, 0, sizeof( pnBuffer ) );
 322     (void)memset( pnDigest, 0, sizeof( pnDigest ) );
 323     return bResult;
 324 }
 325
 326 bool BinaryCodec_RCF::startBlock( sal_Int32 nCounter )
 327 {
 328     // initialize key data array
 329     sal_uInt8 pnKeyData[ 64 ];
 330     (void)memset( pnKeyData, 0, sizeof( pnKeyData ) );
 331
 332     // fill 40 bit of digest value into [0..4]
 333     (void)memcpy( pnKeyData, mpnDigestValue, 5 );
 334
 335     // fill little-endian counter into [5..8], static_cast masks out unneeded bits
 336     pnKeyData[ 5 ] = static_cast< sal_uInt8 >( nCounter );
 337     pnKeyData[ 6 ] = static_cast< sal_uInt8 >( nCounter >> 8 );
 338     pnKeyData[ 7 ] = static_cast< sal_uInt8 >( nCounter >> 16 );
 339     pnKeyData[ 8 ] = static_cast< sal_uInt8 >( nCounter >> 24 );
 340
 341     pnKeyData[ 9 ] = 0x80;
 342     pnKeyData[ 56 ] = 0x48;
 343
 344     // fill raw digest of key data into key data
 345     (void)rtl_digest_updateMD5( mhDigest, pnKeyData, sizeof( pnKeyData ) );
 346     (void)rtl_digest_rawMD5( mhDigest, pnKeyData, RTL_DIGEST_LENGTH_MD5 );
 347
 348     // initialize cipher with key data (for decoding)
 349     rtlCipherError eResult =
 350         rtl_cipher_init( mhCipher, rtl_Cipher_DirectionDecode, pnKeyData, RTL_DIGEST_LENGTH_MD5, 0, 0 );
 351
 352     // rrase key data array and leave
 353     (void)memset( pnKeyData, 0, sizeof( pnKeyData ) );
 354     return eResult == rtl_Cipher_E_None;
 355 }
 356
 357 bool BinaryCodec_RCF::decode( sal_uInt8* pnDestData, const sal_uInt8* pnSrcData, sal_Int32 nBytes )
 358 {
 359     rtlCipherError eResult = rtl_cipher_decode( mhCipher,
 360         pnSrcData, static_cast< sal_Size >( nBytes ),
 361         pnDestData, static_cast< sal_Size >( nBytes ) );
 362     return eResult == rtl_Cipher_E_None;
 363 }
 364
 365 bool BinaryCodec_RCF::skip( sal_Int32 nBytes )
 366 {
 367     // decode dummy data in memory to update internal state of RC4 cipher
 368     sal_uInt8 pnDummy[ 1024 ];
 369     sal_Int32 nBytesLeft = nBytes;
 370     bool bResult = true;
 371     while( bResult && (nBytesLeft > 0) )
 372     {
 373         sal_Int32 nBlockLen = ::std::min( nBytesLeft, static_cast< sal_Int32 >( sizeof( pnDummy ) ) );
 374         bResult = decode( pnDummy, pnDummy, nBlockLen );
 375         nBytesLeft -= nBlockLen;
 376     }
 377     return bResult;
 378 }
 379
 380 // ============================================================================
 381
 382 } // namespace core
 383 } // namespace oox
 384