svx/source/msfilter/mscodec.cxx

   1 /*************************************************************************
   2  *
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * Copyright 2008 by Sun Microsystems, Inc.
   6  *
   7  * OpenOffice.org - a multi-platform office productivity suite
   8  *
   9  * $RCSfile: mscodec.cxx,v $
  10  * $Revision: 1.7 $
  11  *
  12  * This file is part of OpenOffice.org.
  13  *
  14  * OpenOffice.org is free software: you can redistribute it and/or modify
  15  * it under the terms of the GNU Lesser General Public License version 3
  16  * only, as published by the Free Software Foundation.
  17  *
  18  * OpenOffice.org is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU Lesser General Public License version 3 for more details
  22  * (a copy is included in the LICENSE file that accompanied this code).
  23  *
  24  * You should have received a copy of the GNU Lesser General Public License
  25  * version 3 along with OpenOffice.org.  If not, see
  26  * <http://www.openoffice.org/license.html>
  27  * for a copy of the LGPLv3 License.
  28  *
  29  ************************************************************************/
  30
  31 // MARKER(update_precomp.py): autogen include statement, do not remove
  32 #include "precompiled_svx.hxx"
  33 #include "mscodec.hxx"
  34
  35 #include <osl/diagnose.h>
  36 #include <algorithm>
  37 #include <string.h>
  38 #include <tools/solar.h>
  39
  40 #define DEBUG_MSO_ENCRYPTION_STD97 0
  41
  42 #if DEBUG_MSO_ENCRYPTION_STD97
  43 #include <stdio.h>
  44 #endif
  45
  46
  47 namespace svx {
  48
  49 // ============================================================================
  50
  51 namespace {
  52
  53 /** Rotates rnValue left by nBits bits. */
  54 template< typename Type >
  55 inline void lclRotateLeft( Type& rnValue, int nBits )
  56 {
  57     OSL_ASSERT(
  58         nBits >= 0 &&
  59         sal::static_int_cast< unsigned int >(nBits) < sizeof( Type ) * 8 );
  60     rnValue = static_cast< Type >( (rnValue << nBits) | (rnValue >> (sizeof( Type ) * 8 - nBits)) );
  61 }
  62
  63 /** Rotates the lower nWidth bits of rnValue left by nBits bits. */
  64 template< typename Type >
  65 inline void lclRotateLeft( Type& rnValue, sal_uInt8 nBits, sal_uInt8 nWidth )
  66 {
  67     OSL_ASSERT( (nBits < nWidth) && (nWidth < sizeof( Type ) * 8) );
  68     Type nMask = static_cast< Type >( (1UL << nWidth) - 1 );
  69     rnValue = static_cast< Type >(
  70         ((rnValue << nBits) | ((rnValue & nMask) >> (nWidth - nBits))) & nMask );
  71 }
  72
  73 sal_Size lclGetLen( const sal_uInt8* pnPassData, sal_Size nBufferSize )
  74 {
  75     sal_Size nLen = 0;
  76     while( (nLen < nBufferSize) && pnPassData[ nLen ] ) ++nLen;
  77     return nLen;
  78 }
  79
  80 sal_uInt16 lclGetKey( const sal_uInt8* pnPassData, sal_Size nBufferSize )
  81 {
  82     sal_Size nLen = lclGetLen( pnPassData, nBufferSize );
  83     if( !nLen ) return 0;
  84
  85     sal_uInt16 nKey = 0;
  86     sal_uInt16 nKeyBase = 0x8000;
  87     sal_uInt16 nKeyEnd = 0xFFFF;
  88     const sal_uInt8* pnChar = pnPassData + nLen - 1;
  89     for( sal_Size nIndex = 0; nIndex < nLen; ++nIndex, --pnChar )
  90     {
  91         sal_uInt8 cChar = *pnChar & 0x7F;
  92         for( sal_uInt8 nBit = 0; nBit < 8; ++nBit )
  93         {
  94             lclRotateLeft( nKeyBase, 1 );
  95             if( nKeyBase & 1 ) nKeyBase ^= 0x1020;
  96             if( cChar & 1 ) nKey ^= nKeyBase;
  97             cChar >>= 1;
  98             lclRotateLeft( nKeyEnd, 1 );
  99             if( nKeyEnd & 1 ) nKeyEnd ^= 0x1020;
 100         }
 101     }
 102     return nKey ^ nKeyEnd;
 103 }
 104
 105 sal_uInt16 lclGetHash( const sal_uInt8* pnPassData, sal_Size nBufferSize )
 106 {
 107     sal_Size nLen = lclGetLen( pnPassData, nBufferSize );
 108
 109     sal_uInt16 nHash = static_cast< sal_uInt16 >( nLen );
 110     if( nLen )
 111         nHash ^= 0xCE4B;
 112
 113     const sal_uInt8* pnChar = pnPassData;
 114     for( sal_Size nIndex = 0; nIndex < nLen; ++nIndex, ++pnChar )
 115     {
 116         sal_uInt16 cChar = *pnChar;
 117         sal_uInt8 nRot = static_cast< sal_uInt8 >( (nIndex + 1) % 15 );
 118         lclRotateLeft( cChar, nRot, 15 );
 119         nHash ^= cChar;
 120     }
 121     return nHash;
 122 }
 123
 124
 125 } // namespace
 126
 127 // ============================================================================
 128
 129 MSCodec_Xor95::MSCodec_Xor95(int nRotateDistance) :
 130     mnOffset( 0 ),
 131     mnKey( 0 ),
 132     mnHash( 0 ),
 133     mnRotateDistance( nRotateDistance )
 134 {
 135     (void)memset( mpnKey, 0, sizeof( mpnKey ) );
 136 }
 137
 138 MSCodec_Xor95::~MSCodec_Xor95()
 139 {
 140     (void)memset( mpnKey, 0, sizeof( mpnKey ) );
 141     mnKey = mnHash = 0;
 142 }
 143
 144 void MSCodec_Xor95::InitKey( const sal_uInt8 pnPassData[ 16 ] )
 145 {
 146     mnKey = lclGetKey( pnPassData, 16 );
 147     mnHash = lclGetHash( pnPassData, 16 );
 148
 149     (void)memcpy( mpnKey, pnPassData, 16 );
 150
 151     static const sal_uInt8 spnFillChars[] =
 152     {
 153         0xBB, 0xFF, 0xFF, 0xBA,
 154         0xFF, 0xFF, 0xB9, 0x80,
 155         0x00, 0xBE, 0x0F, 0x00,
 156         0xBF, 0x0F, 0x00
 157     };
 158
 159     sal_Size nIndex;
 160     sal_Size nLen = lclGetLen( pnPassData, 16 );
 161     const sal_uInt8* pnFillChar = spnFillChars;
 162     for( nIndex = nLen; nIndex < sizeof( mpnKey ); ++nIndex, ++pnFillChar )
 163         mpnKey[ nIndex ] = *pnFillChar;
 164
 165     SVBT16 pnOrigKey;
 166     ShortToSVBT16( mnKey, pnOrigKey );
 167     sal_uInt8* pnKeyChar = mpnKey;
 168     for( nIndex = 0; nIndex < sizeof( mpnKey ); ++nIndex, ++pnKeyChar )
 169     {
 170         *pnKeyChar ^= pnOrigKey[ nIndex & 0x01 ];
 171         lclRotateLeft( *pnKeyChar, mnRotateDistance );
 172     }
 173 }
 174
 175 bool MSCodec_Xor95::VerifyKey( sal_uInt16 nKey, sal_uInt16 nHash ) const
 176 {
 177     return (nKey == mnKey) && (nHash == mnHash);
 178 }
 179
 180 void MSCodec_Xor95::InitCipher()
 181 {
 182     mnOffset = 0;
 183 }
 184
 185 void MSCodec_XorXLS95::Decode( sal_uInt8* pnData, sal_Size nBytes )
 186 {
 187     const sal_uInt8* pnCurrKey = mpnKey + mnOffset;
 188     const sal_uInt8* pnKeyLast = mpnKey + 0x0F;
 189
 190     for( const sal_uInt8* pnDataEnd = pnData + nBytes; pnData < pnDataEnd; ++pnData )
 191     {
 192         lclRotateLeft( *pnData, 3 );
 193         *pnData ^= *pnCurrKey;
 194         if( pnCurrKey < pnKeyLast ) ++pnCurrKey; else pnCurrKey = mpnKey;
 195     }
 196
 197     // update mnOffset
 198     Skip( nBytes );
 199 }
 200
 201 void MSCodec_XorWord95::Decode( sal_uInt8* pnData, sal_Size nBytes )
 202 {
 203     const sal_uInt8* pnCurrKey = mpnKey + mnOffset;
 204     const sal_uInt8* pnKeyLast = mpnKey + 0x0F;
 205
 206     for( const sal_uInt8* pnDataEnd = pnData + nBytes; pnData < pnDataEnd; ++pnData )
 207     {
 208         const sal_uInt8 cChar = *pnData ^ *pnCurrKey;
 209     if (*pnData && cChar)
 210         *pnData = cChar;
 211         if( pnCurrKey < pnKeyLast ) ++pnCurrKey; else pnCurrKey = mpnKey;
 212     }
 213
 214     // update mnOffset
 215     Skip( nBytes );
 216 }
 217
 218
 219 void MSCodec_Xor95::Skip( sal_Size nBytes )
 220 {
 221     mnOffset = (mnOffset + nBytes) & 0x0F;
 222 }
 223
 224 sal_uInt16 MSCodec_Xor95::GetHash( const sal_uInt8* pnPassData, sal_Size nSize )
 225 {
 226     return lclGetHash( pnPassData, nSize );
 227 }
 228
 229 // ============================================================================
 230
 231 MSCodec_Std97::MSCodec_Std97 ()
 232 {
 233     m_hCipher = rtl_cipher_create (
 234         rtl_Cipher_AlgorithmARCFOUR, rtl_Cipher_ModeStream);
 235     OSL_ASSERT(m_hCipher != 0);
 236
 237     m_hDigest = rtl_digest_create (
 238         rtl_Digest_AlgorithmMD5);
 239     OSL_ASSERT(m_hDigest != 0);
 240
 241     (void)memset (m_pDigestValue, 0, sizeof(m_pDigestValue));
 242 }
 243
 244 MSCodec_Std97::~MSCodec_Std97 ()
 245 {
 246     (void)memset (m_pDigestValue, 0, sizeof(m_pDigestValue));
 247     rtl_digest_destroy (m_hDigest);
 248     rtl_cipher_destroy (m_hCipher);
 249 }
 250
 251 #if DEBUG_MSO_ENCRYPTION_STD97
 252 static void lcl_PrintKeyData(const sal_uInt8* pKeyData, const char* msg)
 253 {
 254     printf("pKeyData: (%s)\n", msg);
 255     for (int j = 0; j < 4; ++j)
 256     {
 257         for (int i = 0; i < 16; ++i)
 258             printf("%2.2x ", pKeyData[j*16+i]);
 259         printf("\n");
 260     }
 261 }
 262 #else
 263 static void lcl_PrintKeyData(const sal_uInt8* /*pKeyData*/, const char* /*msg*/)
 264 {
 265 }
 266 #endif
 267
 268 #if DEBUG_MSO_ENCRYPTION_STD97
 269 static void lcl_PrintDigest(const sal_uInt8* pDigest, const char* msg)
 270 {
 271     printf("digest: (%s)\n", msg);
 272     for (int i = 0; i < 16; ++i)
 273         printf("%2.2x ", pDigest[i]);
 274     printf("\n");
 275 }
 276 #else
 277 static void lcl_PrintDigest(const sal_uInt8* /*pDigest*/, const char* /*msg*/)
 278 {
 279 }
 280 #endif
 281
 282 void MSCodec_Std97::InitKey (
 283     const sal_uInt16 pPassData[16],
 284     const sal_uInt8  pUnique[16])
 285 {
 286 #if DEBUG_MSO_ENCRYPTION_STD97
 287     fprintf(stdout, "MSCodec_Std97::InitKey: --begin\n");fflush(stdout);
 288 #endif
 289     sal_uInt8 pKeyData[64];
 290     int       i, n;
 291
 292     // Fill PassData into KeyData.
 293     (void)memset (pKeyData, 0, sizeof(pKeyData));
 294     lcl_PrintKeyData(pKeyData, "initial");
 295     for (i = 0, n = 16; (i < n) && pPassData[i]; i++)
 296     {
 297         pKeyData[2*i    ] = sal::static_int_cast< sal_uInt8 >(
 298             (pPassData[i] >> 0) & 0xff);
 299         pKeyData[2*i + 1] = sal::static_int_cast< sal_uInt8 >(
 300             (pPassData[i] >> 8) & 0xff);
 301     }
 302     pKeyData[2*i] = 0x80;
 303     pKeyData[ 56] = sal::static_int_cast< sal_uInt8 >(i << 4);
 304
 305     lcl_PrintKeyData(pKeyData, "password data");
 306
 307     // Fill raw digest of KeyData into KeyData.
 308     (void)rtl_digest_updateMD5 (
 309         m_hDigest, pKeyData, sizeof(pKeyData));
 310     (void)rtl_digest_rawMD5 (
 311         m_hDigest, pKeyData, RTL_DIGEST_LENGTH_MD5);
 312
 313     lcl_PrintKeyData(pKeyData, "raw digest of key data");
 314
 315     // Update digest with KeyData and Unique.
 316     for (i = 0; i < 16; i++)
 317     {
 318         rtl_digest_updateMD5 (m_hDigest, pKeyData, 5);
 319         rtl_digest_updateMD5 (m_hDigest, pUnique, 16);
 320     }
 321
 322     // Update digest with padding.
 323     pKeyData[16] = 0x80;
 324     (void)memset (pKeyData + 17, 0, sizeof(pKeyData) - 17);
 325     pKeyData[56] = 0x80;
 326     pKeyData[57] = 0x0a;
 327
 328     lcl_PrintKeyData(pKeyData, "update digest with padding");
 329
 330     rtl_digest_updateMD5 (
 331         m_hDigest, &(pKeyData[16]), sizeof(pKeyData) - 16);
 332
 333     // Fill raw digest of above updates into DigestValue.
 334     rtl_digest_rawMD5 (
 335         m_hDigest, m_pDigestValue, sizeof(m_pDigestValue));
 336
 337     lcl_PrintDigest(m_pDigestValue, "digest value");
 338
 339     // Erase KeyData array and leave.
 340     (void)memset (pKeyData, 0, sizeof(pKeyData));
 341 }
 342
 343 bool MSCodec_Std97::VerifyKey (
 344     const sal_uInt8 pSaltData[16],
 345     const sal_uInt8 pSaltDigest[16])
 346 {
 347     // both the salt data and salt digest (hash) come from the document being imported.
 348
 349 #if DEBUG_MSO_ENCRYPTION_STD97
 350     fprintf(stdout, "MSCodec_Std97::VerifyKey: \n");
 351     lcl_PrintDigest(pSaltData, "salt data");
 352     lcl_PrintDigest(pSaltDigest, "salt hash");
 353 #endif
 354     bool result = false;
 355
 356     if (InitCipher(0))
 357     {
 358         sal_uInt8 pDigest[RTL_DIGEST_LENGTH_MD5];
 359         GetDigestFromSalt(pSaltData, pDigest);
 360
 361         sal_uInt8 pBuffer[16];
 362         // Decode original SaltDigest into Buffer.
 363         rtl_cipher_decode (
 364             m_hCipher, pSaltDigest, 16, pBuffer, sizeof(pBuffer));
 365
 366         // Compare Buffer with computed Digest.
 367         result = (memcmp (pBuffer, pDigest, sizeof(pDigest)) == 0);
 368
 369         // Erase Buffer and Digest arrays.
 370         (void)memset (pBuffer, 0, sizeof(pBuffer));
 371         (void)memset (pDigest, 0, sizeof(pDigest));
 372     }
 373
 374     return (result);
 375 }
 376
 377 bool MSCodec_Std97::InitCipher (sal_uInt32 nCounter)
 378 {
 379     rtlCipherError result;
 380     sal_uInt8      pKeyData[64]; // 512-bit message block
 381
 382     // Initialize KeyData array.
 383     (void)memset (pKeyData, 0, sizeof(pKeyData));
 384
 385     // Fill 40 bit of DigestValue into [0..4].
 386     (void)memcpy (pKeyData, m_pDigestValue, 5);
 387
 388     // Fill counter into [5..8].
 389     pKeyData[ 5] = sal_uInt8((nCounter >>  0) & 0xff);
 390     pKeyData[ 6] = sal_uInt8((nCounter >>  8) & 0xff);
 391     pKeyData[ 7] = sal_uInt8((nCounter >> 16) & 0xff);
 392     pKeyData[ 8] = sal_uInt8((nCounter >> 24) & 0xff);
 393
 394     pKeyData[ 9] = 0x80;
 395     pKeyData[56] = 0x48;
 396
 397     // Fill raw digest of KeyData into KeyData.
 398     (void)rtl_digest_updateMD5 (
 399         m_hDigest, pKeyData, sizeof(pKeyData));
 400     (void)rtl_digest_rawMD5 (
 401         m_hDigest, pKeyData, RTL_DIGEST_LENGTH_MD5);
 402
 403     // Initialize Cipher with KeyData (for decoding).
 404     result = rtl_cipher_init (
 405         m_hCipher, rtl_Cipher_DirectionBoth,
 406         pKeyData, RTL_DIGEST_LENGTH_MD5, 0, 0);
 407
 408     // Erase KeyData array and leave.
 409     (void)memset (pKeyData, 0, sizeof(pKeyData));
 410
 411     return (result == rtl_Cipher_E_None);
 412 }
 413
 414 bool MSCodec_Std97::CreateSaltDigest( const sal_uInt8 nSaltData[16], sal_uInt8 nSaltDigest[16] )
 415 {
 416 #if DEBUG_MSO_ENCRYPTION_STD97
 417     lcl_PrintDigest(pSaltData, "salt data");
 418 #endif
 419     bool result = false;
 420
 421     if (InitCipher(0))
 422     {
 423         sal_uInt8 pDigest[RTL_DIGEST_LENGTH_MD5];
 424         GetDigestFromSalt(nSaltData, pDigest);
 425
 426         rtl_cipher_decode (
 427             m_hCipher, pDigest, 16, pDigest, sizeof(pDigest));
 428
 429         (void)memcpy(nSaltDigest, pDigest, 16);
 430     }
 431
 432     return (result);
 433 }
 434
 435 bool MSCodec_Std97::Encode (
 436     const void *pData,   sal_Size nDatLen,
 437     sal_uInt8  *pBuffer, sal_Size nBufLen)
 438 {
 439     rtlCipherError result;
 440
 441     result = rtl_cipher_encode (
 442         m_hCipher, pData, nDatLen, pBuffer, nBufLen);
 443
 444     return (result == rtl_Cipher_E_None);
 445 }
 446
 447 bool MSCodec_Std97::Decode (
 448     const void *pData,   sal_Size nDatLen,
 449     sal_uInt8  *pBuffer, sal_Size nBufLen)
 450 {
 451     rtlCipherError result;
 452
 453     result = rtl_cipher_decode (
 454         m_hCipher, pData, nDatLen, pBuffer, nBufLen);
 455
 456     return (result == rtl_Cipher_E_None);
 457 }
 458
 459 bool MSCodec_Std97::Skip( sal_Size nDatLen )
 460 {
 461     sal_uInt8 pnDummy[ 1024 ];
 462     sal_Size nDatLeft = nDatLen;
 463     bool bResult = true;
 464
 465     while (bResult && nDatLeft)
 466     {
 467         sal_Size nBlockLen = ::std::min< sal_Size >( nDatLeft, sizeof(pnDummy) );
 468         bResult = Decode( pnDummy, nBlockLen, pnDummy, nBlockLen );
 469         nDatLeft -= nBlockLen;
 470     }
 471
 472     return bResult;
 473 }
 474
 475 void MSCodec_Std97::GetDigestFromSalt( const sal_uInt8 pSaltData[16], sal_uInt8 pDigest[16] )
 476 {
 477     sal_uInt8 pBuffer[64];
 478     sal_uInt8 pDigestLocal[16];
 479
 480     // Decode SaltData into Buffer.
 481     rtl_cipher_decode (
 482         m_hCipher, pSaltData, 16, pBuffer, sizeof(pBuffer));
 483
 484     // set the 129th bit to make the buffer 128-bit in length.
 485     pBuffer[16] = 0x80;
 486
 487     // erase the rest of the buffer with zeros.
 488     (void)memset (pBuffer + 17, 0, sizeof(pBuffer) - 17);
 489
 490     // set the 441st bit.
 491     pBuffer[56] = 0x80;
 492
 493     // Fill raw digest of Buffer into Digest.
 494     rtl_digest_updateMD5 (
 495         m_hDigest, pBuffer, sizeof(pBuffer));
 496     rtl_digest_rawMD5 (
 497         m_hDigest, pDigestLocal, sizeof(pDigestLocal));
 498
 499     memcpy(pDigest, pDigestLocal, 16);
 500 }
 501
 502 void MSCodec_Std97::GetEncryptKey (
 503     const sal_uInt8 pSalt[16],
 504     sal_uInt8 pSaltData[16],
 505     sal_uInt8 pSaltDigest[16])
 506 {
 507     if (InitCipher(0))
 508     {
 509         sal_uInt8 pDigest[RTL_DIGEST_LENGTH_MD5];
 510         sal_uInt8 pBuffer[64];
 511
 512         rtl_cipher_encode (
 513             m_hCipher, pSalt, 16, pSaltData, sizeof(pBuffer));
 514
 515         (void)memcpy( pBuffer, pSalt, 16 );
 516
 517         pBuffer[16] = 0x80;
 518         (void)memset (pBuffer + 17, 0, sizeof(pBuffer) - 17);
 519         pBuffer[56] = 0x80;
 520
 521         rtl_digest_updateMD5 (
 522             m_hDigest, pBuffer, sizeof(pBuffer));
 523         rtl_digest_rawMD5 (
 524             m_hDigest, pDigest, sizeof(pDigest));
 525
 526         rtl_cipher_encode (
 527             m_hCipher, pDigest, 16, pSaltDigest, 16);
 528
 529         (void)memset (pBuffer, 0, sizeof(pBuffer));
 530         (void)memset (pDigest, 0, sizeof(pDigest));
 531     }
 532 }
 533
 534 // ============================================================================
 535
 536 } // namespace svx
 537