applied_patches/0157-cws-scsheetprotection02-svx.diff

   1 diff --git svx/inc/mscodec.hxx svx/inc/mscodec.hxx
   2 index 66dcf5d..be67b46 100644
   3 --- svx/inc/mscodec.hxx
   4 +++ svx/inc/mscodec.hxx
   5 @@ -235,6 +235,14 @@ public:
   6       */
   7      bool                InitCipher( sal_uInt32 nCounter );
   8
   9 +    /** Creates an MD5 digest of salt digest. */
  10 +    bool                CreateSaltDigest(
  11 +                            const sal_uInt8 nSaltData[16], sal_uInt8 nSaltDigest[16] );
  12 +
  13 +    bool                Encode(
  14 +                            const void* pData, sal_Size nDatLen,
  15 +                            sal_uInt8* pBuffer, sal_Size nBufLen );
  16 +
  17      /** Decodes a block of memory.
  18
  19          @see rtl_cipher_decode()
  20 @@ -276,6 +284,9 @@ public:
  21      bool                Skip( sal_Size nDatLen );
  22
  23  private:
  24 +    void                GetDigestFromSalt( const sal_uInt8 pSaltData[16], sal_uInt8 pDigest[16] );
  25 +
  26 +private:
  27                          SVX_DLLPRIVATE MSCodec_Std97( const MSCodec_Std97& );
  28      SVX_DLLPRIVATE MSCodec_Std97&      operator=( const MSCodec_Std97& );
  29
  30 diff --git svx/source/msfilter/mscodec.cxx svx/source/msfilter/mscodec.cxx
  31 index 30baed5..d086754 100644
  32 --- svx/source/msfilter/mscodec.cxx
  33 +++ svx/source/msfilter/mscodec.cxx
  34 @@ -37,6 +37,13 @@
  35  #include <string.h>
  36  #include <tools/solar.h>
  37
  38 +#define DEBUG_MSO_ENCRYPTION_STD97 0
  39 +
  40 +#if DEBUG_MSO_ENCRYPTION_STD97
  41 +#include <stdio.h>
  42 +#endif
  43 +
  44 +
  45  namespace svx {
  46
  47  // ============================================================================
  48 @@ -241,15 +248,50 @@ MSCodec_Std97::~MSCodec_Std97 ()
  49      rtl_cipher_destroy (m_hCipher);
  50  }
  51
  52 +#if DEBUG_MSO_ENCRYPTION_STD97
  53 +static void lcl_PrintKeyData(const sal_uInt8* pKeyData, const char* msg)
  54 +{
  55 +    printf("pKeyData: (%s)\n", msg);
  56 +    for (int j = 0; j < 4; ++j)
  57 +    {
  58 +        for (int i = 0; i < 16; ++i)
  59 +            printf("%2.2x ", pKeyData[j*16+i]);
  60 +        printf("\n");
  61 +    }
  62 +}
  63 +#else
  64 +static void lcl_PrintKeyData(const sal_uInt8* /*pKeyData*/, const char* /*msg*/)
  65 +{
  66 +}
  67 +#endif
  68 +
  69 +#if DEBUG_MSO_ENCRYPTION_STD97
  70 +static void lcl_PrintDigest(const sal_uInt8* pDigest, const char* msg)
  71 +{
  72 +    printf("digest: (%s)\n", msg);
  73 +    for (int i = 0; i < 16; ++i)
  74 +        printf("%2.2x ", pDigest[i]);
  75 +    printf("\n");
  76 +}
  77 +#else
  78 +static void lcl_PrintDigest(const sal_uInt8* /*pDigest*/, const char* /*msg*/)
  79 +{
  80 +}
  81 +#endif
  82 +
  83  void MSCodec_Std97::InitKey (
  84      const sal_uInt16 pPassData[16],
  85      const sal_uInt8  pUnique[16])
  86  {
  87 +#if DEBUG_MSO_ENCRYPTION_STD97
  88 +    fprintf(stdout, "MSCodec_Std97::InitKey: --begin\n");fflush(stdout);
  89 +#endif
  90      sal_uInt8 pKeyData[64];
  91      int       i, n;
  92
  93      // Fill PassData into KeyData.
  94      (void)memset (pKeyData, 0, sizeof(pKeyData));
  95 +    lcl_PrintKeyData(pKeyData, "initial");
  96      for (i = 0, n = 16; (i < n) && pPassData[i]; i++)
  97      {
  98          pKeyData[2*i    ] = sal::static_int_cast< sal_uInt8 >(
  99 @@ -260,12 +302,16 @@ void MSCodec_Std97::InitKey (
 100      pKeyData[2*i] = 0x80;
 101      pKeyData[ 56] = sal::static_int_cast< sal_uInt8 >(i << 4);
 102
 103 +    lcl_PrintKeyData(pKeyData, "password data");
 104 +
 105      // Fill raw digest of KeyData into KeyData.
 106      (void)rtl_digest_updateMD5 (
 107          m_hDigest, pKeyData, sizeof(pKeyData));
 108      (void)rtl_digest_rawMD5 (
 109          m_hDigest, pKeyData, RTL_DIGEST_LENGTH_MD5);
 110
 111 +    lcl_PrintKeyData(pKeyData, "raw digest of key data");
 112 +
 113      // Update digest with KeyData and Unique.
 114      for (i = 0; i < 16; i++)
 115      {
 116 @@ -279,6 +325,8 @@ void MSCodec_Std97::InitKey (
 117      pKeyData[56] = 0x80;
 118      pKeyData[57] = 0x0a;
 119
 120 +    lcl_PrintKeyData(pKeyData, "update digest with padding");
 121 +
 122      rtl_digest_updateMD5 (
 123          m_hDigest, &(pKeyData[16]), sizeof(pKeyData) - 16);
 124
 125 @@ -286,6 +334,8 @@ void MSCodec_Std97::InitKey (
 126      rtl_digest_rawMD5 (
 127          m_hDigest, m_pDigestValue, sizeof(m_pDigestValue));
 128
 129 +    lcl_PrintDigest(m_pDigestValue, "digest value");
 130 +
 131      // Erase KeyData array and leave.
 132      (void)memset (pKeyData, 0, sizeof(pKeyData));
 133  }
 134 @@ -294,27 +344,21 @@ bool MSCodec_Std97::VerifyKey (
 135      const sal_uInt8 pSaltData[16],
 136      const sal_uInt8 pSaltDigest[16])
 137  {
 138 +    // both the salt data and salt digest (hash) come from the document being imported.
 139 +
 140 +#if DEBUG_MSO_ENCRYPTION_STD97
 141 +    fprintf(stdout, "MSCodec_Std97::VerifyKey: \n");
 142 +    lcl_PrintDigest(pSaltData, "salt data");
 143 +    lcl_PrintDigest(pSaltDigest, "salt hash");
 144 +#endif
 145      bool result = false;
 146
 147      if (InitCipher(0))
 148      {
 149          sal_uInt8 pDigest[RTL_DIGEST_LENGTH_MD5];
 150 -        sal_uInt8 pBuffer[64];
 151 -
 152 -        // Decode SaltData into Buffer.
 153 -        rtl_cipher_decode (
 154 -            m_hCipher, pSaltData, 16, pBuffer, sizeof(pBuffer));
 155 -
 156 -        pBuffer[16] = 0x80;
 157 -        (void)memset (pBuffer + 17, 0, sizeof(pBuffer) - 17);
 158 -        pBuffer[56] = 0x80;
 159 -
 160 -        // Fill raw digest of Buffer into Digest.
 161 -        rtl_digest_updateMD5 (
 162 -            m_hDigest, pBuffer, sizeof(pBuffer));
 163 -        rtl_digest_rawMD5 (
 164 -            m_hDigest, pDigest, sizeof(pDigest));
 165 +        GetDigestFromSalt(pSaltData, pDigest);
 166
 167 +        sal_uInt8 pBuffer[16];
 168          // Decode original SaltDigest into Buffer.
 169          rtl_cipher_decode (
 170              m_hCipher, pSaltDigest, 16, pBuffer, sizeof(pBuffer));
 171 @@ -333,7 +377,7 @@ bool MSCodec_Std97::VerifyKey (
 172  bool MSCodec_Std97::InitCipher (sal_uInt32 nCounter)
 173  {
 174      rtlCipherError result;
 175 -    sal_uInt8      pKeyData[64];
 176 +    sal_uInt8      pKeyData[64]; // 512-bit message block
 177
 178      // Initialize KeyData array.
 179      (void)memset (pKeyData, 0, sizeof(pKeyData));
 180 @@ -358,7 +402,7 @@ bool MSCodec_Std97::InitCipher (sal_uInt32 nCounter)
 181
 182      // Initialize Cipher with KeyData (for decoding).
 183      result = rtl_cipher_init (
 184 -        m_hCipher, rtl_Cipher_DirectionDecode,
 185 +        m_hCipher, rtl_Cipher_DirectionBoth,
 186          pKeyData, RTL_DIGEST_LENGTH_MD5, 0, 0);
 187
 188      // Erase KeyData array and leave.
 189 @@ -367,6 +411,38 @@ bool MSCodec_Std97::InitCipher (sal_uInt32 nCounter)
 190      return (result == rtl_Cipher_E_None);
 191  }
 192
 193 +bool MSCodec_Std97::CreateSaltDigest( const sal_uInt8 nSaltData[16], sal_uInt8 nSaltDigest[16] )
 194 +{
 195 +#if DEBUG_MSO_ENCRYPTION_STD97
 196 +    lcl_PrintDigest(pSaltData, "salt data");
 197 +#endif
 198 +    bool result = false;
 199 +
 200 +    if (InitCipher(0))
 201 +    {
 202 +        sal_uInt8 pDigest[RTL_DIGEST_LENGTH_MD5];
 203 +        GetDigestFromSalt(nSaltData, pDigest);
 204 +
 205 +        rtl_cipher_decode (
 206 +            m_hCipher, pDigest, 16, pDigest, sizeof(pDigest));
 207 +
 208 +        (void)memcpy(nSaltDigest, pDigest, 16);
 209 +    }
 210 +
 211 +    return (result);
 212 +}
 213 +
 214 +bool MSCodec_Std97::Encode(
 215 +    const void *pData,   sal_Size nDatLen,
 216 +    sal_uInt8  *pBuffer, sal_Size nBufLen)
 217 +{
 218 +    rtlCipherError result;
 219 +    result = rtl_cipher_encode (
 220 +        m_hCipher, pData, nDatLen, pBuffer, nBufLen);
 221 +
 222 +    return (result == rtl_Cipher_E_None);
 223 +}
 224 +
 225  bool MSCodec_Std97::Decode (
 226      const void *pData,   sal_Size nDatLen,
 227      sal_uInt8  *pBuffer, sal_Size nBufLen)
 228 @@ -395,6 +471,33 @@ bool MSCodec_Std97::Skip( sal_Size nDatLen )
 229      return bResult;
 230  }
 231
 232 +void MSCodec_Std97::GetDigestFromSalt( const sal_uInt8 pSaltData[16], sal_uInt8 pDigest[16] )
 233 +{
 234 +    sal_uInt8 pBuffer[64];
 235 +    sal_uInt8 pDigestLocal[16];
 236 +
 237 +    // Decode SaltData into Buffer.
 238 +    rtl_cipher_decode (
 239 +        m_hCipher, pSaltData, 16, pBuffer, sizeof(pBuffer));
 240 +
 241 +    // set the 129th bit to make the buffer 128-bit in length.
 242 +    pBuffer[16] = 0x80;
 243 +
 244 +    // erase the rest of the buffer with zeros.
 245 +    (void)memset (pBuffer + 17, 0, sizeof(pBuffer) - 17);
 246 +
 247 +    // set the 441st bit.
 248 +    pBuffer[56] = 0x80;
 249 +
 250 +    // Fill raw digest of Buffer into Digest.
 251 +    rtl_digest_updateMD5 (
 252 +        m_hDigest, pBuffer, sizeof(pBuffer));
 253 +    rtl_digest_rawMD5 (
 254 +        m_hDigest, pDigestLocal, sizeof(pDigestLocal));
 255 +
 256 +    memcpy(pDigest, pDigestLocal, 16);
 257 +}
 258 +
 259  // ============================================================================
 260
 261  } // namespace svx