?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
upgrade to xpdf 3.00.
[swftools.git] / pdf2swf / xpdf / Decrypt.cc
blobdab075093c3b7ab5a16516a9026fb8d2952de89b
1 //========================================================================
2 //
3 // Decrypt.cc
4 //
5 // Copyright 1996-2003 Glyph & Cog, LLC
6 //
7 //========================================================================
8
9 #include <aconf.h>
10
11 #ifdef USE_GCC_PRAGMAS
12 #pragma implementation
13 #endif
14
15 #include "gmem.h"
16 #include "Decrypt.h"
17
18 static void rc4InitKey(Guchar *key, int keyLen, Guchar *state);
19 static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c);
20 static void md5(Guchar *msg, int msgLen, Guchar *digest);
21
22 static Guchar passwordPad[32] = {
23 0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41,
24 0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08,
25 0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68, 0x3e, 0x80,
26 0x2f, 0x0c, 0xa9, 0xfe, 0x64, 0x53, 0x69, 0x7a
27 };
28
29 //------------------------------------------------------------------------
30 // Decrypt
31 //------------------------------------------------------------------------
32
33 Decrypt::Decrypt(Guchar *fileKey, int keyLength, int objNum, int objGen) {
34 int i;
35
36 // construct object key
37 for (i = 0; i < keyLength; ++i) {
38 objKey[i] = fileKey[i];
39 }
40 objKey[keyLength] = objNum & 0xff;
41 objKey[keyLength + 1] = (objNum >> 8) & 0xff;
42 objKey[keyLength + 2] = (objNum >> 16) & 0xff;
43 objKey[keyLength + 3] = objGen & 0xff;
44 objKey[keyLength + 4] = (objGen >> 8) & 0xff;
45 md5(objKey, keyLength + 5, objKey);
46
47 // set up for decryption
48 x = y = 0;
49 if ((objKeyLength = keyLength + 5) > 16) {
50 objKeyLength = 16;
51 }
52 rc4InitKey(objKey, objKeyLength, state);
53 }
54
55 void Decrypt::reset() {
56 x = y = 0;
57 rc4InitKey(objKey, objKeyLength, state);
58 }
59
60 Guchar Decrypt::decryptByte(Guchar c) {
61 return rc4DecryptByte(state, &x, &y, c);
62 }
63
64 GBool Decrypt::makeFileKey(int encVersion, int encRevision, int keyLength,
65 GString *ownerKey, GString *userKey,
66 int permissions, GString *fileID,
67 GString *ownerPassword, GString *userPassword,
68 Guchar *fileKey, GBool *ownerPasswordOk) {
69 Guchar test[32], test2[32];
70 GString *userPassword2;
71 Guchar fState[256];
72 Guchar tmpKey[16];
73 Guchar fx, fy;
74 int len, i, j;
75
76 // try using the supplied owner password to generate the user password
77 *ownerPasswordOk = gFalse;
78 if (ownerPassword) {
79 len = ownerPassword->getLength();
80 if (len < 32) {
81 memcpy(test, ownerPassword->getCString(), len);
82 memcpy(test + len, passwordPad, 32 - len);
83 } else {
84 memcpy(test, ownerPassword->getCString(), 32);
85 }
86 md5(test, 32, test);
87 if (encRevision == 3) {
88 for (i = 0; i < 50; ++i) {
89 md5(test, 16, test);
90 }
91 }
92 if (encRevision == 2) {
93 rc4InitKey(test, keyLength, fState);
94 fx = fy = 0;
95 for (i = 0; i < 32; ++i) {
96 test2[i] = rc4DecryptByte(fState, &fx, &fy, ownerKey->getChar(i));
97 }
98 } else {
99 memcpy(test2, ownerKey->getCString(), 32);
100 for (i = 19; i >= 0; --i) {
101 for (j = 0; j < keyLength; ++j) {
102 tmpKey[j] = test[j] ^ i;
103 }
104 rc4InitKey(tmpKey, keyLength, fState);
105 fx = fy = 0;
106 for (j = 0; j < 32; ++j) {
107 test2[j] = rc4DecryptByte(fState, &fx, &fy, test2[j]);
108 }
109 }
110 }
111 userPassword2 = new GString((char *)test2, 32);
112 if (makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey,
113 permissions, fileID, userPassword2, fileKey)) {
114 *ownerPasswordOk = gTrue;
115 delete userPassword2;
116 return gTrue;
117 }
118 delete userPassword2;
119 }
120
121 // try using the supplied user password
122 return makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey,
123 permissions, fileID, userPassword, fileKey);
124 }
125
126 GBool Decrypt::makeFileKey2(int encVersion, int encRevision, int keyLength,
127 GString *ownerKey, GString *userKey,
128 int permissions, GString *fileID,
129 GString *userPassword, Guchar *fileKey) {
130 Guchar *buf;
131 Guchar test[32];
132 Guchar fState[256];
133 Guchar tmpKey[16];
134 Guchar fx, fy;
135 int len, i, j;
136 GBool ok;
137
138 // generate file key
139 buf = (Guchar *)gmalloc(68 + fileID->getLength());
140 if (userPassword) {
141 len = userPassword->getLength();
142 if (len < 32) {
143 memcpy(buf, userPassword->getCString(), len);
144 memcpy(buf + len, passwordPad, 32 - len);
145 } else {
146 memcpy(buf, userPassword->getCString(), 32);
147 }
148 } else {
149 memcpy(buf, passwordPad, 32);
150 }
151 memcpy(buf + 32, ownerKey->getCString(), 32);
152 buf[64] = permissions & 0xff;
153 buf[65] = (permissions >> 8) & 0xff;
154 buf[66] = (permissions >> 16) & 0xff;
155 buf[67] = (permissions >> 24) & 0xff;
156 memcpy(buf + 68, fileID->getCString(), fileID->getLength());
157 md5(buf, 68 + fileID->getLength(), fileKey);
158 if (encRevision == 3) {
159 for (i = 0; i < 50; ++i) {
160 md5(fileKey, keyLength, fileKey);
161 }
162 }
163
164 // test user password
165 if (encRevision == 2) {
166 rc4InitKey(fileKey, keyLength, fState);
167 fx = fy = 0;
168 for (i = 0; i < 32; ++i) {
169 test[i] = rc4DecryptByte(fState, &fx, &fy, userKey->getChar(i));
170 }
171 ok = memcmp(test, passwordPad, 32) == 0;
172 } else if (encRevision == 3) {
173 memcpy(test, userKey->getCString(), 32);
174 for (i = 19; i >= 0; --i) {
175 for (j = 0; j < keyLength; ++j) {
176 tmpKey[j] = fileKey[j] ^ i;
177 }
178 rc4InitKey(tmpKey, keyLength, fState);
179 fx = fy = 0;
180 for (j = 0; j < 32; ++j) {
181 test[j] = rc4DecryptByte(fState, &fx, &fy, test[j]);
182 }
183 }
184 memcpy(buf, passwordPad, 32);
185 memcpy(buf + 32, fileID->getCString(), fileID->getLength());
186 md5(buf, 32 + fileID->getLength(), buf);
187 ok = memcmp(test, buf, 16) == 0;
188 } else {
189 ok = gFalse;
190 }
191
192 gfree(buf);
193 return ok;
194 }
195
196 //------------------------------------------------------------------------
197 // RC4-compatible decryption
198 //------------------------------------------------------------------------
199
200 static void rc4InitKey(Guchar *key, int keyLen, Guchar *state) {
201 Guchar index1, index2;
202 Guchar t;
203 int i;
204
205 for (i = 0; i < 256; ++i)
206 state[i] = i;
207 index1 = index2 = 0;
208 for (i = 0; i < 256; ++i) {
209 index2 = (key[index1] + state[i] + index2) % 256;
210 t = state[i];
211 state[i] = state[index2];
212 state[index2] = t;
213 index1 = (index1 + 1) % keyLen;
214 }
215 }
216
217 static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c) {
218 Guchar x1, y1, tx, ty;
219
220 x1 = *x = (*x + 1) % 256;
221 y1 = *y = (state[*x] + *y) % 256;
222 tx = state[x1];
223 ty = state[y1];
224 state[x1] = ty;
225 state[y1] = tx;
226 return c ^ state[(tx + ty) % 256];
227 }
228
229 //------------------------------------------------------------------------
230 // MD5 message digest
231 //------------------------------------------------------------------------
232
233 // this works around a bug in older Sun compilers
234 static inline Gulong rotateLeft(Gulong x, int r) {
235 x &= 0xffffffff;
236 return ((x << r) | (x >> (32 - r))) & 0xffffffff;
237 }
238
239 static inline Gulong md5Round1(Gulong a, Gulong b, Gulong c, Gulong d,
240 Gulong Xk, Gulong s, Gulong Ti) {
241 return b + rotateLeft((a + ((b & c) | (~b & d)) + Xk + Ti), s);
242 }
243
244 static inline Gulong md5Round2(Gulong a, Gulong b, Gulong c, Gulong d,
245 Gulong Xk, Gulong s, Gulong Ti) {
246 return b + rotateLeft((a + ((b & d) | (c & ~d)) + Xk + Ti), s);
247 }
248
249 static inline Gulong md5Round3(Gulong a, Gulong b, Gulong c, Gulong d,
250 Gulong Xk, Gulong s, Gulong Ti) {
251 return b + rotateLeft((a + (b ^ c ^ d) + Xk + Ti), s);
252 }
253
254 static inline Gulong md5Round4(Gulong a, Gulong b, Gulong c, Gulong d,
255 Gulong Xk, Gulong s, Gulong Ti) {
256 return b + rotateLeft((a + (c ^ (b | ~d)) + Xk + Ti), s);
257 }
258
259 static void md5(Guchar *msg, int msgLen, Guchar *digest) {
260 Gulong x[16];
261 Gulong a, b, c, d, aa, bb, cc, dd;
262 int n64;
263 int i, j, k;
264
265 // compute number of 64-byte blocks
266 // (length + pad byte (0x80) + 8 bytes for length)
267 n64 = (msgLen + 1 + 8 + 63) / 64;
268
269 // initialize a, b, c, d
270 a = 0x67452301;
271 b = 0xefcdab89;
272 c = 0x98badcfe;
273 d = 0x10325476;
274
275 // loop through blocks
276 k = 0;
277 for (i = 0; i < n64; ++i) {
278
279 // grab a 64-byte block
280 for (j = 0; j < 16 && k < msgLen - 3; ++j, k += 4)
281 x[j] = (((((msg[k+3] << 8) + msg[k+2]) << 8) + msg[k+1]) << 8) + msg[k];
282 if (i == n64 - 1) {
283 if (k == msgLen - 3)
284 x[j] = 0x80000000 + (((msg[k+2] << 8) + msg[k+1]) << 8) + msg[k];
285 else if (k == msgLen - 2)
286 x[j] = 0x800000 + (msg[k+1] << 8) + msg[k];
287 else if (k == msgLen - 1)
288 x[j] = 0x8000 + msg[k];
289 else
290 x[j] = 0x80;
291 ++j;
292 while (j < 16)
293 x[j++] = 0;
294 x[14] = msgLen << 3;
295 }
296
297 // save a, b, c, d
298 aa = a;
299 bb = b;
300 cc = c;
301 dd = d;
302
303 // round 1
304 a = md5Round1(a, b, c, d, x[0], 7, 0xd76aa478);
305 d = md5Round1(d, a, b, c, x[1], 12, 0xe8c7b756);
306 c = md5Round1(c, d, a, b, x[2], 17, 0x242070db);
307 b = md5Round1(b, c, d, a, x[3], 22, 0xc1bdceee);
308 a = md5Round1(a, b, c, d, x[4], 7, 0xf57c0faf);
309 d = md5Round1(d, a, b, c, x[5], 12, 0x4787c62a);
310 c = md5Round1(c, d, a, b, x[6], 17, 0xa8304613);
311 b = md5Round1(b, c, d, a, x[7], 22, 0xfd469501);
312 a = md5Round1(a, b, c, d, x[8], 7, 0x698098d8);
313 d = md5Round1(d, a, b, c, x[9], 12, 0x8b44f7af);
314 c = md5Round1(c, d, a, b, x[10], 17, 0xffff5bb1);
315 b = md5Round1(b, c, d, a, x[11], 22, 0x895cd7be);
316 a = md5Round1(a, b, c, d, x[12], 7, 0x6b901122);
317 d = md5Round1(d, a, b, c, x[13], 12, 0xfd987193);
318 c = md5Round1(c, d, a, b, x[14], 17, 0xa679438e);
319 b = md5Round1(b, c, d, a, x[15], 22, 0x49b40821);
320
321 // round 2
322 a = md5Round2(a, b, c, d, x[1], 5, 0xf61e2562);
323 d = md5Round2(d, a, b, c, x[6], 9, 0xc040b340);
324 c = md5Round2(c, d, a, b, x[11], 14, 0x265e5a51);
325 b = md5Round2(b, c, d, a, x[0], 20, 0xe9b6c7aa);
326 a = md5Round2(a, b, c, d, x[5], 5, 0xd62f105d);
327 d = md5Round2(d, a, b, c, x[10], 9, 0x02441453);
328 c = md5Round2(c, d, a, b, x[15], 14, 0xd8a1e681);
329 b = md5Round2(b, c, d, a, x[4], 20, 0xe7d3fbc8);
330 a = md5Round2(a, b, c, d, x[9], 5, 0x21e1cde6);
331 d = md5Round2(d, a, b, c, x[14], 9, 0xc33707d6);
332 c = md5Round2(c, d, a, b, x[3], 14, 0xf4d50d87);
333 b = md5Round2(b, c, d, a, x[8], 20, 0x455a14ed);
334 a = md5Round2(a, b, c, d, x[13], 5, 0xa9e3e905);
335 d = md5Round2(d, a, b, c, x[2], 9, 0xfcefa3f8);
336 c = md5Round2(c, d, a, b, x[7], 14, 0x676f02d9);
337 b = md5Round2(b, c, d, a, x[12], 20, 0x8d2a4c8a);
338
339 // round 3
340 a = md5Round3(a, b, c, d, x[5], 4, 0xfffa3942);
341 d = md5Round3(d, a, b, c, x[8], 11, 0x8771f681);
342 c = md5Round3(c, d, a, b, x[11], 16, 0x6d9d6122);
343 b = md5Round3(b, c, d, a, x[14], 23, 0xfde5380c);
344 a = md5Round3(a, b, c, d, x[1], 4, 0xa4beea44);
345 d = md5Round3(d, a, b, c, x[4], 11, 0x4bdecfa9);
346 c = md5Round3(c, d, a, b, x[7], 16, 0xf6bb4b60);
347 b = md5Round3(b, c, d, a, x[10], 23, 0xbebfbc70);
348 a = md5Round3(a, b, c, d, x[13], 4, 0x289b7ec6);
349 d = md5Round3(d, a, b, c, x[0], 11, 0xeaa127fa);
350 c = md5Round3(c, d, a, b, x[3], 16, 0xd4ef3085);
351 b = md5Round3(b, c, d, a, x[6], 23, 0x04881d05);
352 a = md5Round3(a, b, c, d, x[9], 4, 0xd9d4d039);
353 d = md5Round3(d, a, b, c, x[12], 11, 0xe6db99e5);
354 c = md5Round3(c, d, a, b, x[15], 16, 0x1fa27cf8);
355 b = md5Round3(b, c, d, a, x[2], 23, 0xc4ac5665);
356
357 // round 4
358 a = md5Round4(a, b, c, d, x[0], 6, 0xf4292244);
359 d = md5Round4(d, a, b, c, x[7], 10, 0x432aff97);
360 c = md5Round4(c, d, a, b, x[14], 15, 0xab9423a7);
361 b = md5Round4(b, c, d, a, x[5], 21, 0xfc93a039);
362 a = md5Round4(a, b, c, d, x[12], 6, 0x655b59c3);
363 d = md5Round4(d, a, b, c, x[3], 10, 0x8f0ccc92);
364 c = md5Round4(c, d, a, b, x[10], 15, 0xffeff47d);
365 b = md5Round4(b, c, d, a, x[1], 21, 0x85845dd1);
366 a = md5Round4(a, b, c, d, x[8], 6, 0x6fa87e4f);
367 d = md5Round4(d, a, b, c, x[15], 10, 0xfe2ce6e0);
368 c = md5Round4(c, d, a, b, x[6], 15, 0xa3014314);
369 b = md5Round4(b, c, d, a, x[13], 21, 0x4e0811a1);
370 a = md5Round4(a, b, c, d, x[4], 6, 0xf7537e82);
371 d = md5Round4(d, a, b, c, x[11], 10, 0xbd3af235);
372 c = md5Round4(c, d, a, b, x[2], 15, 0x2ad7d2bb);
373 b = md5Round4(b, c, d, a, x[9], 21, 0xeb86d391);
374
375 // increment a, b, c, d
376 a += aa;
377 b += bb;
378 c += cc;
379 d += dd;
380 }
381
382 // break digest into bytes
383 digest[0] = (Guchar)(a & 0xff);
384 digest[1] = (Guchar)((a >>= 8) & 0xff);
385 digest[2] = (Guchar)((a >>= 8) & 0xff);
386 digest[3] = (Guchar)((a >>= 8) & 0xff);
387 digest[4] = (Guchar)(b & 0xff);
388 digest[5] = (Guchar)((b >>= 8) & 0xff);
389 digest[6] = (Guchar)((b >>= 8) & 0xff);
390 digest[7] = (Guchar)((b >>= 8) & 0xff);
391 digest[8] = (Guchar)(c & 0xff);
392 digest[9] = (Guchar)((c >>= 8) & 0xff);
393 digest[10] = (Guchar)((c >>= 8) & 0xff);
394 digest[11] = (Guchar)((c >>= 8) & 0xff);
395 digest[12] = (Guchar)(d & 0xff);
396 digest[13] = (Guchar)((d >>= 8) & 0xff);
397 digest[14] = (Guchar)((d >>= 8) & 0xff);
398 digest[15] = (Guchar)((d >>= 8) & 0xff);
399 }
SWFTools is a collection of utilities for working with Adobe Flash files