| blob | ad48032ca8e06bbf9ff471f6953c3b48e57b5aa1 |
1 /*
2 * AES modes (ECB/CBC/CTR/XTS) for PPC AES implementation
3 *
4 * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the Free
8 * Software Foundation; either version 2 of the License, or (at your option)
9 * any later version.
10 *
11 */
13 #include <asm/ppc_asm.h>
14 #include "aes-spe-regs.h"
16 #ifdef __BIG_ENDIAN__ /* Macros for big endian builds */
18 #define LOAD_DATA(reg, off) \
19 lwz reg,off(rSP); /* load with offset */
20 #define SAVE_DATA(reg, off) \
21 stw reg,off(rDP); /* save with offset */
22 #define NEXT_BLOCK \
23 addi rSP,rSP,16; /* increment pointers per bloc */ \
24 addi rDP,rDP,16;
25 #define LOAD_IV(reg, off) \
26 lwz reg,off(rIP); /* IV loading with offset */
27 #define SAVE_IV(reg, off) \
28 stw reg,off(rIP); /* IV saving with offset */
29 #define START_IV /* nothing to reset */
30 #define CBC_DEC 16 /* CBC decrement per block */
31 #define CTR_DEC 1 /* CTR decrement one byte */
33 #else /* Macros for little endian */
35 #define LOAD_DATA(reg, off) \
36 lwbrx reg,0,rSP; /* load reversed */ \
37 addi rSP,rSP,4; /* and increment pointer */
38 #define SAVE_DATA(reg, off) \
39 stwbrx reg,0,rDP; /* save reversed */ \
40 addi rDP,rDP,4; /* and increment pointer */
41 #define NEXT_BLOCK /* nothing todo */
42 #define LOAD_IV(reg, off) \
43 lwbrx reg,0,rIP; /* load reversed */ \
44 addi rIP,rIP,4; /* and increment pointer */
45 #define SAVE_IV(reg, off) \
46 stwbrx reg,0,rIP; /* load reversed */ \
47 addi rIP,rIP,4; /* and increment pointer */
48 #define START_IV \
49 subi rIP,rIP,16; /* must reset pointer */
50 #define CBC_DEC 32 /* 2 blocks because of incs */
51 #define CTR_DEC 17 /* 1 block because of incs */
53 #endif
55 #define SAVE_0_REGS
56 #define LOAD_0_REGS
58 #define SAVE_4_REGS \
59 stw rI0,96(r1); /* save 32 bit registers */ \
60 stw rI1,100(r1); \
61 stw rI2,104(r1); \
62 stw rI3,108(r1);
64 #define LOAD_4_REGS \
65 lwz rI0,96(r1); /* restore 32 bit registers */ \
66 lwz rI1,100(r1); \
67 lwz rI2,104(r1); \
68 lwz rI3,108(r1);
70 #define SAVE_8_REGS \
71 SAVE_4_REGS \
72 stw rG0,112(r1); /* save 32 bit registers */ \
73 stw rG1,116(r1); \
74 stw rG2,120(r1); \
75 stw rG3,124(r1);
77 #define LOAD_8_REGS \
78 LOAD_4_REGS \
79 lwz rG0,112(r1); /* restore 32 bit registers */ \
80 lwz rG1,116(r1); \
81 lwz rG2,120(r1); \
82 lwz rG3,124(r1);
84 #define INITIALIZE_CRYPT(tab,nr32bitregs) \
85 mflr r0; \
86 stwu r1,-160(r1); /* create stack frame */ \
87 lis rT0,tab@h; /* en-/decryption table pointer */ \
88 stw r0,8(r1); /* save link register */ \
89 ori rT0,rT0,tab@l; \
90 evstdw r14,16(r1); \
91 mr rKS,rKP; \
92 evstdw r15,24(r1); /* We must save non volatile */ \
93 evstdw r16,32(r1); /* registers. Take the chance */ \
94 evstdw r17,40(r1); /* and save the SPE part too */ \
95 evstdw r18,48(r1); \
96 evstdw r19,56(r1); \
97 evstdw r20,64(r1); \
98 evstdw r21,72(r1); \
99 evstdw r22,80(r1); \
100 evstdw r23,88(r1); \
101 SAVE_##nr32bitregs##_REGS
103 #define FINALIZE_CRYPT(nr32bitregs) \
104 lwz r0,8(r1); \
105 evldw r14,16(r1); /* restore SPE registers */ \
106 evldw r15,24(r1); \
107 evldw r16,32(r1); \
108 evldw r17,40(r1); \
109 evldw r18,48(r1); \
110 evldw r19,56(r1); \
111 evldw r20,64(r1); \
112 evldw r21,72(r1); \
113 evldw r22,80(r1); \
114 evldw r23,88(r1); \
115 LOAD_##nr32bitregs##_REGS \
116 mtlr r0; /* restore link register */ \
117 xor r0,r0,r0; \
118 stw r0,16(r1); /* delete sensitive data */ \
119 stw r0,24(r1); /* that we might have pushed */ \
120 stw r0,32(r1); /* from other context that runs */ \
121 stw r0,40(r1); /* the same code */ \
122 stw r0,48(r1); \
123 stw r0,56(r1); \
124 stw r0,64(r1); \
125 stw r0,72(r1); \
126 stw r0,80(r1); \
127 stw r0,88(r1); \
128 addi r1,r1,160; /* cleanup stack frame */
130 #define ENDIAN_SWAP(t0, t1, s0, s1) \
131 rotrwi t0,s0,8; /* swap endianness for 2 GPRs */ \
132 rotrwi t1,s1,8; \
133 rlwimi t0,s0,8,8,15; \
134 rlwimi t1,s1,8,8,15; \
135 rlwimi t0,s0,8,24,31; \
136 rlwimi t1,s1,8,24,31;
138 #define GF128_MUL(d0, d1, d2, d3, t0) \
139 li t0,0x87; /* multiplication in GF128 */ \
140 cmpwi d3,-1; \
141 iselgt t0,0,t0; \
142 rlwimi d3,d2,0,0,0; /* propagate "carry" bits */ \
143 rotlwi d3,d3,1; \
144 rlwimi d2,d1,0,0,0; \
145 rotlwi d2,d2,1; \
146 rlwimi d1,d0,0,0,0; \
147 slwi d0,d0,1; /* shift left 128 bit */ \
148 rotlwi d1,d1,1; \
149 xor d0,d0,t0;
151 #define START_KEY(d0, d1, d2, d3) \
152 lwz rW0,0(rKP); \
153 mtctr rRR; \
154 lwz rW1,4(rKP); \
155 lwz rW2,8(rKP); \
156 lwz rW3,12(rKP); \
157 xor rD0,d0,rW0; \
158 xor rD1,d1,rW1; \
159 xor rD2,d2,rW2; \
160 xor rD3,d3,rW3;
162 /*
163 * ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc,
164 * u32 rounds)
165 *
166 * called from glue layer to encrypt a single 16 byte block
167 * round values are AES128 = 4, AES192 = 5, AES256 = 6
168 *
169 */
170 _GLOBAL(ppc_encrypt_aes)
171 INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0)
172 LOAD_DATA(rD0, 0)
173 LOAD_DATA(rD1, 4)
174 LOAD_DATA(rD2, 8)
175 LOAD_DATA(rD3, 12)
176 START_KEY(rD0, rD1, rD2, rD3)
177 bl ppc_encrypt_block
178 xor rD0,rD0,rW0
179 SAVE_DATA(rD0, 0)
180 xor rD1,rD1,rW1
181 SAVE_DATA(rD1, 4)
182 xor rD2,rD2,rW2
183 SAVE_DATA(rD2, 8)
184 xor rD3,rD3,rW3
185 SAVE_DATA(rD3, 12)
186 FINALIZE_CRYPT(0)
187 blr
189 /*
190 * ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec,
191 * u32 rounds)
192 *
193 * called from glue layer to decrypt a single 16 byte block
194 * round values are AES128 = 4, AES192 = 5, AES256 = 6
195 *
196 */
197 _GLOBAL(ppc_decrypt_aes)
198 INITIALIZE_CRYPT(PPC_AES_4K_DECTAB,0)
199 LOAD_DATA(rD0, 0)
200 addi rT1,rT0,4096
201 LOAD_DATA(rD1, 4)
202 LOAD_DATA(rD2, 8)
203 LOAD_DATA(rD3, 12)
204 START_KEY(rD0, rD1, rD2, rD3)
205 bl ppc_decrypt_block
206 xor rD0,rD0,rW0
207 SAVE_DATA(rD0, 0)
208 xor rD1,rD1,rW1
209 SAVE_DATA(rD1, 4)
210 xor rD2,rD2,rW2
211 SAVE_DATA(rD2, 8)
212 xor rD3,rD3,rW3
213 SAVE_DATA(rD3, 12)
214 FINALIZE_CRYPT(0)
215 blr
217 /*
218 * ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc,
219 * u32 rounds, u32 bytes);
220 *
221 * called from glue layer to encrypt multiple blocks via ECB
222 * Bytes must be larger or equal 16 and only whole blocks are
223 * processed. round values are AES128 = 4, AES192 = 5 and
224 * AES256 = 6
225 *
226 */
227 _GLOBAL(ppc_encrypt_ecb)
228 INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0)
229 ppc_encrypt_ecb_loop:
230 LOAD_DATA(rD0, 0)
231 mr rKP,rKS
232 LOAD_DATA(rD1, 4)
233 subi rLN,rLN,16
234 LOAD_DATA(rD2, 8)
235 cmpwi rLN,15
236 LOAD_DATA(rD3, 12)
237 START_KEY(rD0, rD1, rD2, rD3)
238 bl ppc_encrypt_block
239 xor rD0,rD0,rW0
240 SAVE_DATA(rD0, 0)
241 xor rD1,rD1,rW1
242 SAVE_DATA(rD1, 4)
243 xor rD2,rD2,rW2
244 SAVE_DATA(rD2, 8)
245 xor rD3,rD3,rW3
246 SAVE_DATA(rD3, 12)
247 NEXT_BLOCK
248 bt gt,ppc_encrypt_ecb_loop
249 FINALIZE_CRYPT(0)
250 blr
252 /*
253 * ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec,
254 * u32 rounds, u32 bytes);
255 *
256 * called from glue layer to decrypt multiple blocks via ECB
257 * Bytes must be larger or equal 16 and only whole blocks are
258 * processed. round values are AES128 = 4, AES192 = 5 and
259 * AES256 = 6
260 *
261 */
262 _GLOBAL(ppc_decrypt_ecb)
263 INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 0)
264 addi rT1,rT0,4096
265 ppc_decrypt_ecb_loop:
266 LOAD_DATA(rD0, 0)
267 mr rKP,rKS
268 LOAD_DATA(rD1, 4)
269 subi rLN,rLN,16
270 LOAD_DATA(rD2, 8)
271 cmpwi rLN,15
272 LOAD_DATA(rD3, 12)
273 START_KEY(rD0, rD1, rD2, rD3)
274 bl ppc_decrypt_block
275 xor rD0,rD0,rW0
276 SAVE_DATA(rD0, 0)
277 xor rD1,rD1,rW1
278 SAVE_DATA(rD1, 4)
279 xor rD2,rD2,rW2
280 SAVE_DATA(rD2, 8)
281 xor rD3,rD3,rW3
282 SAVE_DATA(rD3, 12)
283 NEXT_BLOCK
284 bt gt,ppc_decrypt_ecb_loop
285 FINALIZE_CRYPT(0)
286 blr
288 /*
289 * ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc,
290 * 32 rounds, u32 bytes, u8 *iv);
291 *
292 * called from glue layer to encrypt multiple blocks via CBC
293 * Bytes must be larger or equal 16 and only whole blocks are
294 * processed. round values are AES128 = 4, AES192 = 5 and
295 * AES256 = 6
296 *
297 */
298 _GLOBAL(ppc_encrypt_cbc)
299 INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4)
300 LOAD_IV(rI0, 0)
301 LOAD_IV(rI1, 4)
302 LOAD_IV(rI2, 8)
303 LOAD_IV(rI3, 12)
304 ppc_encrypt_cbc_loop:
305 LOAD_DATA(rD0, 0)
306 mr rKP,rKS
307 LOAD_DATA(rD1, 4)
308 subi rLN,rLN,16
309 LOAD_DATA(rD2, 8)
310 cmpwi rLN,15
311 LOAD_DATA(rD3, 12)
312 xor rD0,rD0,rI0
313 xor rD1,rD1,rI1
314 xor rD2,rD2,rI2
315 xor rD3,rD3,rI3
316 START_KEY(rD0, rD1, rD2, rD3)
317 bl ppc_encrypt_block
318 xor rI0,rD0,rW0
319 SAVE_DATA(rI0, 0)
320 xor rI1,rD1,rW1
321 SAVE_DATA(rI1, 4)
322 xor rI2,rD2,rW2
323 SAVE_DATA(rI2, 8)
324 xor rI3,rD3,rW3
325 SAVE_DATA(rI3, 12)
326 NEXT_BLOCK
327 bt gt,ppc_encrypt_cbc_loop
328 START_IV
329 SAVE_IV(rI0, 0)
330 SAVE_IV(rI1, 4)
331 SAVE_IV(rI2, 8)
332 SAVE_IV(rI3, 12)
333 FINALIZE_CRYPT(4)
334 blr
336 /*
337 * ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec,
338 * u32 rounds, u32 bytes, u8 *iv);
339 *
340 * called from glue layer to decrypt multiple blocks via CBC
341 * round values are AES128 = 4, AES192 = 5, AES256 = 6
342 *
343 */
344 _GLOBAL(ppc_decrypt_cbc)
345 INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 4)
346 li rT1,15
347 LOAD_IV(rI0, 0)
348 andc rLN,rLN,rT1
349 LOAD_IV(rI1, 4)
350 subi rLN,rLN,16
351 LOAD_IV(rI2, 8)
352 add rSP,rSP,rLN /* reverse processing */
353 LOAD_IV(rI3, 12)
354 add rDP,rDP,rLN
355 LOAD_DATA(rD0, 0)
356 addi rT1,rT0,4096
357 LOAD_DATA(rD1, 4)
358 LOAD_DATA(rD2, 8)
359 LOAD_DATA(rD3, 12)
360 START_IV
361 SAVE_IV(rD0, 0)
362 SAVE_IV(rD1, 4)
363 SAVE_IV(rD2, 8)
364 cmpwi rLN,16
365 SAVE_IV(rD3, 12)
366 bt lt,ppc_decrypt_cbc_end
367 ppc_decrypt_cbc_loop:
368 mr rKP,rKS
369 START_KEY(rD0, rD1, rD2, rD3)
370 bl ppc_decrypt_block
371 subi rLN,rLN,16
372 subi rSP,rSP,CBC_DEC
373 xor rW0,rD0,rW0
374 LOAD_DATA(rD0, 0)
375 xor rW1,rD1,rW1
376 LOAD_DATA(rD1, 4)
377 xor rW2,rD2,rW2
378 LOAD_DATA(rD2, 8)
379 xor rW3,rD3,rW3
380 LOAD_DATA(rD3, 12)
381 xor rW0,rW0,rD0
382 SAVE_DATA(rW0, 0)
383 xor rW1,rW1,rD1
384 SAVE_DATA(rW1, 4)
385 xor rW2,rW2,rD2
386 SAVE_DATA(rW2, 8)
387 xor rW3,rW3,rD3
388 SAVE_DATA(rW3, 12)
389 cmpwi rLN,15
390 subi rDP,rDP,CBC_DEC
391 bt gt,ppc_decrypt_cbc_loop
392 ppc_decrypt_cbc_end:
393 mr rKP,rKS
394 START_KEY(rD0, rD1, rD2, rD3)
395 bl ppc_decrypt_block
396 xor rW0,rW0,rD0
397 xor rW1,rW1,rD1
398 xor rW2,rW2,rD2
399 xor rW3,rW3,rD3
400 xor rW0,rW0,rI0 /* decrypt with initial IV */
401 SAVE_DATA(rW0, 0)
402 xor rW1,rW1,rI1
403 SAVE_DATA(rW1, 4)
404 xor rW2,rW2,rI2
405 SAVE_DATA(rW2, 8)
406 xor rW3,rW3,rI3
407 SAVE_DATA(rW3, 12)
408 FINALIZE_CRYPT(4)
409 blr
411 /*
412 * ppc_crypt_ctr(u8 *out, const u8 *in, u32 *key_enc,
413 * u32 rounds, u32 bytes, u8 *iv);
414 *
415 * called from glue layer to encrypt/decrypt multiple blocks
416 * via CTR. Number of bytes does not need to be a multiple of
417 * 16. Round values are AES128 = 4, AES192 = 5, AES256 = 6
418 *
419 */
420 _GLOBAL(ppc_crypt_ctr)
421 INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4)
422 LOAD_IV(rI0, 0)
423 LOAD_IV(rI1, 4)
424 LOAD_IV(rI2, 8)
425 cmpwi rLN,16
426 LOAD_IV(rI3, 12)
427 START_IV
428 bt lt,ppc_crypt_ctr_partial
429 ppc_crypt_ctr_loop:
430 mr rKP,rKS
431 START_KEY(rI0, rI1, rI2, rI3)
432 bl ppc_encrypt_block
433 xor rW0,rD0,rW0
434 xor rW1,rD1,rW1
435 xor rW2,rD2,rW2
436 xor rW3,rD3,rW3
437 LOAD_DATA(rD0, 0)
438 subi rLN,rLN,16
439 LOAD_DATA(rD1, 4)
440 LOAD_DATA(rD2, 8)
441 LOAD_DATA(rD3, 12)
442 xor rD0,rD0,rW0
443 SAVE_DATA(rD0, 0)
444 xor rD1,rD1,rW1
445 SAVE_DATA(rD1, 4)
446 xor rD2,rD2,rW2
447 SAVE_DATA(rD2, 8)
448 xor rD3,rD3,rW3
449 SAVE_DATA(rD3, 12)
450 addic rI3,rI3,1 /* increase counter */
451 addze rI2,rI2
452 addze rI1,rI1
453 addze rI0,rI0
454 NEXT_BLOCK
455 cmpwi rLN,15
456 bt gt,ppc_crypt_ctr_loop
457 ppc_crypt_ctr_partial:
458 cmpwi rLN,0
459 bt eq,ppc_crypt_ctr_end
460 mr rKP,rKS
461 START_KEY(rI0, rI1, rI2, rI3)
462 bl ppc_encrypt_block
463 xor rW0,rD0,rW0
464 SAVE_IV(rW0, 0)
465 xor rW1,rD1,rW1
466 SAVE_IV(rW1, 4)
467 xor rW2,rD2,rW2
468 SAVE_IV(rW2, 8)
469 xor rW3,rD3,rW3
470 SAVE_IV(rW3, 12)
471 mtctr rLN
472 subi rIP,rIP,CTR_DEC
473 subi rSP,rSP,1
474 subi rDP,rDP,1
475 ppc_crypt_ctr_xorbyte:
476 lbzu rW4,1(rIP) /* bytewise xor for partial block */
477 lbzu rW5,1(rSP)
478 xor rW4,rW4,rW5
479 stbu rW4,1(rDP)
480 bdnz ppc_crypt_ctr_xorbyte
481 subf rIP,rLN,rIP
482 addi rIP,rIP,1
483 addic rI3,rI3,1
484 addze rI2,rI2
485 addze rI1,rI1
486 addze rI0,rI0
487 ppc_crypt_ctr_end:
488 SAVE_IV(rI0, 0)
489 SAVE_IV(rI1, 4)
490 SAVE_IV(rI2, 8)
491 SAVE_IV(rI3, 12)
492 FINALIZE_CRYPT(4)
493 blr
495 /*
496 * ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc,
497 * u32 rounds, u32 bytes, u8 *iv, u32 *key_twk);
498 *
499 * called from glue layer to encrypt multiple blocks via XTS
500 * If key_twk is given, the initial IV encryption will be
501 * processed too. Round values are AES128 = 4, AES192 = 5,
502 * AES256 = 6
503 *
504 */
505 _GLOBAL(ppc_encrypt_xts)
506 INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 8)
507 LOAD_IV(rI0, 0)
508 LOAD_IV(rI1, 4)
509 LOAD_IV(rI2, 8)
510 cmpwi rKT,0
511 LOAD_IV(rI3, 12)
512 bt eq,ppc_encrypt_xts_notweak
513 mr rKP,rKT
514 START_KEY(rI0, rI1, rI2, rI3)
515 bl ppc_encrypt_block
516 xor rI0,rD0,rW0
517 xor rI1,rD1,rW1
518 xor rI2,rD2,rW2
519 xor rI3,rD3,rW3
520 ppc_encrypt_xts_notweak:
521 ENDIAN_SWAP(rG0, rG1, rI0, rI1)
522 ENDIAN_SWAP(rG2, rG3, rI2, rI3)
523 ppc_encrypt_xts_loop:
524 LOAD_DATA(rD0, 0)
525 mr rKP,rKS
526 LOAD_DATA(rD1, 4)
527 subi rLN,rLN,16
528 LOAD_DATA(rD2, 8)
529 LOAD_DATA(rD3, 12)
530 xor rD0,rD0,rI0
531 xor rD1,rD1,rI1
532 xor rD2,rD2,rI2
533 xor rD3,rD3,rI3
534 START_KEY(rD0, rD1, rD2, rD3)
535 bl ppc_encrypt_block
536 xor rD0,rD0,rW0
537 xor rD1,rD1,rW1
538 xor rD2,rD2,rW2
539 xor rD3,rD3,rW3
540 xor rD0,rD0,rI0
541 SAVE_DATA(rD0, 0)
542 xor rD1,rD1,rI1
543 SAVE_DATA(rD1, 4)
544 xor rD2,rD2,rI2
545 SAVE_DATA(rD2, 8)
546 xor rD3,rD3,rI3
547 SAVE_DATA(rD3, 12)
548 GF128_MUL(rG0, rG1, rG2, rG3, rW0)
549 ENDIAN_SWAP(rI0, rI1, rG0, rG1)
550 ENDIAN_SWAP(rI2, rI3, rG2, rG3)
551 cmpwi rLN,0
552 NEXT_BLOCK
553 bt gt,ppc_encrypt_xts_loop
554 START_IV
555 SAVE_IV(rI0, 0)
556 SAVE_IV(rI1, 4)
557 SAVE_IV(rI2, 8)
558 SAVE_IV(rI3, 12)
559 FINALIZE_CRYPT(8)
560 blr
562 /*
563 * ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec,
564 * u32 rounds, u32 blocks, u8 *iv, u32 *key_twk);
565 *
566 * called from glue layer to decrypt multiple blocks via XTS
567 * If key_twk is given, the initial IV encryption will be
568 * processed too. Round values are AES128 = 4, AES192 = 5,
569 * AES256 = 6
570 *
571 */
572 _GLOBAL(ppc_decrypt_xts)
573 INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 8)
574 LOAD_IV(rI0, 0)
575 addi rT1,rT0,4096
576 LOAD_IV(rI1, 4)
577 LOAD_IV(rI2, 8)
578 cmpwi rKT,0
579 LOAD_IV(rI3, 12)
580 bt eq,ppc_decrypt_xts_notweak
581 subi rT0,rT0,4096
582 mr rKP,rKT
583 START_KEY(rI0, rI1, rI2, rI3)
584 bl ppc_encrypt_block
585 xor rI0,rD0,rW0
586 xor rI1,rD1,rW1
587 xor rI2,rD2,rW2
588 xor rI3,rD3,rW3
589 addi rT0,rT0,4096
590 ppc_decrypt_xts_notweak:
591 ENDIAN_SWAP(rG0, rG1, rI0, rI1)
592 ENDIAN_SWAP(rG2, rG3, rI2, rI3)
593 ppc_decrypt_xts_loop:
594 LOAD_DATA(rD0, 0)
595 mr rKP,rKS
596 LOAD_DATA(rD1, 4)
597 subi rLN,rLN,16
598 LOAD_DATA(rD2, 8)
599 LOAD_DATA(rD3, 12)
600 xor rD0,rD0,rI0
601 xor rD1,rD1,rI1
602 xor rD2,rD2,rI2
603 xor rD3,rD3,rI3
604 START_KEY(rD0, rD1, rD2, rD3)
605 bl ppc_decrypt_block
606 xor rD0,rD0,rW0
607 xor rD1,rD1,rW1
608 xor rD2,rD2,rW2
609 xor rD3,rD3,rW3
610 xor rD0,rD0,rI0
611 SAVE_DATA(rD0, 0)
612 xor rD1,rD1,rI1
613 SAVE_DATA(rD1, 4)
614 xor rD2,rD2,rI2
615 SAVE_DATA(rD2, 8)
616 xor rD3,rD3,rI3
617 SAVE_DATA(rD3, 12)
618 GF128_MUL(rG0, rG1, rG2, rG3, rW0)
619 ENDIAN_SWAP(rI0, rI1, rG0, rG1)
620 ENDIAN_SWAP(rI2, rI3, rG2, rG3)
621 cmpwi rLN,0
622 NEXT_BLOCK
623 bt gt,ppc_decrypt_xts_loop
624 START_IV
625 SAVE_IV(rI0, 0)
626 SAVE_IV(rI1, 4)
627 SAVE_IV(rI2, 8)
628 SAVE_IV(rI3, 12)
629 FINALIZE_CRYPT(8)
630 blr