arch/arm64/crypto/sm4-ce-core.S

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 /*
   3  * SM4 Cipher Algorithm for ARMv8 with Crypto Extensions
   4  * as specified in
   5  * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
   6  *
   7  * Copyright (C) 2022, Alibaba Group.
   8  * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
   9  */
  10
  11 #include <linux/linkage.h>
  12 #include <asm/assembler.h>
  13 #include "sm4-ce-asm.h"
  14
  15 .arch   armv8-a+crypto
  16
  17 .irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, \
  18                 20, 24, 25, 26, 27, 28, 29, 30, 31
  19         .set .Lv\b\().4s, \b
  20 .endr
  21
  22 .macro sm4e, vd, vn
  23         .inst 0xcec08400 | (.L\vn << 5) | .L\vd
  24 .endm
  25
  26 .macro sm4ekey, vd, vn, vm
  27         .inst 0xce60c800 | (.L\vm << 16) | (.L\vn << 5) | .L\vd
  28 .endm
  29
  30 /* Register macros */
  31
  32 #define RTMP0   v16
  33 #define RTMP1   v17
  34 #define RTMP2   v18
  35 #define RTMP3   v19
  36
  37 #define RIV     v20
  38 #define RMAC    v20
  39 #define RMASK   v21
  40
  41
  42 .align 3
  43 SYM_FUNC_START(sm4_ce_expand_key)
  44         /* input:
  45          *   x0: 128-bit key
  46          *   x1: rkey_enc
  47          *   x2: rkey_dec
  48          *   x3: fk array
  49          *   x4: ck array
  50          */
  51         ld1             {v0.16b}, [x0];
  52         rev32           v0.16b, v0.16b;
  53         ld1             {v1.16b}, [x3];
  54         /* load ck */
  55         ld1             {v24.16b-v27.16b}, [x4], #64;
  56         ld1             {v28.16b-v31.16b}, [x4];
  57
  58         /* input ^ fk */
  59         eor             v0.16b, v0.16b, v1.16b;
  60
  61         sm4ekey         v0.4s, v0.4s, v24.4s;
  62         sm4ekey         v1.4s, v0.4s, v25.4s;
  63         sm4ekey         v2.4s, v1.4s, v26.4s;
  64         sm4ekey         v3.4s, v2.4s, v27.4s;
  65         sm4ekey         v4.4s, v3.4s, v28.4s;
  66         sm4ekey         v5.4s, v4.4s, v29.4s;
  67         sm4ekey         v6.4s, v5.4s, v30.4s;
  68         sm4ekey         v7.4s, v6.4s, v31.4s;
  69
  70         adr_l           x5, .Lbswap128_mask
  71         ld1             {v24.16b}, [x5]
  72
  73         st1             {v0.16b-v3.16b}, [x1], #64;
  74         st1             {v4.16b-v7.16b}, [x1];
  75
  76         tbl             v16.16b, {v7.16b}, v24.16b
  77         tbl             v17.16b, {v6.16b}, v24.16b
  78         tbl             v18.16b, {v5.16b}, v24.16b
  79         tbl             v19.16b, {v4.16b}, v24.16b
  80         tbl             v20.16b, {v3.16b}, v24.16b
  81         tbl             v21.16b, {v2.16b}, v24.16b
  82         tbl             v22.16b, {v1.16b}, v24.16b
  83         tbl             v23.16b, {v0.16b}, v24.16b
  84
  85         st1             {v16.16b-v19.16b}, [x2], #64
  86         st1             {v20.16b-v23.16b}, [x2]
  87
  88         ret;
  89 SYM_FUNC_END(sm4_ce_expand_key)
  90
  91 .align 3
  92 SYM_FUNC_START(sm4_ce_crypt_block)
  93         /* input:
  94          *   x0: round key array, CTX
  95          *   x1: dst
  96          *   x2: src
  97          */
  98         SM4_PREPARE(x0)
  99
 100         ld1             {v0.16b}, [x2];
 101         SM4_CRYPT_BLK(v0);
 102         st1             {v0.16b}, [x1];
 103
 104         ret;
 105 SYM_FUNC_END(sm4_ce_crypt_block)
 106
 107 .align 3
 108 SYM_FUNC_START(sm4_ce_crypt)
 109         /* input:
 110          *   x0: round key array, CTX
 111          *   x1: dst
 112          *   x2: src
 113          *   w3: nblocks
 114          */
 115         SM4_PREPARE(x0)
 116
 117 .Lcrypt_loop_blk:
 118         sub             w3, w3, #8;
 119         tbnz            w3, #31, .Lcrypt_tail8;
 120
 121         ld1             {v0.16b-v3.16b}, [x2], #64;
 122         ld1             {v4.16b-v7.16b}, [x2], #64;
 123
 124         SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7);
 125
 126         st1             {v0.16b-v3.16b}, [x1], #64;
 127         st1             {v4.16b-v7.16b}, [x1], #64;
 128
 129         cbz             w3, .Lcrypt_end;
 130         b               .Lcrypt_loop_blk;
 131
 132 .Lcrypt_tail8:
 133         add             w3, w3, #8;
 134         cmp             w3, #4;
 135         blt             .Lcrypt_tail4;
 136
 137         sub             w3, w3, #4;
 138
 139         ld1             {v0.16b-v3.16b}, [x2], #64;
 140         SM4_CRYPT_BLK4(v0, v1, v2, v3);
 141         st1             {v0.16b-v3.16b}, [x1], #64;
 142
 143         cbz             w3, .Lcrypt_end;
 144
 145 .Lcrypt_tail4:
 146         sub             w3, w3, #1;
 147
 148         ld1             {v0.16b}, [x2], #16;
 149         SM4_CRYPT_BLK(v0);
 150         st1             {v0.16b}, [x1], #16;
 151
 152         cbnz            w3, .Lcrypt_tail4;
 153
 154 .Lcrypt_end:
 155         ret;
 156 SYM_FUNC_END(sm4_ce_crypt)
 157
 158 .align 3
 159 SYM_FUNC_START(sm4_ce_cbc_enc)
 160         /* input:
 161          *   x0: round key array, CTX
 162          *   x1: dst
 163          *   x2: src
 164          *   x3: iv (big endian, 128 bit)
 165          *   w4: nblocks
 166          */
 167         SM4_PREPARE(x0)
 168
 169         ld1             {RIV.16b}, [x3]
 170
 171 .Lcbc_enc_loop_4x:
 172         cmp             w4, #4
 173         blt             .Lcbc_enc_loop_1x
 174
 175         sub             w4, w4, #4
 176
 177         ld1             {v0.16b-v3.16b}, [x2], #64
 178
 179         eor             v0.16b, v0.16b, RIV.16b
 180         SM4_CRYPT_BLK(v0)
 181         eor             v1.16b, v1.16b, v0.16b
 182         SM4_CRYPT_BLK(v1)
 183         eor             v2.16b, v2.16b, v1.16b
 184         SM4_CRYPT_BLK(v2)
 185         eor             v3.16b, v3.16b, v2.16b
 186         SM4_CRYPT_BLK(v3)
 187
 188         st1             {v0.16b-v3.16b}, [x1], #64
 189         mov             RIV.16b, v3.16b
 190
 191         cbz             w4, .Lcbc_enc_end
 192         b               .Lcbc_enc_loop_4x
 193
 194 .Lcbc_enc_loop_1x:
 195         sub             w4, w4, #1
 196
 197         ld1             {v0.16b}, [x2], #16
 198
 199         eor             RIV.16b, RIV.16b, v0.16b
 200         SM4_CRYPT_BLK(RIV)
 201
 202         st1             {RIV.16b}, [x1], #16
 203
 204         cbnz            w4, .Lcbc_enc_loop_1x
 205
 206 .Lcbc_enc_end:
 207         /* store new IV */
 208         st1             {RIV.16b}, [x3]
 209
 210         ret
 211 SYM_FUNC_END(sm4_ce_cbc_enc)
 212
 213 .align 3
 214 SYM_FUNC_START(sm4_ce_cbc_dec)
 215         /* input:
 216          *   x0: round key array, CTX
 217          *   x1: dst
 218          *   x2: src
 219          *   x3: iv (big endian, 128 bit)
 220          *   w4: nblocks
 221          */
 222         SM4_PREPARE(x0)
 223
 224         ld1             {RIV.16b}, [x3]
 225
 226 .Lcbc_dec_loop_8x:
 227         sub             w4, w4, #8
 228         tbnz            w4, #31, .Lcbc_dec_4x
 229
 230         ld1             {v0.16b-v3.16b}, [x2], #64
 231         ld1             {v4.16b-v7.16b}, [x2], #64
 232
 233         rev32           v8.16b, v0.16b
 234         rev32           v9.16b, v1.16b
 235         rev32           v10.16b, v2.16b
 236         rev32           v11.16b, v3.16b
 237         rev32           v12.16b, v4.16b
 238         rev32           v13.16b, v5.16b
 239         rev32           v14.16b, v6.16b
 240         rev32           v15.16b, v7.16b
 241
 242         SM4_CRYPT_BLK8_BE(v8, v9, v10, v11, v12, v13, v14, v15)
 243
 244         eor             v8.16b, v8.16b, RIV.16b
 245         eor             v9.16b, v9.16b, v0.16b
 246         eor             v10.16b, v10.16b, v1.16b
 247         eor             v11.16b, v11.16b, v2.16b
 248         eor             v12.16b, v12.16b, v3.16b
 249         eor             v13.16b, v13.16b, v4.16b
 250         eor             v14.16b, v14.16b, v5.16b
 251         eor             v15.16b, v15.16b, v6.16b
 252
 253         st1             {v8.16b-v11.16b}, [x1], #64
 254         st1             {v12.16b-v15.16b}, [x1], #64
 255
 256         mov             RIV.16b, v7.16b
 257
 258         cbz             w4, .Lcbc_dec_end
 259         b               .Lcbc_dec_loop_8x
 260
 261 .Lcbc_dec_4x:
 262         add             w4, w4, #8
 263         cmp             w4, #4
 264         blt             .Lcbc_dec_loop_1x
 265
 266         sub             w4, w4, #4
 267
 268         ld1             {v0.16b-v3.16b}, [x2], #64
 269
 270         rev32           v8.16b, v0.16b
 271         rev32           v9.16b, v1.16b
 272         rev32           v10.16b, v2.16b
 273         rev32           v11.16b, v3.16b
 274
 275         SM4_CRYPT_BLK4_BE(v8, v9, v10, v11)
 276
 277         eor             v8.16b, v8.16b, RIV.16b
 278         eor             v9.16b, v9.16b, v0.16b
 279         eor             v10.16b, v10.16b, v1.16b
 280         eor             v11.16b, v11.16b, v2.16b
 281
 282         st1             {v8.16b-v11.16b}, [x1], #64
 283
 284         mov             RIV.16b, v3.16b
 285
 286         cbz             w4, .Lcbc_dec_end
 287
 288 .Lcbc_dec_loop_1x:
 289         sub             w4, w4, #1
 290
 291         ld1             {v0.16b}, [x2], #16
 292
 293         rev32           v8.16b, v0.16b
 294
 295         SM4_CRYPT_BLK_BE(v8)
 296
 297         eor             v8.16b, v8.16b, RIV.16b
 298         st1             {v8.16b}, [x1], #16
 299
 300         mov             RIV.16b, v0.16b
 301
 302         cbnz            w4, .Lcbc_dec_loop_1x
 303
 304 .Lcbc_dec_end:
 305         /* store new IV */
 306         st1             {RIV.16b}, [x3]
 307
 308         ret
 309 SYM_FUNC_END(sm4_ce_cbc_dec)
 310
 311 .align 3
 312 SYM_FUNC_START(sm4_ce_cbc_cts_enc)
 313         /* input:
 314          *   x0: round key array, CTX
 315          *   x1: dst
 316          *   x2: src
 317          *   x3: iv (big endian, 128 bit)
 318          *   w4: nbytes
 319          */
 320         SM4_PREPARE(x0)
 321
 322         sub             w5, w4, #16
 323         uxtw            x5, w5
 324
 325         ld1             {RIV.16b}, [x3]
 326
 327         ld1             {v0.16b}, [x2]
 328         eor             RIV.16b, RIV.16b, v0.16b
 329         SM4_CRYPT_BLK(RIV)
 330
 331         /* load permute table */
 332         adr_l           x6, .Lcts_permute_table
 333         add             x7, x6, #32
 334         add             x6, x6, x5
 335         sub             x7, x7, x5
 336         ld1             {v3.16b}, [x6]
 337         ld1             {v4.16b}, [x7]
 338
 339         /* overlapping loads */
 340         add             x2, x2, x5
 341         ld1             {v1.16b}, [x2]
 342
 343         /* create Cn from En-1 */
 344         tbl             v0.16b, {RIV.16b}, v3.16b
 345         /* padding Pn with zeros */
 346         tbl             v1.16b, {v1.16b}, v4.16b
 347
 348         eor             v1.16b, v1.16b, RIV.16b
 349         SM4_CRYPT_BLK(v1)
 350
 351         /* overlapping stores */
 352         add             x5, x1, x5
 353         st1             {v0.16b}, [x5]
 354         st1             {v1.16b}, [x1]
 355
 356         ret
 357 SYM_FUNC_END(sm4_ce_cbc_cts_enc)
 358
 359 .align 3
 360 SYM_FUNC_START(sm4_ce_cbc_cts_dec)
 361         /* input:
 362          *   x0: round key array, CTX
 363          *   x1: dst
 364          *   x2: src
 365          *   x3: iv (big endian, 128 bit)
 366          *   w4: nbytes
 367          */
 368         SM4_PREPARE(x0)
 369
 370         sub             w5, w4, #16
 371         uxtw            x5, w5
 372
 373         ld1             {RIV.16b}, [x3]
 374
 375         /* load permute table */
 376         adr_l           x6, .Lcts_permute_table
 377         add             x7, x6, #32
 378         add             x6, x6, x5
 379         sub             x7, x7, x5
 380         ld1             {v3.16b}, [x6]
 381         ld1             {v4.16b}, [x7]
 382
 383         /* overlapping loads */
 384         ld1             {v0.16b}, [x2], x5
 385         ld1             {v1.16b}, [x2]
 386
 387         SM4_CRYPT_BLK(v0)
 388         /* select the first Ln bytes of Xn to create Pn */
 389         tbl             v2.16b, {v0.16b}, v3.16b
 390         eor             v2.16b, v2.16b, v1.16b
 391
 392         /* overwrite the first Ln bytes with Cn to create En-1 */
 393         tbx             v0.16b, {v1.16b}, v4.16b
 394         SM4_CRYPT_BLK(v0)
 395         eor             v0.16b, v0.16b, RIV.16b
 396
 397         /* overlapping stores */
 398         add             x5, x1, x5
 399         st1             {v2.16b}, [x5]
 400         st1             {v0.16b}, [x1]
 401
 402         ret
 403 SYM_FUNC_END(sm4_ce_cbc_cts_dec)
 404
 405 .align 3
 406 SYM_FUNC_START(sm4_ce_ctr_enc)
 407         /* input:
 408          *   x0: round key array, CTX
 409          *   x1: dst
 410          *   x2: src
 411          *   x3: ctr (big endian, 128 bit)
 412          *   w4: nblocks
 413          */
 414         SM4_PREPARE(x0)
 415
 416         ldp             x7, x8, [x3]
 417         rev             x7, x7
 418         rev             x8, x8
 419
 420 .Lctr_loop_8x:
 421         sub             w4, w4, #8
 422         tbnz            w4, #31, .Lctr_4x
 423
 424 #define inc_le128(vctr)                                 \
 425                 mov             vctr.d[1], x8;          \
 426                 mov             vctr.d[0], x7;          \
 427                 adds            x8, x8, #1;             \
 428                 rev64           vctr.16b, vctr.16b;     \
 429                 adc             x7, x7, xzr;
 430
 431         /* construct CTRs */
 432         inc_le128(v0)                   /* +0 */
 433         inc_le128(v1)                   /* +1 */
 434         inc_le128(v2)                   /* +2 */
 435         inc_le128(v3)                   /* +3 */
 436         inc_le128(v4)                   /* +4 */
 437         inc_le128(v5)                   /* +5 */
 438         inc_le128(v6)                   /* +6 */
 439         inc_le128(v7)                   /* +7 */
 440
 441         ld1             {v8.16b-v11.16b}, [x2], #64
 442         ld1             {v12.16b-v15.16b}, [x2], #64
 443
 444         SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7)
 445
 446         eor             v0.16b, v0.16b, v8.16b
 447         eor             v1.16b, v1.16b, v9.16b
 448         eor             v2.16b, v2.16b, v10.16b
 449         eor             v3.16b, v3.16b, v11.16b
 450         eor             v4.16b, v4.16b, v12.16b
 451         eor             v5.16b, v5.16b, v13.16b
 452         eor             v6.16b, v6.16b, v14.16b
 453         eor             v7.16b, v7.16b, v15.16b
 454
 455         st1             {v0.16b-v3.16b}, [x1], #64
 456         st1             {v4.16b-v7.16b}, [x1], #64
 457
 458         cbz             w4, .Lctr_end
 459         b               .Lctr_loop_8x
 460
 461 .Lctr_4x:
 462         add             w4, w4, #8
 463         cmp             w4, #4
 464         blt             .Lctr_loop_1x
 465
 466         sub             w4, w4, #4
 467
 468         /* construct CTRs */
 469         inc_le128(v0)                   /* +0 */
 470         inc_le128(v1)                   /* +1 */
 471         inc_le128(v2)                   /* +2 */
 472         inc_le128(v3)                   /* +3 */
 473
 474         ld1             {v8.16b-v11.16b}, [x2], #64
 475
 476         SM4_CRYPT_BLK4(v0, v1, v2, v3)
 477
 478         eor             v0.16b, v0.16b, v8.16b
 479         eor             v1.16b, v1.16b, v9.16b
 480         eor             v2.16b, v2.16b, v10.16b
 481         eor             v3.16b, v3.16b, v11.16b
 482
 483         st1             {v0.16b-v3.16b}, [x1], #64
 484
 485         cbz             w4, .Lctr_end
 486
 487 .Lctr_loop_1x:
 488         sub             w4, w4, #1
 489
 490         /* construct CTRs */
 491         inc_le128(v0)
 492
 493         ld1             {v8.16b}, [x2], #16
 494
 495         SM4_CRYPT_BLK(v0)
 496
 497         eor             v0.16b, v0.16b, v8.16b
 498         st1             {v0.16b}, [x1], #16
 499
 500         cbnz            w4, .Lctr_loop_1x
 501
 502 .Lctr_end:
 503         /* store new CTR */
 504         rev             x7, x7
 505         rev             x8, x8
 506         stp             x7, x8, [x3]
 507
 508         ret
 509 SYM_FUNC_END(sm4_ce_ctr_enc)
 510
 511
 512 #define tweak_next(vt, vin, RTMP)                                       \
 513                 sshr            RTMP.2d, vin.2d, #63;                   \
 514                 and             RTMP.16b, RTMP.16b, RMASK.16b;          \
 515                 add             vt.2d, vin.2d, vin.2d;                  \
 516                 ext             RTMP.16b, RTMP.16b, RTMP.16b, #8;       \
 517                 eor             vt.16b, vt.16b, RTMP.16b;
 518
 519 .align 3
 520 SYM_FUNC_START(sm4_ce_xts_enc)
 521         /* input:
 522          *   x0: round key array, CTX
 523          *   x1: dst
 524          *   x2: src
 525          *   x3: tweak (big endian, 128 bit)
 526          *   w4: nbytes
 527          *   x5: round key array for IV
 528          */
 529         ld1             {v8.16b}, [x3]
 530
 531         cbz             x5, .Lxts_enc_nofirst
 532
 533         SM4_PREPARE(x5)
 534
 535         /* Generate first tweak */
 536         SM4_CRYPT_BLK(v8)
 537
 538 .Lxts_enc_nofirst:
 539         SM4_PREPARE(x0)
 540
 541         ands            w5, w4, #15
 542         lsr             w4, w4, #4
 543         sub             w6, w4, #1
 544         csel            w4, w4, w6, eq
 545         uxtw            x5, w5
 546
 547         movi            RMASK.2s, #0x1
 548         movi            RTMP0.2s, #0x87
 549         uzp1            RMASK.4s, RMASK.4s, RTMP0.4s
 550
 551         cbz             w4, .Lxts_enc_cts
 552
 553 .Lxts_enc_loop_8x:
 554         sub             w4, w4, #8
 555         tbnz            w4, #31, .Lxts_enc_4x
 556
 557         tweak_next( v9,  v8, RTMP0)
 558         tweak_next(v10,  v9, RTMP1)
 559         tweak_next(v11, v10, RTMP2)
 560         tweak_next(v12, v11, RTMP3)
 561         tweak_next(v13, v12, RTMP0)
 562         tweak_next(v14, v13, RTMP1)
 563         tweak_next(v15, v14, RTMP2)
 564
 565         ld1             {v0.16b-v3.16b}, [x2], #64
 566         ld1             {v4.16b-v7.16b}, [x2], #64
 567         eor             v0.16b, v0.16b,  v8.16b
 568         eor             v1.16b, v1.16b,  v9.16b
 569         eor             v2.16b, v2.16b, v10.16b
 570         eor             v3.16b, v3.16b, v11.16b
 571         eor             v4.16b, v4.16b, v12.16b
 572         eor             v5.16b, v5.16b, v13.16b
 573         eor             v6.16b, v6.16b, v14.16b
 574         eor             v7.16b, v7.16b, v15.16b
 575
 576         SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7)
 577
 578         eor             v0.16b, v0.16b,  v8.16b
 579         eor             v1.16b, v1.16b,  v9.16b
 580         eor             v2.16b, v2.16b, v10.16b
 581         eor             v3.16b, v3.16b, v11.16b
 582         eor             v4.16b, v4.16b, v12.16b
 583         eor             v5.16b, v5.16b, v13.16b
 584         eor             v6.16b, v6.16b, v14.16b
 585         eor             v7.16b, v7.16b, v15.16b
 586         st1             {v0.16b-v3.16b}, [x1], #64
 587         st1             {v4.16b-v7.16b}, [x1], #64
 588
 589         tweak_next(v8, v15, RTMP3)
 590
 591         cbz             w4, .Lxts_enc_cts
 592         b               .Lxts_enc_loop_8x
 593
 594 .Lxts_enc_4x:
 595         add             w4, w4, #8
 596         cmp             w4, #4
 597         blt             .Lxts_enc_loop_1x
 598
 599         sub             w4, w4, #4
 600
 601         tweak_next( v9,  v8, RTMP0)
 602         tweak_next(v10,  v9, RTMP1)
 603         tweak_next(v11, v10, RTMP2)
 604
 605         ld1             {v0.16b-v3.16b}, [x2], #64
 606         eor             v0.16b, v0.16b,  v8.16b
 607         eor             v1.16b, v1.16b,  v9.16b
 608         eor             v2.16b, v2.16b, v10.16b
 609         eor             v3.16b, v3.16b, v11.16b
 610
 611         SM4_CRYPT_BLK4(v0, v1, v2, v3)
 612
 613         eor             v0.16b, v0.16b,  v8.16b
 614         eor             v1.16b, v1.16b,  v9.16b
 615         eor             v2.16b, v2.16b, v10.16b
 616         eor             v3.16b, v3.16b, v11.16b
 617         st1             {v0.16b-v3.16b}, [x1], #64
 618
 619         tweak_next(v8, v11, RTMP3)
 620
 621         cbz             w4, .Lxts_enc_cts
 622
 623 .Lxts_enc_loop_1x:
 624         sub             w4, w4, #1
 625
 626         ld1             {v0.16b}, [x2], #16
 627         eor             v0.16b, v0.16b, v8.16b
 628
 629         SM4_CRYPT_BLK(v0)
 630
 631         eor             v0.16b, v0.16b, v8.16b
 632         st1             {v0.16b}, [x1], #16
 633
 634         tweak_next(v8, v8, RTMP0)
 635
 636         cbnz            w4, .Lxts_enc_loop_1x
 637
 638 .Lxts_enc_cts:
 639         cbz             x5, .Lxts_enc_end
 640
 641         /* cipher text stealing */
 642
 643         tweak_next(v9, v8, RTMP0)
 644         ld1             {v0.16b}, [x2]
 645         eor             v0.16b, v0.16b, v8.16b
 646         SM4_CRYPT_BLK(v0)
 647         eor             v0.16b, v0.16b, v8.16b
 648
 649         /* load permute table */
 650         adr_l           x6, .Lcts_permute_table
 651         add             x7, x6, #32
 652         add             x6, x6, x5
 653         sub             x7, x7, x5
 654         ld1             {v3.16b}, [x6]
 655         ld1             {v4.16b}, [x7]
 656
 657         /* overlapping loads */
 658         add             x2, x2, x5
 659         ld1             {v1.16b}, [x2]
 660
 661         /* create Cn from En-1 */
 662         tbl             v2.16b, {v0.16b}, v3.16b
 663         /* padding Pn with En-1 at the end */
 664         tbx             v0.16b, {v1.16b}, v4.16b
 665
 666         eor             v0.16b, v0.16b, v9.16b
 667         SM4_CRYPT_BLK(v0)
 668         eor             v0.16b, v0.16b, v9.16b
 669
 670
 671         /* overlapping stores */
 672         add             x5, x1, x5
 673         st1             {v2.16b}, [x5]
 674         st1             {v0.16b}, [x1]
 675
 676         b               .Lxts_enc_ret
 677
 678 .Lxts_enc_end:
 679         /* store new tweak */
 680         st1             {v8.16b}, [x3]
 681
 682 .Lxts_enc_ret:
 683         ret
 684 SYM_FUNC_END(sm4_ce_xts_enc)
 685
 686 .align 3
 687 SYM_FUNC_START(sm4_ce_xts_dec)
 688         /* input:
 689          *   x0: round key array, CTX
 690          *   x1: dst
 691          *   x2: src
 692          *   x3: tweak (big endian, 128 bit)
 693          *   w4: nbytes
 694          *   x5: round key array for IV
 695          */
 696         ld1             {v8.16b}, [x3]
 697
 698         cbz             x5, .Lxts_dec_nofirst
 699
 700         SM4_PREPARE(x5)
 701
 702         /* Generate first tweak */
 703         SM4_CRYPT_BLK(v8)
 704
 705 .Lxts_dec_nofirst:
 706         SM4_PREPARE(x0)
 707
 708         ands            w5, w4, #15
 709         lsr             w4, w4, #4
 710         sub             w6, w4, #1
 711         csel            w4, w4, w6, eq
 712         uxtw            x5, w5
 713
 714         movi            RMASK.2s, #0x1
 715         movi            RTMP0.2s, #0x87
 716         uzp1            RMASK.4s, RMASK.4s, RTMP0.4s
 717
 718         cbz             w4, .Lxts_dec_cts
 719
 720 .Lxts_dec_loop_8x:
 721         sub             w4, w4, #8
 722         tbnz            w4, #31, .Lxts_dec_4x
 723
 724         tweak_next( v9,  v8, RTMP0)
 725         tweak_next(v10,  v9, RTMP1)
 726         tweak_next(v11, v10, RTMP2)
 727         tweak_next(v12, v11, RTMP3)
 728         tweak_next(v13, v12, RTMP0)
 729         tweak_next(v14, v13, RTMP1)
 730         tweak_next(v15, v14, RTMP2)
 731
 732         ld1             {v0.16b-v3.16b}, [x2], #64
 733         ld1             {v4.16b-v7.16b}, [x2], #64
 734         eor             v0.16b, v0.16b,  v8.16b
 735         eor             v1.16b, v1.16b,  v9.16b
 736         eor             v2.16b, v2.16b, v10.16b
 737         eor             v3.16b, v3.16b, v11.16b
 738         eor             v4.16b, v4.16b, v12.16b
 739         eor             v5.16b, v5.16b, v13.16b
 740         eor             v6.16b, v6.16b, v14.16b
 741         eor             v7.16b, v7.16b, v15.16b
 742
 743         SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7)
 744
 745         eor             v0.16b, v0.16b,  v8.16b
 746         eor             v1.16b, v1.16b,  v9.16b
 747         eor             v2.16b, v2.16b, v10.16b
 748         eor             v3.16b, v3.16b, v11.16b
 749         eor             v4.16b, v4.16b, v12.16b
 750         eor             v5.16b, v5.16b, v13.16b
 751         eor             v6.16b, v6.16b, v14.16b
 752         eor             v7.16b, v7.16b, v15.16b
 753         st1             {v0.16b-v3.16b}, [x1], #64
 754         st1             {v4.16b-v7.16b}, [x1], #64
 755
 756         tweak_next(v8, v15, RTMP3)
 757
 758         cbz             w4, .Lxts_dec_cts
 759         b               .Lxts_dec_loop_8x
 760
 761 .Lxts_dec_4x:
 762         add             w4, w4, #8
 763         cmp             w4, #4
 764         blt             .Lxts_dec_loop_1x
 765
 766         sub             w4, w4, #4
 767
 768         tweak_next( v9,  v8, RTMP0)
 769         tweak_next(v10,  v9, RTMP1)
 770         tweak_next(v11, v10, RTMP2)
 771
 772         ld1             {v0.16b-v3.16b}, [x2], #64
 773         eor             v0.16b, v0.16b,  v8.16b
 774         eor             v1.16b, v1.16b,  v9.16b
 775         eor             v2.16b, v2.16b, v10.16b
 776         eor             v3.16b, v3.16b, v11.16b
 777
 778         SM4_CRYPT_BLK4(v0, v1, v2, v3)
 779
 780         eor             v0.16b, v0.16b,  v8.16b
 781         eor             v1.16b, v1.16b,  v9.16b
 782         eor             v2.16b, v2.16b, v10.16b
 783         eor             v3.16b, v3.16b, v11.16b
 784         st1             {v0.16b-v3.16b}, [x1], #64
 785
 786         tweak_next(v8, v11, RTMP3)
 787
 788         cbz             w4, .Lxts_dec_cts
 789
 790 .Lxts_dec_loop_1x:
 791         sub             w4, w4, #1
 792
 793         ld1             {v0.16b}, [x2], #16
 794         eor             v0.16b, v0.16b, v8.16b
 795
 796         SM4_CRYPT_BLK(v0)
 797
 798         eor             v0.16b, v0.16b, v8.16b
 799         st1             {v0.16b}, [x1], #16
 800
 801         tweak_next(v8, v8, RTMP0)
 802
 803         cbnz            w4, .Lxts_dec_loop_1x
 804
 805 .Lxts_dec_cts:
 806         cbz             x5, .Lxts_dec_end
 807
 808         /* cipher text stealing */
 809
 810         tweak_next(v9, v8, RTMP0)
 811         ld1             {v0.16b}, [x2]
 812         eor             v0.16b, v0.16b, v9.16b
 813         SM4_CRYPT_BLK(v0)
 814         eor             v0.16b, v0.16b, v9.16b
 815
 816         /* load permute table */
 817         adr_l           x6, .Lcts_permute_table
 818         add             x7, x6, #32
 819         add             x6, x6, x5
 820         sub             x7, x7, x5
 821         ld1             {v3.16b}, [x6]
 822         ld1             {v4.16b}, [x7]
 823
 824         /* overlapping loads */
 825         add             x2, x2, x5
 826         ld1             {v1.16b}, [x2]
 827
 828         /* create Cn from En-1 */
 829         tbl             v2.16b, {v0.16b}, v3.16b
 830         /* padding Pn with En-1 at the end */
 831         tbx             v0.16b, {v1.16b}, v4.16b
 832
 833         eor             v0.16b, v0.16b, v8.16b
 834         SM4_CRYPT_BLK(v0)
 835         eor             v0.16b, v0.16b, v8.16b
 836
 837
 838         /* overlapping stores */
 839         add             x5, x1, x5
 840         st1             {v2.16b}, [x5]
 841         st1             {v0.16b}, [x1]
 842
 843         b               .Lxts_dec_ret
 844
 845 .Lxts_dec_end:
 846         /* store new tweak */
 847         st1             {v8.16b}, [x3]
 848
 849 .Lxts_dec_ret:
 850         ret
 851 SYM_FUNC_END(sm4_ce_xts_dec)
 852
 853 .align 3
 854 SYM_FUNC_START(sm4_ce_mac_update)
 855         /* input:
 856          *   x0: round key array, CTX
 857          *   x1: digest
 858          *   x2: src
 859          *   w3: nblocks
 860          *   w4: enc_before
 861          *   w5: enc_after
 862          */
 863         SM4_PREPARE(x0)
 864
 865         ld1             {RMAC.16b}, [x1]
 866
 867         cbz             w4, .Lmac_update
 868
 869         SM4_CRYPT_BLK(RMAC)
 870
 871 .Lmac_update:
 872         cbz             w3, .Lmac_ret
 873
 874         sub             w6, w3, #1
 875         cmp             w5, wzr
 876         csel            w3, w3, w6, ne
 877
 878         cbz             w3, .Lmac_end
 879
 880 .Lmac_loop_4x:
 881         cmp             w3, #4
 882         blt             .Lmac_loop_1x
 883
 884         sub             w3, w3, #4
 885
 886         ld1             {v0.16b-v3.16b}, [x2], #64
 887
 888         eor             RMAC.16b, RMAC.16b, v0.16b
 889         SM4_CRYPT_BLK(RMAC)
 890         eor             RMAC.16b, RMAC.16b, v1.16b
 891         SM4_CRYPT_BLK(RMAC)
 892         eor             RMAC.16b, RMAC.16b, v2.16b
 893         SM4_CRYPT_BLK(RMAC)
 894         eor             RMAC.16b, RMAC.16b, v3.16b
 895         SM4_CRYPT_BLK(RMAC)
 896
 897         cbz             w3, .Lmac_end
 898         b               .Lmac_loop_4x
 899
 900 .Lmac_loop_1x:
 901         sub             w3, w3, #1
 902
 903         ld1             {v0.16b}, [x2], #16
 904
 905         eor             RMAC.16b, RMAC.16b, v0.16b
 906         SM4_CRYPT_BLK(RMAC)
 907
 908         cbnz            w3, .Lmac_loop_1x
 909
 910
 911 .Lmac_end:
 912         cbnz            w5, .Lmac_ret
 913
 914         ld1             {v0.16b}, [x2], #16
 915         eor             RMAC.16b, RMAC.16b, v0.16b
 916
 917 .Lmac_ret:
 918         st1             {RMAC.16b}, [x1]
 919         ret
 920 SYM_FUNC_END(sm4_ce_mac_update)
 921
 922
 923         .section        ".rodata", "a"
 924         .align 4
 925 .Lbswap128_mask:
 926         .byte           0x0c, 0x0d, 0x0e, 0x0f, 0x08, 0x09, 0x0a, 0x0b
 927         .byte           0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03
 928
 929 .Lcts_permute_table:
 930         .byte           0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 931         .byte           0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 932         .byte            0x0,  0x1,  0x2,  0x3,  0x4,  0x5,  0x6,  0x7
 933         .byte            0x8,  0x9,  0xa,  0xb,  0xc,  0xd,  0xe,  0xf
 934         .byte           0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 935         .byte           0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff