gcc/config/arm/crypto.md

   1 ;; ARMv8-A crypto patterns.
   2 ;; Copyright (C) 2013-2025 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4
   5 ;; This file is part of GCC.
   6
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published
   9 ;; by the Free Software Foundation; either version 3, or (at your
  10 ;; option) any later version.
  11
  12 ;; GCC is distributed in the hope that it will be useful, but WITHOUT
  13 ;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  14 ;; or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
  15 ;; License for more details.
  16
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21
  22 (define_insn "crypto_<CRYPTO_AESMC:crypto_pattern>"
  23   [(set (match_operand:<crypto_mode> 0 "register_operand" "=w")
  24         (unspec:<crypto_mode>
  25          [(match_operand:<crypto_mode> 1 "register_operand" "w")]
  26          CRYPTO_AESMC))]
  27   "TARGET_CRYPTO"
  28   "<crypto_pattern>.<crypto_size_sfx>\\t%q0, %q1"
  29   [(set_attr "type" "<crypto_type>")]
  30 )
  31
  32 (define_expand "crypto_<CRYPTO_AES:crypto_pattern>"
  33   [(set (match_operand:<crypto_mode> 0 "register_operand" "=w")
  34         (unspec:<crypto_mode>
  35                 [(xor:<crypto_mode>
  36                      (match_operand:<crypto_mode> 1 "register_operand" "%0")
  37                      (match_operand:<crypto_mode> 2 "register_operand" "w"))]
  38         CRYPTO_AES))]
  39   "TARGET_CRYPTO"
  40 {
  41   if (fix_aes_erratum_1742098)
  42     {
  43       rtx op1_protect = gen_reg_rtx (V16QImode);
  44       emit_insn (gen_aes_op_protect (op1_protect, operands[1]));
  45       operands[1] = op1_protect;
  46       rtx op2_protect = gen_reg_rtx (V16QImode);
  47       emit_insn (gen_aes_op_protect (op2_protect, operands[2]));
  48       operands[2] = op2_protect;
  49     }
  50   /* Fall through to default expansion.  */
  51 })
  52
  53 (define_insn "*crypto_<CRYPTO_AES:crypto_pattern>_insn"
  54   [(set (match_operand:<crypto_mode> 0 "register_operand" "=w")
  55         (unspec:<crypto_mode>
  56          [(xor:<crypto_mode>
  57            (match_operand:<crypto_mode> 1 "register_operand" "%0")
  58            (match_operand:<crypto_mode> 2 "register_operand" "w"))]
  59          CRYPTO_AES))]
  60   "TARGET_CRYPTO"
  61   "<crypto_pattern>.<crypto_size_sfx>\\t%q0, %q2"
  62   [(set_attr "type" "<crypto_type>")]
  63 )
  64
  65 ;; Mitigate against AES erratum on Cortex-A57 and Cortex-A72 by
  66 ;; performing a 128-bit operation on an operand producer.  This can be
  67 ;; eliminated only if we know that the operand was produced by a
  68 ;; full-width operation.  V16QImode matches <crypto_mode> for the AES
  69 ;; instructions.  Handle some very common cases where the source is
  70 ;; known to be safe (transfers from core registers and memory).
  71 (define_insn "aes_op_protect"
  72   [(set (match_operand:V16QI 0 "register_operand" "=w,w,w")
  73         (unspec:V16QI [(match_operand:V16QI 1 "general_operand" "w,r,Uni")]
  74          UNSPEC_AES_PROTECT))]
  75   "TARGET_CRYPTO && fix_aes_erratum_1742098"
  76   {
  77     switch (which_alternative)
  78       {
  79       case 0: return "vmov\t%q0, %q1";
  80       case 1: return "vmov\t%e0, %Q1, %R1  @ V16QI\;vmov\t%f0, %J1, %K1";
  81       case 2: return output_move_neon (operands);
  82       default: gcc_unreachable ();
  83       }
  84   }
  85   [(set_attr "type" "neon_move_q,neon_from_gp_q,neon_load1_4reg")
  86    (set_attr "length" "4,8,8")
  87    (set_attr "arm_pool_range" "*,*,1020")
  88    (set_attr "thumb2_pool_range" "*,*,1018")
  89    (set_attr "neg_pool_range" "*,*,996")]
  90 )
  91
  92 ;; Another safe case is when a movmisalign load is used as the source.
  93 (define_insn "*aes_op_protect_misalign_load"
  94   [(set (match_operand:V16QI 0 "s_register_operand" "=w")
  95         (unspec:V16QI
  96          [(unspec:V16QI
  97            [(match_operand:V16QI 1 "neon_permissive_struct_operand" "Um")]
  98            UNSPEC_MISALIGNED_ACCESS)]
  99          UNSPEC_AES_PROTECT))]
 100   "TARGET_CRYPTO && fix_aes_erratum_1742098"
 101   "vld1.8\t%{q0}, %A1"
 102   [(set_attr "type" "neon_load1_1reg_q")]
 103 )
 104
 105 ;; Similarly for the vld1 intrinsic
 106 (define_insn "aes_op_protect_neon_vld1v16qi"
 107   [(set (match_operand:V16QI 0 "s_register_operand" "=w")
 108         (unspec:V16QI
 109          [(unspec:V16QI [(match_operand:V16QI 1 "neon_struct_operand" "Um")]
 110            UNSPEC_VLD1)]
 111          UNSPEC_AES_PROTECT))]
 112   "TARGET_NEON"
 113   "vld1.8\t%h0, %A1"
 114   [(set_attr "type" "neon_load1_1reg_q")]
 115 )
 116
 117 ;; An AESMC operation can feed directly into a subsequent AES
 118 ;; operation without needing mitigation.
 119 (define_insn "*crypto_<CRYPTO_AESMC:crypto_pattern>_protected"
 120   [(set (match_operand:<crypto_mode> 0 "register_operand" "=w")
 121         (unspec:<crypto_mode>
 122          [(unspec:<crypto_mode>
 123            [(match_operand:<crypto_mode> 1 "register_operand" "w")]
 124            CRYPTO_AESMC)]
 125          UNSPEC_AES_PROTECT))]
 126   "TARGET_CRYPTO && fix_aes_erratum_1742098"
 127   "<crypto_pattern>.<crypto_size_sfx>\\t%q0, %q1"
 128   [(set_attr "type" "<crypto_type>")]
 129 )
 130
 131 ;; When AESE/AESMC fusion is enabled we really want to keep the two together
 132 ;; and enforce the register dependency without scheduling or register
 133 ;; allocation messing up the order or introducing moves inbetween.
 134 ;; Mash the two together during combine.
 135
 136 (define_insn "*aarch32_crypto_aese_fused"
 137   [(set (match_operand:V16QI 0 "register_operand" "=w")
 138         (unspec:V16QI
 139          [(unspec:V16QI [(xor:V16QI
 140                           (match_operand:V16QI 1 "register_operand" "%0")
 141                           (match_operand:V16QI 2 "register_operand" "w"))]
 142            UNSPEC_AESE)]
 143          UNSPEC_AESMC))]
 144   "TARGET_CRYPTO
 145    && arm_fusion_enabled_p (tune_params::FUSE_AES_AESMC)"
 146   "aese.8\\t%q0, %q2\;aesmc.8\\t%q0, %q0"
 147   [(set_attr "type" "crypto_aese")
 148    (set_attr "length" "8")]
 149 )
 150
 151 ;; And similarly when mitigation is enabled, but not needed in this
 152 ;; case.
 153 (define_insn "*aarch32_crypto_aese_fused_protected"
 154   [(set (match_operand:V16QI 0 "register_operand" "=w")
 155         (unspec:V16QI
 156          [(unspec:V16QI
 157            [(unspec:V16QI [(xor:V16QI
 158                             (match_operand:V16QI 1 "register_operand" "%0")
 159                             (match_operand:V16QI 2 "register_operand" "w"))]
 160              UNSPEC_AESE)]
 161            UNSPEC_AESMC)]
 162          UNSPEC_AES_PROTECT))]
 163   "TARGET_CRYPTO && fix_aes_erratum_1742098
 164    && arm_fusion_enabled_p (tune_params::FUSE_AES_AESMC)"
 165   "aese.8\\t%q0, %q2\;aesmc.8\\t%q0, %q0"
 166   [(set_attr "type" "crypto_aese")
 167    (set_attr "length" "8")]
 168 )
 169
 170 ;; When AESD/AESIMC fusion is enabled we really want to keep the two together
 171 ;; and enforce the register dependency without scheduling or register
 172 ;; allocation messing up the order or introducing moves inbetween.
 173 ;; Mash the two together during combine.
 174
 175 (define_insn "*aarch32_crypto_aesd_fused"
 176   [(set (match_operand:V16QI 0 "register_operand" "=w")
 177         (unspec:V16QI
 178          [(unspec:V16QI [(xor:V16QI
 179                           (match_operand:V16QI 1 "register_operand" "%0")
 180                           (match_operand:V16QI 2 "register_operand" "w"))]
 181            UNSPEC_AESD)]
 182          UNSPEC_AESIMC))]
 183   "TARGET_CRYPTO
 184    && arm_fusion_enabled_p (tune_params::FUSE_AES_AESMC)"
 185   "aesd.8\\t%q0, %q2\;aesimc.8\\t%q0, %q0"
 186   [(set_attr "type" "crypto_aese")
 187    (set_attr "length" "8")]
 188 )
 189
 190 (define_insn "*aarch32_crypto_aesd_fused_protected"
 191   [(set (match_operand:V16QI 0 "register_operand" "=w")
 192         (unspec:V16QI
 193          [(unspec:V16QI
 194            [(unspec:V16QI [(xor:V16QI
 195                             (match_operand:V16QI 1 "register_operand" "%0")
 196                             (match_operand:V16QI 2 "register_operand" "w"))]
 197              UNSPEC_AESD)]
 198            UNSPEC_AESIMC)]
 199          UNSPEC_AES_PROTECT))]
 200   "TARGET_CRYPTO && fix_aes_erratum_1742098
 201    && arm_fusion_enabled_p (tune_params::FUSE_AES_AESMC)"
 202   "aesd.8\\t%q0, %q2\;aesimc.8\\t%q0, %q0"
 203   [(set_attr "type" "crypto_aese")
 204    (set_attr "length" "8")]
 205 )
 206
 207 (define_insn "crypto_<CRYPTO_BINARY:crypto_pattern>"
 208   [(set (match_operand:<crypto_mode> 0 "register_operand" "=w")
 209         (unspec:<crypto_mode>
 210          [(match_operand:<crypto_mode> 1 "register_operand" "0")
 211           (match_operand:<crypto_mode> 2 "register_operand" "w")]
 212          CRYPTO_BINARY))]
 213   "TARGET_CRYPTO"
 214   "<crypto_pattern>.<crypto_size_sfx>\\t%q0, %q2"
 215   [(set_attr "type" "<crypto_type>")]
 216 )
 217
 218 (define_insn "crypto_<CRYPTO_TERNARY:crypto_pattern>"
 219   [(set (match_operand:<crypto_mode> 0 "register_operand" "=w")
 220         (unspec:<crypto_mode>
 221          [(match_operand:<crypto_mode> 1 "register_operand" "0")
 222           (match_operand:<crypto_mode> 2 "register_operand" "w")
 223           (match_operand:<crypto_mode> 3 "register_operand" "w")]
 224          CRYPTO_TERNARY))]
 225   "TARGET_CRYPTO"
 226   "<crypto_pattern>.<crypto_size_sfx>\\t%q0, %q2, %q3"
 227   [(set_attr "type" "<crypto_type>")]
 228 )
 229
 230 ;; The vec_select operation always selects index 0 from the lower V2SI
 231 ;; subreg of the V4SI, adjusted for endianness. Required due to
 232 ;; neon_vget_lane and neon_set_lane that change the element ordering
 233 ;; in memory for big-endian.
 234
 235 (define_expand "crypto_sha1h"
 236   [(set (match_operand:V4SI 0 "register_operand")
 237         (match_operand:V4SI 1 "register_operand"))]
 238   "TARGET_CRYPTO"
 239 {
 240   rtx op2 = GEN_INT (NEON_ENDIAN_LANE_N (V2SImode, 0));
 241   emit_insn (gen_crypto_sha1h_lb (operands[0], operands[1], op2));
 242   DONE;
 243 })
 244
 245 (define_insn "crypto_sha1h_lb"
 246   [(set (match_operand:V4SI 0 "register_operand" "=w")
 247         (unspec:V4SI
 248          [(vec_select:SI
 249            (match_operand:V4SI 1 "register_operand" "w")
 250            (parallel [(match_operand:SI 2 "immediate_operand" "i")]))]
 251          UNSPEC_SHA1H))]
 252   "TARGET_CRYPTO && INTVAL (operands[2]) == NEON_ENDIAN_LANE_N (V2SImode, 0)"
 253   "sha1h.32\\t%q0, %q1"
 254   [(set_attr "type" "crypto_sha1_fast")]
 255 )
 256
 257 (define_insn "crypto_vmullp64"
 258   [(set (match_operand:TI 0 "register_operand" "=w")
 259         (unspec:TI [(match_operand:DI 1 "register_operand" "w")
 260                     (match_operand:DI 2 "register_operand" "w")]
 261          UNSPEC_VMULLP64))]
 262   "TARGET_CRYPTO"
 263   "vmull.p64\\t%q0, %P1, %P2"
 264   [(set_attr "type" "crypto_pmull")]
 265 )
 266
 267 /* The vec_select operation always selects index 0 from the lower V2SI subreg
 268    of the V4SI, adjusted for endianness. Required due to neon_vget_lane and
 269    neon_set_lane that change the element ordering in memory for big-endian.  */
 270
 271 (define_expand "crypto_<CRYPTO_SELECTING:crypto_pattern>"
 272   [(set (match_operand:V4SI 0 "register_operand")
 273         (unspec:<crypto_mode>
 274          [(match_operand:<crypto_mode> 1 "register_operand")
 275           (match_operand:<crypto_mode> 2 "register_operand")
 276           (match_operand:<crypto_mode> 3 "register_operand")]
 277          CRYPTO_SELECTING))]
 278   "TARGET_CRYPTO"
 279 {
 280   rtx op4 = GEN_INT (NEON_ENDIAN_LANE_N (V2SImode, 0));
 281   emit_insn (gen_crypto_<crypto_pattern>_lb
 282              (operands[0], operands[1], operands[2], operands[3], op4));
 283   DONE;
 284 })
 285
 286 (define_insn "crypto_<CRYPTO_SELECTING:crypto_pattern>_lb"
 287   [(set (match_operand:V4SI 0 "register_operand" "=w")
 288         (unspec:<crypto_mode>
 289          [(match_operand:<crypto_mode> 1 "register_operand" "0")
 290           (vec_select:SI
 291            (match_operand:<crypto_mode> 2 "register_operand" "w")
 292            (parallel [(match_operand:SI 4 "immediate_operand" "i")]))
 293           (match_operand:<crypto_mode> 3 "register_operand" "w")]
 294          CRYPTO_SELECTING))]
 295   "TARGET_CRYPTO && INTVAL (operands[4]) == NEON_ENDIAN_LANE_N (V2SImode, 0)"
 296   "<crypto_pattern>.<crypto_size_sfx>\\t%q0, %q2, %q3"
 297   [(set_attr "type" "<crypto_type>")]
 298 )