gcc/config/riscv/autovec.md

   1 ;; Machine description for auto-vectorization using RVV for GNU compiler.
   2 ;; Copyright (C) 2023-2025 Free Software Foundation, Inc.
   3 ;; Contributed by Juzhe Zhong (juzhe.zhong@rivai.ai), RiVAI Technologies Ltd.
   4
   5 ;; This file is part of GCC.
   6
   7 ;; GCC is free software; you can redistribute it and/or modify
   8 ;; it under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11
  12 ;; GCC is distributed in the hope that it will be useful,
  13 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 ;; GNU General Public License for more details.
  16
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; =========================================================================
  22 ;; == Loads/Stores
  23 ;; =========================================================================
  24
  25 (define_expand "mask_len_load<mode><vm>"
  26   [(match_operand:V 0 "register_operand")
  27    (match_operand:V 1 "memory_operand")
  28    (match_operand:<VM> 2 "vector_mask_operand")
  29    (match_operand:V 3 "maskload_else_operand")
  30    (match_operand 4 "autovec_length_operand")
  31    (match_operand 5 "const_0_operand")]
  32   "TARGET_VECTOR"
  33 {
  34   riscv_vector::expand_load_store (operands, true);
  35   DONE;
  36 })
  37
  38 (define_expand "mask_len_store<mode><vm>"
  39   [(match_operand:V 0 "memory_operand")
  40    (match_operand:V 1 "register_operand")
  41    (match_operand:<VM> 2 "vector_mask_operand")
  42    (match_operand 3 "autovec_length_operand")
  43    (match_operand 4 "const_0_operand")]
  44   "TARGET_VECTOR"
  45 {
  46   riscv_vector::expand_load_store (operands, false);
  47   DONE;
  48 })
  49
  50 ;; =========================================================================
  51 ;; == Gather Load
  52 ;; =========================================================================
  53
  54 (define_expand "mask_len_gather_load<RATIO64:mode><RATIO64I:mode>"
  55   [(match_operand:RATIO64 0 "register_operand")
  56    (match_operand 1 "pmode_reg_or_0_operand")
  57    (match_operand:RATIO64I 2 "register_operand")
  58    (match_operand 3 "<RATIO64:gs_extension>")
  59    (match_operand 4 "<RATIO64:gs_scale>")
  60    (match_operand:<RATIO64:VM> 5 "vector_mask_operand")
  61    (match_operand 6 "maskload_else_operand")
  62    (match_operand 7 "autovec_length_operand")
  63    (match_operand 8 "const_0_operand")]
  64   "TARGET_VECTOR && riscv_vector::gather_scatter_valid_offset_p (<RATIO64I:MODE>mode)"
  65 {
  66   riscv_vector::expand_gather_scatter (operands, true);
  67   DONE;
  68 })
  69
  70 (define_expand "mask_len_gather_load<RATIO32:mode><RATIO32I:mode>"
  71   [(match_operand:RATIO32 0 "register_operand")
  72    (match_operand 1 "pmode_reg_or_0_operand")
  73    (match_operand:RATIO32I 2 "register_operand")
  74    (match_operand 3 "<RATIO32:gs_extension>")
  75    (match_operand 4 "<RATIO32:gs_scale>")
  76    (match_operand:<RATIO32:VM> 5 "vector_mask_operand")
  77    (match_operand 6 "maskload_else_operand")
  78    (match_operand 7 "autovec_length_operand")
  79    (match_operand 8 "const_0_operand")]
  80   "TARGET_VECTOR && riscv_vector::gather_scatter_valid_offset_p (<RATIO32I:MODE>mode)"
  81 {
  82   riscv_vector::expand_gather_scatter (operands, true);
  83   DONE;
  84 })
  85
  86 (define_expand "mask_len_gather_load<RATIO16:mode><RATIO16I:mode>"
  87   [(match_operand:RATIO16 0 "register_operand")
  88    (match_operand 1 "pmode_reg_or_0_operand")
  89    (match_operand:RATIO16I 2 "register_operand")
  90    (match_operand 3 "<RATIO16:gs_extension>")
  91    (match_operand 4 "<RATIO16:gs_scale>")
  92    (match_operand:<RATIO16:VM> 5 "vector_mask_operand")
  93    (match_operand 6 "maskload_else_operand")
  94    (match_operand 7 "autovec_length_operand")
  95    (match_operand 8 "const_0_operand")]
  96   "TARGET_VECTOR && riscv_vector::gather_scatter_valid_offset_p (<RATIO16I:MODE>mode)"
  97 {
  98   riscv_vector::expand_gather_scatter (operands, true);
  99   DONE;
 100 })
 101
 102 (define_expand "mask_len_gather_load<RATIO8:mode><RATIO8I:mode>"
 103   [(match_operand:RATIO8 0 "register_operand")
 104    (match_operand 1 "pmode_reg_or_0_operand")
 105    (match_operand:RATIO8I 2 "register_operand")
 106    (match_operand 3 "<RATIO8:gs_extension>")
 107    (match_operand 4 "<RATIO8:gs_scale>")
 108    (match_operand:<RATIO8:VM> 5 "vector_mask_operand")
 109    (match_operand 6 "maskload_else_operand")
 110    (match_operand 7 "autovec_length_operand")
 111    (match_operand 8 "const_0_operand")]
 112   "TARGET_VECTOR && riscv_vector::gather_scatter_valid_offset_p (<RATIO8I:MODE>mode)"
 113 {
 114   riscv_vector::expand_gather_scatter (operands, true);
 115   DONE;
 116 })
 117
 118 (define_expand "mask_len_gather_load<RATIO4:mode><RATIO4I:mode>"
 119   [(match_operand:RATIO4 0 "register_operand")
 120    (match_operand 1 "pmode_reg_or_0_operand")
 121    (match_operand:RATIO4I 2 "register_operand")
 122    (match_operand 3 "<RATIO4:gs_extension>")
 123    (match_operand 4 "<RATIO4:gs_scale>")
 124    (match_operand:<RATIO4:VM> 5 "vector_mask_operand")
 125    (match_operand 6 "maskload_else_operand")
 126    (match_operand 7 "autovec_length_operand")
 127    (match_operand 8 "const_0_operand")]
 128   "TARGET_VECTOR && riscv_vector::gather_scatter_valid_offset_p (<RATIO4I:MODE>mode)"
 129 {
 130   riscv_vector::expand_gather_scatter (operands, true);
 131   DONE;
 132 })
 133
 134 (define_expand "mask_len_gather_load<RATIO2:mode><RATIO2I:mode>"
 135   [(match_operand:RATIO2 0 "register_operand")
 136    (match_operand 1 "pmode_reg_or_0_operand")
 137    (match_operand:RATIO2I 2 "register_operand")
 138    (match_operand 3 "<RATIO2:gs_extension>")
 139    (match_operand 4 "<RATIO2:gs_scale>")
 140    (match_operand:<RATIO2:VM> 5 "vector_mask_operand")
 141    (match_operand 6 "maskload_else_operand")
 142    (match_operand 7 "autovec_length_operand")
 143    (match_operand 8 "const_0_operand")]
 144   "TARGET_VECTOR && riscv_vector::gather_scatter_valid_offset_p (<RATIO2I:MODE>mode)"
 145 {
 146   riscv_vector::expand_gather_scatter (operands, true);
 147   DONE;
 148 })
 149
 150 ;; When SEW = 8 and LMUL = 8, we can't find any index mode with
 151 ;; larger SEW. Since RVV indexed load/store support zero extend
 152 ;; implicitly and not support scaling, we should only allow
 153 ;; operands[3] and operands[4] to be const_1_operand.
 154 (define_expand "mask_len_gather_load<mode><mode>"
 155   [(match_operand:RATIO1 0 "register_operand")
 156    (match_operand 1 "pmode_reg_or_0_operand")
 157    (match_operand:RATIO1 2 "register_operand")
 158    (match_operand 3 "<gs_extension>")
 159    (match_operand 4 "<gs_scale>")
 160    (match_operand:<VM> 5 "vector_mask_operand")
 161    (match_operand 6 "maskload_else_operand")
 162    (match_operand 7 "autovec_length_operand")
 163    (match_operand 8 "const_0_operand")]
 164   "TARGET_VECTOR"
 165 {
 166   riscv_vector::expand_gather_scatter (operands, true);
 167   DONE;
 168 })
 169
 170 ;; =========================================================================
 171 ;; == Scatter Store
 172 ;; =========================================================================
 173
 174 (define_expand "mask_len_scatter_store<RATIO64:mode><RATIO64I:mode>"
 175   [(match_operand 0 "pmode_reg_or_0_operand")
 176    (match_operand:RATIO64I 1 "register_operand")
 177    (match_operand 2 "<RATIO64:gs_extension>")
 178    (match_operand 3 "<RATIO64:gs_scale>")
 179    (match_operand:RATIO64 4 "register_operand")
 180    (match_operand:<RATIO64:VM> 5 "vector_mask_operand")
 181    (match_operand 6 "autovec_length_operand")
 182    (match_operand 7 "const_0_operand")]
 183   "TARGET_VECTOR && riscv_vector::gather_scatter_valid_offset_p (<RATIO64I:MODE>mode)"
 184 {
 185   riscv_vector::expand_gather_scatter (operands, false);
 186   DONE;
 187 })
 188
 189 (define_expand "mask_len_scatter_store<RATIO32:mode><RATIO32I:mode>"
 190   [(match_operand 0 "pmode_reg_or_0_operand")
 191    (match_operand:RATIO32I 1 "register_operand")
 192    (match_operand 2 "<RATIO32:gs_extension>")
 193    (match_operand 3 "<RATIO32:gs_scale>")
 194    (match_operand:RATIO32 4 "register_operand")
 195    (match_operand:<RATIO32:VM> 5 "vector_mask_operand")
 196    (match_operand 6 "autovec_length_operand")
 197    (match_operand 7 "const_0_operand")]
 198   "TARGET_VECTOR && riscv_vector::gather_scatter_valid_offset_p (<RATIO32I:MODE>mode)"
 199 {
 200   riscv_vector::expand_gather_scatter (operands, false);
 201   DONE;
 202 })
 203
 204 (define_expand "mask_len_scatter_store<RATIO16:mode><RATIO16I:mode>"
 205   [(match_operand 0 "pmode_reg_or_0_operand")
 206    (match_operand:RATIO16I 1 "register_operand")
 207    (match_operand 2 "<RATIO16:gs_extension>")
 208    (match_operand 3 "<RATIO16:gs_scale>")
 209    (match_operand:RATIO16 4 "register_operand")
 210    (match_operand:<RATIO16:VM> 5 "vector_mask_operand")
 211    (match_operand 6 "autovec_length_operand")
 212    (match_operand 7 "const_0_operand")]
 213   "TARGET_VECTOR && riscv_vector::gather_scatter_valid_offset_p (<RATIO16I:MODE>mode)"
 214 {
 215   riscv_vector::expand_gather_scatter (operands, false);
 216   DONE;
 217 })
 218
 219 (define_expand "mask_len_scatter_store<RATIO8:mode><RATIO8I:mode>"
 220   [(match_operand 0 "pmode_reg_or_0_operand")
 221    (match_operand:RATIO8I 1 "register_operand")
 222    (match_operand 2 "<RATIO8:gs_extension>")
 223    (match_operand 3 "<RATIO8:gs_scale>")
 224    (match_operand:RATIO8 4 "register_operand")
 225    (match_operand:<RATIO8:VM> 5 "vector_mask_operand")
 226    (match_operand 6 "autovec_length_operand")
 227    (match_operand 7 "const_0_operand")]
 228   "TARGET_VECTOR && riscv_vector::gather_scatter_valid_offset_p (<RATIO8I:MODE>mode)"
 229 {
 230   riscv_vector::expand_gather_scatter (operands, false);
 231   DONE;
 232 })
 233
 234 (define_expand "mask_len_scatter_store<RATIO4:mode><RATIO4I:mode>"
 235   [(match_operand 0 "pmode_reg_or_0_operand")
 236    (match_operand:RATIO4I 1 "register_operand")
 237    (match_operand 2 "<RATIO4:gs_extension>")
 238    (match_operand 3 "<RATIO4:gs_scale>")
 239    (match_operand:RATIO4 4 "register_operand")
 240    (match_operand:<RATIO4:VM> 5 "vector_mask_operand")
 241    (match_operand 6 "autovec_length_operand")
 242    (match_operand 7 "const_0_operand")]
 243   "TARGET_VECTOR && riscv_vector::gather_scatter_valid_offset_p (<RATIO4I:MODE>mode)"
 244 {
 245   riscv_vector::expand_gather_scatter (operands, false);
 246   DONE;
 247 })
 248
 249 (define_expand "mask_len_scatter_store<RATIO2:mode><RATIO2I:mode>"
 250   [(match_operand 0 "pmode_reg_or_0_operand")
 251    (match_operand:RATIO2I 1 "register_operand")
 252    (match_operand 2 "<RATIO2:gs_extension>")
 253    (match_operand 3 "<RATIO2:gs_scale>")
 254    (match_operand:RATIO2 4 "register_operand")
 255    (match_operand:<RATIO2:VM> 5 "vector_mask_operand")
 256    (match_operand 6 "autovec_length_operand")
 257    (match_operand 7 "const_0_operand")]
 258   "TARGET_VECTOR && riscv_vector::gather_scatter_valid_offset_p (<RATIO2I:MODE>mode)"
 259 {
 260   riscv_vector::expand_gather_scatter (operands, false);
 261   DONE;
 262 })
 263
 264 ;; When SEW = 8 and LMUL = 8, we can't find any index mode with
 265 ;; larger SEW. Since RVV indexed load/store support zero extend
 266 ;; implicitly and not support scaling, we should only allow
 267 ;; operands[3] and operands[4] to be const_1_operand.
 268 (define_expand "mask_len_scatter_store<mode><mode>"
 269   [(match_operand 0 "pmode_reg_or_0_operand")
 270    (match_operand:RATIO1 1 "register_operand")
 271    (match_operand 2 "<gs_extension>")
 272    (match_operand 3 "<gs_scale>")
 273    (match_operand:RATIO1 4 "register_operand")
 274    (match_operand:<VM> 5 "vector_mask_operand")
 275    (match_operand 6 "autovec_length_operand")
 276    (match_operand 7 "const_0_operand")]
 277   "TARGET_VECTOR"
 278 {
 279   riscv_vector::expand_gather_scatter (operands, false);
 280   DONE;
 281 })
 282
 283 ;; =========================================================================
 284 ;; == Array Load/Store
 285 ;; =========================================================================
 286
 287 (define_expand "vec_mask_len_load_lanes<mode><vsingle>"
 288   [(match_operand:VT 0 "register_operand")
 289    (match_operand:VT 1 "memory_operand")
 290    (match_operand:<VM> 2 "vector_mask_operand")
 291    (match_operand 3 "maskload_else_operand")
 292    (match_operand 4 "autovec_length_operand")
 293    (match_operand 5 "const_0_operand")]
 294   "TARGET_VECTOR_AUTOVEC_SEGMENT"
 295   {
 296     riscv_vector::expand_lanes_load_store (operands, true);
 297     DONE;
 298   }
 299 )
 300
 301 (define_expand "vec_mask_len_store_lanes<mode><vsingle>"
 302   [(match_operand:VT 0 "memory_operand")
 303    (match_operand:VT 1 "register_operand")
 304    (match_operand:<VM> 2 "vector_mask_operand")
 305    (match_operand 3 "autovec_length_operand")
 306    (match_operand 4 "const_0_operand")]
 307   "TARGET_VECTOR_AUTOVEC_SEGMENT"
 308   {
 309     riscv_vector::expand_lanes_load_store (operands, false);
 310     DONE;
 311   }
 312 )
 313
 314 ;; =========================================================================
 315 ;; == Vector creation
 316 ;; =========================================================================
 317
 318 ;; -------------------------------------------------------------------------
 319 ;; ---- [BOOL] Duplicate element
 320 ;; -------------------------------------------------------------------------
 321 ;; The patterns in this section are synthetic.
 322 ;; -------------------------------------------------------------------------
 323
 324 ;; Implement a predicate broadcast by shifting the low bit of the scalar
 325 ;; input into the top bit by duplicate the input and do a compare with zero.
 326 (define_expand "vec_duplicate<mode>"
 327   [(set (match_operand:VB 0 "register_operand")
 328         (vec_duplicate:VB (match_operand:QI 1 "register_operand")))]
 329   "TARGET_VECTOR"
 330   {
 331     poly_int64 nunits = GET_MODE_NUNITS (<MODE>mode);
 332     machine_mode mode = riscv_vector::get_vector_mode (QImode, nunits).require ();
 333     rtx dup = expand_vector_broadcast (mode, operands[1]);
 334     riscv_vector::expand_vec_cmp (operands[0], NE, dup, CONST0_RTX (mode));
 335     DONE;
 336   }
 337 )
 338
 339 ;; -------------------------------------------------------------------------
 340 ;; ---- [INT] Linear series
 341 ;; -------------------------------------------------------------------------
 342 ;; Includes:
 343 ;; - vid.v
 344 ;; - vmul.vx
 345 ;; - vadd.vx/vadd.vi
 346 ;; -------------------------------------------------------------------------
 347
 348 (define_expand "vec_series<mode>"
 349   [(match_operand:V_VLSI 0 "register_operand")
 350    (match_operand:<VEL> 1 "reg_or_int_operand")
 351    (match_operand:<VEL> 2 "reg_or_int_operand")]
 352   "TARGET_VECTOR"
 353   {
 354     riscv_vector::expand_vec_series (operands[0], operands[1], operands[2]);
 355     DONE;
 356   }
 357 )
 358
 359 ;; -------------------------------------------------------------------------
 360 ;; ---- [INT,FP] permutation
 361 ;; -------------------------------------------------------------------------
 362 ;; This is the pattern permutes the vector
 363 ;; -------------------------------------------------------------------------
 364
 365 (define_expand "vec_perm<mode>"
 366   [(match_operand:V_VLS 0 "register_operand")
 367    (match_operand:V_VLS 1 "register_operand")
 368    (match_operand:V_VLS 2 "register_operand")
 369    (match_operand:<VINDEX> 3 "vector_perm_operand")]
 370   "TARGET_VECTOR && GET_MODE_NUNITS (<MODE>mode).is_constant ()"
 371   {
 372     riscv_vector::expand_vec_perm (operands[0], operands[1],
 373                                    operands[2], operands[3]);
 374     DONE;
 375   }
 376 )
 377
 378 ;; -------------------------------------------------------------------------
 379 ;; ---- [INT,FP] Initialize from individual elements
 380 ;; -------------------------------------------------------------------------
 381 ;; This is the pattern initialize the vector
 382 ;; -------------------------------------------------------------------------
 383
 384 (define_expand "vec_init<mode><vel>"
 385   [(match_operand:V_VLSI 0 "register_operand")
 386    (match_operand 1 "")]
 387   "TARGET_VECTOR"
 388   {
 389     riscv_vector::expand_vec_init (operands[0], operands[1]);
 390     DONE;
 391   }
 392 )
 393
 394 ;; We split RVV floating-point because we are going to
 395 ;; use vfslide1down/vfslide1up for FP16 which need TARGET_ZVFH.
 396 (define_expand "vec_init<mode><vel>"
 397   [(match_operand:V_VLSF 0 "register_operand")
 398    (match_operand 1 "")]
 399   "TARGET_VECTOR"
 400   {
 401     riscv_vector::expand_vec_init (operands[0], operands[1]);
 402     DONE;
 403   }
 404 )
 405
 406 ;; Provide a vec_init for mask registers by initializing
 407 ;; a QImode vector and comparing it against 0.
 408 (define_expand "vec_init<mode>qi"
 409   [(match_operand:VB 0 "register_operand")
 410    (match_operand 1 "")]
 411   "TARGET_VECTOR"
 412   {
 413     machine_mode qimode = riscv_vector::get_vector_mode
 414         (QImode, GET_MODE_NUNITS (<MODE>mode)).require ();
 415     rtx tmp = gen_reg_rtx (qimode);
 416     riscv_vector::expand_vec_init (tmp, operands[1]);
 417     riscv_vector::expand_vec_cmp (operands[0], NE, tmp, CONST0_RTX (qimode));
 418     DONE;
 419   }
 420 )
 421
 422 ;; Slide an RVV vector left and insert a scalar into element 0.
 423 (define_expand "vec_shl_insert_<mode>"
 424   [(match_operand:VI 0 "register_operand")
 425    (match_operand:VI 1 "register_operand")
 426    (match_operand:<VEL> 2 "reg_or_0_operand")]
 427   "TARGET_VECTOR"
 428 {
 429   insn_code icode = code_for_pred_slide (UNSPEC_VSLIDE1UP, <MODE>mode);
 430   rtx ops[] = {operands[0], operands[1], operands[2]};
 431   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops);
 432   DONE;
 433 })
 434
 435 (define_expand "vec_shl_insert_<mode>"
 436   [(match_operand:VF 0 "register_operand")
 437    (match_operand:VF 1 "register_operand")
 438    (match_operand:<VEL> 2 "register_operand")]
 439   "TARGET_VECTOR"
 440 {
 441   insn_code icode = code_for_pred_slide (UNSPEC_VFSLIDE1UP, <MODE>mode);
 442   rtx ops[] = {operands[0], operands[1], operands[2]};
 443   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops);
 444   DONE;
 445 })
 446
 447 ;; ========================================================================
 448 ;; == Vector operations
 449 ;; =========================================================================
 450
 451 ;; -------------------------------------------------------------------------
 452 ;; ---- [INT] Binary operations
 453 ;; -------------------------------------------------------------------------
 454 ;; Includes:
 455 ;; - vadd.vv/vsub.vv/...
 456 ;; - vadd.vi/vsub.vi/...
 457 ;; -------------------------------------------------------------------------
 458
 459 (define_insn_and_split "<optab><mode>3"
 460   [(set (match_operand:V_VLSI 0 "register_operand")
 461     (any_int_binop_no_shift:V_VLSI
 462      (match_operand:V_VLSI 1 "<binop_rhs1_predicate>")
 463      (match_operand:V_VLSI 2 "<binop_rhs2_predicate>")))]
 464   "TARGET_VECTOR && can_create_pseudo_p ()"
 465   "#"
 466   "&& 1"
 467   [(const_int 0)]
 468 {
 469   riscv_vector::emit_vlmax_insn (code_for_pred (<CODE>, <MODE>mode),
 470                                  riscv_vector::BINARY_OP, operands);
 471   DONE;
 472 }
 473 [(set_attr "type" "vialu")])
 474
 475 ;; -------------------------------------------------------------------------
 476 ;; ---- [INT] Binary shifts by scalar.
 477 ;; -------------------------------------------------------------------------
 478 ;; Includes:
 479 ;; - vsll.vx/vsra.vx/vsrl.vx
 480 ;; - vsll.vi/vsra.vi/vsrl.vi
 481 ;; -------------------------------------------------------------------------
 482
 483 (define_insn_and_split "<optab><mode>3"
 484   [(set (match_operand:V_VLSI 0 "register_operand"        "=vr")
 485     (any_shift:V_VLSI
 486      (match_operand:V_VLSI 1 "register_operand"           " vr")
 487      (match_operand:<VEL> 2 "vector_scalar_shift_operand" " rK")))]
 488   "TARGET_VECTOR && can_create_pseudo_p ()"
 489   "#"
 490   "&& 1"
 491   [(const_int 0)]
 492 {
 493   operands[2] = gen_lowpart (Pmode, operands[2]);
 494   riscv_vector::emit_vlmax_insn (code_for_pred_scalar (<CODE>, <MODE>mode),
 495                                  riscv_vector::BINARY_OP, operands);
 496   DONE;
 497 }
 498  [(set_attr "type" "vshift")
 499   (set_attr "mode" "<MODE>")])
 500
 501 ;; -------------------------------------------------------------------------
 502 ;; ---- [INT] Binary shifts by vector.
 503 ;; -------------------------------------------------------------------------
 504 ;; Includes:
 505 ;; - vsll.vv/vsra.vv/vsrl.vv
 506 ;; -------------------------------------------------------------------------
 507
 508 (define_insn_and_split "v<optab><mode>3"
 509   [(set (match_operand:V_VLSI 0 "register_operand"  "=vr,vr")
 510     (any_shift:V_VLSI
 511      (match_operand:V_VLSI 1 "register_operand"     " vr,vr")
 512      (match_operand:V_VLSI 2 "vector_shift_operand" " vr,vk")))]
 513   "TARGET_VECTOR && can_create_pseudo_p ()"
 514   "#"
 515   "&& 1"
 516   [(const_int 0)]
 517 {
 518   riscv_vector::emit_vlmax_insn (code_for_pred (<CODE>, <MODE>mode),
 519                                  riscv_vector::BINARY_OP, operands);
 520   DONE;
 521 }
 522  [(set_attr "type" "vshift")
 523   (set_attr "mode" "<MODE>")])
 524
 525 ;; -------------------------------------------------------------------------
 526 ;; ---- [BOOL] Binary logical operations
 527 ;; -------------------------------------------------------------------------
 528 ;; Includes:
 529 ;; - vmand.mm
 530 ;; - vmxor.mm
 531 ;; - vmor.mm
 532 ;; -------------------------------------------------------------------------
 533
 534 (define_insn_and_split "<optab><mode>3"
 535   [(set (match_operand:VB_VLS 0 "register_operand"                 "=vr")
 536         (any_bitwise:VB_VLS (match_operand:VB_VLS 1 "register_operand" " vr")
 537                             (match_operand:VB_VLS 2 "register_operand" " vr")))]
 538   "TARGET_VECTOR && can_create_pseudo_p ()"
 539   "#"
 540   "&& 1"
 541   [(const_int 0)]
 542   {
 543     insn_code icode = code_for_pred (<CODE>, <MODE>mode);
 544     riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_MASK_OP, operands);
 545     DONE;
 546   }
 547   [(set_attr "type" "vmalu")
 548    (set_attr "mode" "<MODE>")])
 549
 550 ;; -------------------------------------------------------------------------
 551 ;; ---- [BOOL] Inverse
 552 ;; -------------------------------------------------------------------------
 553 ;; Includes:
 554 ;; - vmnot.m
 555 ;; -------------------------------------------------------------------------
 556
 557 (define_insn_and_split "one_cmpl<mode>2"
 558   [(set (match_operand:VB_VLS 0 "register_operand"         "=vr")
 559         (not:VB_VLS (match_operand:VB_VLS 1 "register_operand" " vr")))]
 560   "TARGET_VECTOR && can_create_pseudo_p ()"
 561   "#"
 562   "&& 1"
 563   [(const_int 0)]
 564   {
 565     insn_code icode = code_for_pred_not (<MODE>mode);
 566     riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_MASK_OP, operands);
 567     DONE;
 568   }
 569   [(set_attr "type" "vmalu")
 570    (set_attr "mode" "<MODE>")])
 571
 572 ;; =========================================================================
 573 ;; == Comparisons and selects
 574 ;; =========================================================================
 575
 576 ;; -------------------------------------------------------------------------
 577 ;; ---- [INT,FP] Select based on masks
 578 ;; -------------------------------------------------------------------------
 579 ;; Includes merging patterns for:
 580 ;; - vmerge.vv
 581 ;; - vmerge.vx
 582 ;; - vfmerge.vf
 583 ;; -------------------------------------------------------------------------
 584
 585 (define_insn_and_split "vcond_mask_<mode><vm>"
 586   [(set (match_operand:V_VLS 0 "register_operand")
 587         (if_then_else:V_VLS
 588           (match_operand:<VM> 3 "register_operand")
 589           (match_operand:V_VLS 1 "nonmemory_operand")
 590           (match_operand:V_VLS 2 "register_operand")))]
 591   "TARGET_VECTOR && can_create_pseudo_p ()"
 592   "#"
 593   "&& 1"
 594   [(const_int 0)]
 595   {
 596     /* The order of vcond_mask is opposite to pred_merge.  */
 597     std::swap (operands[1], operands[2]);
 598     riscv_vector::emit_vlmax_insn (code_for_pred_merge (<MODE>mode),
 599                                    riscv_vector::MERGE_OP, operands);
 600     DONE;
 601   }
 602   [(set_attr "type" "vector")]
 603 )
 604
 605 (define_expand "vcond_mask_len_<mode>"
 606   [(match_operand:V 0 "register_operand")
 607     (match_operand:<VM> 1 "nonmemory_operand")
 608     (match_operand:V 2 "nonmemory_operand")
 609     (match_operand:V 3 "autovec_else_operand")
 610     (match_operand 4 "autovec_length_operand")
 611     (match_operand 5 "const_0_operand")]
 612   "TARGET_VECTOR"
 613   {
 614     if (satisfies_constraint_Wc1 (operands[1]))
 615       riscv_vector::expand_cond_len_unop (code_for_pred_mov (<MODE>mode),
 616                                           operands);
 617     else
 618       {
 619         /* The order of then and else is opposite to pred_merge.  */
 620         rtx ops[] = {operands[0], operands[3], operands[3], operands[2],
 621                      operands[1]};
 622         riscv_vector::emit_nonvlmax_insn (code_for_pred_merge (<MODE>mode),
 623                                           riscv_vector::MERGE_OP_TU,
 624                                           ops, operands[4]);
 625       }
 626     DONE;
 627   }
 628   [(set_attr "type" "vector")]
 629 )
 630
 631 ;; -------------------------------------------------------------------------
 632 ;; ---- [BOOL] Select based on masks
 633 ;; -------------------------------------------------------------------------
 634 ;; Includes merging patterns for:
 635 ;; - vmand.mm
 636 ;; - vmor.mm
 637 ;; - vmnot.m
 638 ;; -------------------------------------------------------------------------
 639
 640 (define_expand "vcond_mask_<mode><mode>"
 641   [(match_operand:VB_VLS 0 "register_operand")
 642    (match_operand:VB_VLS 1 "register_operand")
 643    (match_operand:VB_VLS 2 "register_operand")
 644    (match_operand:VB_VLS 3 "register_operand")]
 645   "TARGET_VECTOR"
 646   {
 647     /* mask1 = operands[3] & operands[1].  */
 648     rtx mask1 = expand_binop (<MODE>mode, and_optab, operands[1],
 649                               operands[3], NULL_RTX, 0,
 650                               OPTAB_DIRECT);
 651     /* mask2 = ~operands[3] & operands[2].  */
 652     rtx inverse = expand_unop (<MODE>mode, one_cmpl_optab, operands[3],
 653                                NULL_RTX, 0);
 654     rtx mask2 = expand_binop (<MODE>mode, and_optab, operands[2],
 655                               inverse, NULL_RTX, 0,
 656                               OPTAB_DIRECT);
 657     /* result = mask1 | mask2.  */
 658     rtx result = expand_binop (<MODE>mode, ior_optab, mask1,
 659                                mask2, NULL_RTX, 0,
 660                                OPTAB_DIRECT);
 661     emit_move_insn (operands[0], result);
 662     DONE;
 663   })
 664
 665 ;; -------------------------------------------------------------------------
 666 ;; ---- [INT,FP] Comparisons
 667 ;; -------------------------------------------------------------------------
 668 ;; Includes:
 669 ;; - vms<eq/ne/ltu/lt/leu/le/gtu/gt>.<vv/vx/vi>
 670 ;; -------------------------------------------------------------------------
 671
 672 (define_expand "vec_cmp<mode><vm>"
 673   [(set (match_operand:<VM> 0 "register_operand")
 674         (match_operator:<VM> 1 "comparison_operator"
 675           [(match_operand:V_VLSI 2 "register_operand")
 676            (match_operand:V_VLSI 3 "nonmemory_operand")]))]
 677   "TARGET_VECTOR"
 678   {
 679     riscv_vector::expand_vec_cmp (operands[0], GET_CODE (operands[1]),
 680                                   operands[2], operands[3]);
 681     DONE;
 682   }
 683 )
 684
 685 (define_expand "vec_cmpu<mode><vm>"
 686   [(set (match_operand:<VM> 0 "register_operand")
 687         (match_operator:<VM> 1 "comparison_operator"
 688           [(match_operand:V_VLSI 2 "register_operand")
 689            (match_operand:V_VLSI 3 "nonmemory_operand")]))]
 690   "TARGET_VECTOR"
 691   {
 692     riscv_vector::expand_vec_cmp (operands[0], GET_CODE (operands[1]),
 693                                   operands[2], operands[3]);
 694     DONE;
 695   }
 696 )
 697
 698 (define_expand "vec_cmp<mode><vm>"
 699   [(set (match_operand:<VM> 0 "register_operand")
 700         (match_operator:<VM> 1 "comparison_operator"
 701           [(match_operand:V_VLSF 2 "register_operand")
 702            (match_operand:V_VLSF 3 "register_operand")]))]
 703   "TARGET_VECTOR"
 704   {
 705     riscv_vector::expand_vec_cmp_float (operands[0], GET_CODE (operands[1]),
 706                                         operands[2], operands[3], false);
 707     DONE;
 708   }
 709 )
 710
 711 ;; -------------------------------------------------------------------------
 712 ;; ---- [INT] Sign and zero extension
 713 ;; -------------------------------------------------------------------------
 714 ;; Includes:
 715 ;; - vzext.vf[2|4|8]
 716 ;; - vsext.vf[2|4|8]
 717 ;; -------------------------------------------------------------------------
 718
 719 ;; Use define_insn_and_split to define vsext.vf2/vzext.vf2 will help
 720 ;; to combine instructions as below:
 721 ;;   vsext.vf2 + vsext.vf2 + vadd.vv ==> vwadd.vv
 722 (define_insn_and_split "<optab><v_double_trunc><mode>2"
 723   [(set (match_operand:VWEXTI 0 "register_operand" "=&vr")
 724     (any_extend:VWEXTI
 725      (match_operand:<V_DOUBLE_TRUNC> 1 "register_operand" "vr")))]
 726   "TARGET_VECTOR && can_create_pseudo_p ()"
 727   "#"
 728   "&& 1"
 729   [(const_int 0)]
 730 {
 731   insn_code icode = code_for_pred_vf2 (<CODE>, <MODE>mode);
 732   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 733   DONE;
 734 }
 735   [(set_attr "type" "vext")
 736    (set_attr "mode" "<MODE>")])
 737
 738 (define_insn_and_split "<optab><v_quad_trunc><mode>2"
 739   [(set (match_operand:VQEXTI 0 "register_operand")
 740     (any_extend:VQEXTI
 741      (match_operand:<V_QUAD_TRUNC> 1 "register_operand")))]
 742   "TARGET_VECTOR && can_create_pseudo_p ()"
 743   "#"
 744   "&& 1"
 745   [(const_int 0)]
 746 {
 747   insn_code icode = code_for_pred_vf4 (<CODE>, <MODE>mode);
 748   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 749   DONE;
 750 }
 751 [(set_attr "type" "vext")])
 752
 753 (define_insn_and_split "<optab><v_oct_trunc><mode>2"
 754   [(set (match_operand:VOEXTI 0 "register_operand")
 755     (any_extend:VOEXTI
 756      (match_operand:<V_OCT_TRUNC> 1 "register_operand")))]
 757   "TARGET_VECTOR && can_create_pseudo_p ()"
 758   "#"
 759   "&& 1"
 760   [(const_int 0)]
 761 {
 762   insn_code icode = code_for_pred_vf8 (<CODE>, <MODE>mode);
 763   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 764   DONE;
 765 }
 766 [(set_attr "type" "vext")])
 767
 768 ;; -------------------------------------------------------------------------
 769 ;; ---- [INT] Truncation
 770 ;; -------------------------------------------------------------------------
 771 ;; - vncvt.x.x.w
 772 ;; -------------------------------------------------------------------------
 773 (define_insn_and_split "trunc<mode><v_double_trunc>2"
 774   [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand" "=vr")
 775     (truncate:<V_DOUBLE_TRUNC>
 776      (match_operand:VWEXTI 1 "register_operand"              " vr")))]
 777   "TARGET_VECTOR && can_create_pseudo_p ()"
 778   "#"
 779   "&& 1"
 780   [(const_int 0)]
 781 {
 782   insn_code icode = code_for_pred_trunc (<MODE>mode);
 783   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 784   DONE;
 785 }
 786   [(set_attr "type" "vshift")
 787    (set_attr "mode" "<MODE>")])
 788
 789 ;; -------------------------------------------------------------------------
 790 ;; Truncation to a mode whose inner mode size is a quarter of mode's.
 791 ;; We emulate this with two consecutive vncvts.
 792 ;; -------------------------------------------------------------------------
 793 (define_expand "trunc<mode><v_quad_trunc>2"
 794   [(set (match_operand:<V_QUAD_TRUNC> 0 "register_operand")
 795     (truncate:<V_QUAD_TRUNC>
 796      (match_operand:VQEXTI 1 "register_operand")))]
 797   "TARGET_VECTOR"
 798 {
 799   rtx half = gen_reg_rtx (<V_DOUBLE_TRUNC>mode);
 800   emit_insn (gen_trunc<mode><v_double_trunc>2 (half, operands[1]));
 801   emit_insn (gen_trunc<v_double_trunc><v_quad_trunc>2 (operands[0], half));
 802   DONE;
 803 })
 804
 805 ;; -------------------------------------------------------------------------
 806 ;; Truncation to a mode whose inner mode size is an eighth of mode's.
 807 ;; We emulate this with three consecutive vncvts.
 808 ;; -------------------------------------------------------------------------
 809 (define_expand "trunc<mode><v_oct_trunc>2"
 810   [(set (match_operand:<V_OCT_TRUNC> 0 "register_operand")
 811     (truncate:<V_OCT_TRUNC>
 812      (match_operand:VOEXTI 1 "register_operand")))]
 813   "TARGET_VECTOR"
 814 {
 815   rtx quarter = gen_reg_rtx (<V_QUAD_TRUNC>mode);
 816   emit_insn (gen_trunc<mode><v_quad_trunc>2 (quarter, operands[1]));
 817   emit_insn (gen_trunc<v_quad_trunc><v_oct_trunc>2 (operands[0], quarter));
 818   DONE;
 819 })
 820
 821 ;; -------------------------------------------------------------------------
 822 ;; ---- [FP] Widening.
 823 ;; -------------------------------------------------------------------------
 824 ;; - vfwcvt.f.f.v
 825 ;; -------------------------------------------------------------------------
 826 (define_insn_and_split "extend<v_double_trunc><mode>2"
 827   [(set (match_operand:VWEXTF_ZVFHMIN 0 "register_operand" "=&vr")
 828     (float_extend:VWEXTF_ZVFHMIN
 829      (match_operand:<V_DOUBLE_TRUNC>  1 "register_operand" "  vr")))]
 830   "TARGET_VECTOR && can_create_pseudo_p ()"
 831   "#"
 832   "&& 1"
 833   [(const_int 0)]
 834 {
 835   insn_code icode = code_for_pred_extend (<MODE>mode);
 836   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 837   DONE;
 838 }
 839   [(set_attr "type" "vfwcvtftof")
 840    (set_attr "mode" "<MODE>")])
 841
 842 (define_expand "extend<v_quad_trunc><mode>2"
 843   [(set (match_operand:VQEXTF 0 "register_operand")
 844     (float_extend:VQEXTF
 845      (match_operand:<V_QUAD_TRUNC> 1 "register_operand")))]
 846   "TARGET_VECTOR && (TARGET_ZVFHMIN || TARGET_ZVFH)"
 847 {
 848   rtx dblw = gen_reg_rtx (<V_DOUBLE_TRUNC>mode);
 849   emit_insn (gen_extend<v_quad_trunc><v_double_trunc>2 (dblw, operands[1]));
 850   emit_insn (gen_extend<v_double_trunc><mode>2 (operands[0], dblw));
 851   DONE;
 852 })
 853
 854 ;; -------------------------------------------------------------------------
 855 ;; ---- [FP] Narrowing.
 856 ;; -------------------------------------------------------------------------
 857 ;; - vfncvt.f.f.w
 858 ;; -------------------------------------------------------------------------
 859 (define_insn_and_split "trunc<mode><v_double_trunc>2"
 860   [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand" "=vr")
 861     (float_truncate:<V_DOUBLE_TRUNC>
 862      (match_operand:VWEXTF_ZVFHMIN 1 "register_operand"      " vr")))]
 863   "TARGET_VECTOR && can_create_pseudo_p ()"
 864   "#"
 865   "&& 1"
 866   [(const_int 0)]
 867 {
 868   insn_code icode = code_for_pred_trunc (<MODE>mode);
 869   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP_FRM_DYN, operands);
 870   DONE;
 871 }
 872   [(set_attr "type" "vfncvtftof")
 873    (set_attr "mode" "<MODE>")])
 874
 875 ;; -------------------------------------------------------------------------
 876 ;; Narrowing to a mode whose inner mode size is a quarter of mode's.
 877 ;; We emulate this with two consecutive vfncvts.
 878 ;; -------------------------------------------------------------------------
 879 (define_expand "trunc<mode><v_quad_trunc>2"
 880   [(set (match_operand:<V_QUAD_TRUNC> 0 "register_operand")
 881     (float_truncate:<V_QUAD_TRUNC>
 882      (match_operand:VQEXTF 1 "register_operand")))]
 883   "TARGET_VECTOR && (TARGET_ZVFHMIN || TARGET_ZVFH)"
 884 {
 885   rtx half = gen_reg_rtx (<V_DOUBLE_TRUNC>mode);
 886   rtx opshalf[] = {half, operands[1]};
 887
 888   /* According to the RISC-V V Spec 13.19. we need to use
 889      vfncvt.rod.f.f.w for all steps but the last.  */
 890   insn_code icode = code_for_pred_rod_trunc (<MODE>mode);
 891   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, opshalf);
 892
 893   emit_insn (gen_trunc<v_double_trunc><v_quad_trunc>2 (operands[0], half));
 894   DONE;
 895 })
 896
 897
 898 ;; =========================================================================
 899 ;; == Conversions
 900 ;; =========================================================================
 901
 902 ;; -------------------------------------------------------------------------
 903 ;; ---- [INT<-FP] Conversions
 904 ;; -------------------------------------------------------------------------
 905 ;; Includes:
 906 ;; - vfcvt.rtz.xu.f.v
 907 ;; - vfcvt.rtz.x.f.v
 908 ;; -------------------------------------------------------------------------
 909
 910 (define_insn_and_split "<optab><mode><vconvert>2"
 911   [(set (match_operand:<VCONVERT> 0 "register_operand")
 912         (any_fix:<VCONVERT>
 913           (match_operand:V_VLSF 1 "register_operand")))]
 914   "TARGET_VECTOR && can_create_pseudo_p ()"
 915   "#"
 916   "&& 1"
 917   [(const_int 0)]
 918 {
 919   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
 920   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 921   DONE;
 922 }
 923 [(set_attr "type" "vfcvtftoi")])
 924
 925 ;; -------------------------------------------------------------------------
 926 ;; ---- [FP<-INT] Conversions
 927 ;; -------------------------------------------------------------------------
 928 ;; Includes:
 929 ;; - vfcvt.f.xu.v
 930 ;; - vfcvt.f.x.v
 931 ;; -------------------------------------------------------------------------
 932
 933 (define_insn_and_split "<float_cvt><vconvert><mode>2"
 934   [(set (match_operand:V_VLSF 0 "register_operand")
 935         (any_float:V_VLSF
 936           (match_operand:<VCONVERT> 1 "register_operand")))]
 937   "TARGET_VECTOR && can_create_pseudo_p ()"
 938   "#"
 939   "&& 1"
 940   [(const_int 0)]
 941 {
 942   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
 943   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP_FRM_DYN, operands);
 944   DONE;
 945 }
 946 [(set_attr "type" "vfcvtitof")])
 947
 948 ;; =========================================================================
 949 ;; == Widening/narrowing Conversions
 950 ;; =========================================================================
 951
 952 ;; -------------------------------------------------------------------------
 953 ;; ---- [INT<-FP] Widening Conversions
 954 ;; -------------------------------------------------------------------------
 955 ;; Includes:
 956 ;; - vfwcvt.rtz.xu.f.v
 957 ;; - vfwcvt.rtz.x.f.v
 958 ;; -------------------------------------------------------------------------
 959 (define_insn_and_split "<optab><vnconvert><mode>2"
 960   [(set (match_operand:VWCONVERTI 0 "register_operand")
 961         (any_fix:VWCONVERTI
 962           (match_operand:<VNCONVERT> 1 "register_operand")))]
 963   "TARGET_VECTOR && can_create_pseudo_p ()"
 964   "#"
 965   "&& 1"
 966   [(const_int 0)]
 967 {
 968   insn_code icode = code_for_pred_widen (<CODE>, <MODE>mode);
 969   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 970   DONE;
 971 }
 972 [(set_attr "type" "vfwcvtftoi")])
 973
 974 ;; -------------------------------------------------------------------------
 975 ;; ---- [FP<-INT] Widening Conversions
 976 ;; -------------------------------------------------------------------------
 977 ;; Includes:
 978 ;; - vfwcvt.f.xu.v
 979 ;; - vfwcvt.f.x.v
 980 ;; -------------------------------------------------------------------------
 981 (define_insn_and_split "<float_cvt><vnconvert><mode>2"
 982   [(set (match_operand:V_VLSF 0 "register_operand")
 983         (any_float:V_VLSF
 984           (match_operand:<VNCONVERT> 1 "register_operand")))]
 985   "TARGET_VECTOR && can_create_pseudo_p ()"
 986   "#"
 987   "&& 1"
 988   [(const_int 0)]
 989 {
 990   insn_code icode = code_for_pred_widen (<CODE>, <MODE>mode);
 991   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 992   DONE;
 993 }
 994 [(set_attr "type" "vfwcvtitof")])
 995
 996 ;; -------------------------------------------------------------------------
 997 ;; ---- [INT<-FP] Narrowing Conversions
 998 ;; -------------------------------------------------------------------------
 999 ;; Includes:
1000 ;; - vfncvt.rtz.xu.f.v
1001 ;; - vfncvt.rtz.x.f.v
1002 ;; -------------------------------------------------------------------------
1003 (define_insn_and_split "<optab><mode><vnconvert>2"
1004   [(set (match_operand:<VNCONVERT> 0 "register_operand")
1005         (any_fix:<VNCONVERT>
1006           (match_operand:V_VLSF 1 "register_operand")))]
1007   "TARGET_VECTOR && can_create_pseudo_p ()"
1008   "#"
1009   "&& 1"
1010   [(const_int 0)]
1011 {
1012   insn_code icode = code_for_pred_narrow (<CODE>, <MODE>mode);
1013   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
1014   DONE;
1015 }
1016 [(set_attr "type" "vfncvtftoi")])
1017
1018 ;; -------------------------------------------------------------------------
1019 ;; ---- [FP<-INT] Narrowing Conversions
1020 ;; -------------------------------------------------------------------------
1021 ;; Includes:
1022 ;; - vfncvt.f.xu.w
1023 ;; - vfncvt.f.x.w
1024 ;; -------------------------------------------------------------------------
1025 (define_insn_and_split "<float_cvt><mode><vnconvert>2"
1026   [(set (match_operand:<VNCONVERT> 0 "register_operand")
1027         (any_float:<VNCONVERT>
1028           (match_operand:VWCONVERTI 1 "register_operand")))]
1029   "TARGET_VECTOR && can_create_pseudo_p ()"
1030   "#"
1031   "&& 1"
1032   [(const_int 0)]
1033 {
1034   insn_code icode = code_for_pred_narrow (<CODE>, <MODE>mode);
1035   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP_FRM_DYN, operands);
1036   DONE;
1037 }
1038 [(set_attr "type" "vfncvtitof")])
1039
1040 ;; This operation can be performed in the loop vectorizer but unfortunately
1041 ;; not applicable for now. We can remove this pattern after loop vectorizer
1042 ;; is able to take care of INT64 to FP16 conversion.
1043 (define_expand "<float_cvt><mode><vnnconvert>2"
1044   [(set (match_operand:<VNNCONVERT>  0 "register_operand")
1045         (any_float:<VNNCONVERT>
1046           (match_operand:VWWCONVERTI 1 "register_operand")))]
1047   "TARGET_VECTOR && TARGET_ZVFH && can_create_pseudo_p () && !flag_trapping_math"
1048   {
1049     rtx single = gen_reg_rtx (<VNCONVERT>mode); /* Get vector SF mode.  */
1050
1051     /* Step-1, INT64 => FP32.  */
1052     emit_insn (gen_<float_cvt><mode><vnconvert>2 (single, operands[1]));
1053     /* Step-2, FP32 => FP16.  */
1054     emit_insn (gen_trunc<vnconvert><vnnconvert>2 (operands[0], single));
1055
1056     DONE;
1057   })
1058
1059 ;; =========================================================================
1060 ;; == Unary arithmetic
1061 ;; =========================================================================
1062
1063 ;; -------------------------------------------------------------------------------
1064 ;; ---- [INT] Unary operations
1065 ;; -------------------------------------------------------------------------------
1066 ;; Includes:
1067 ;; - vneg.v/vnot.v
1068 ;; -------------------------------------------------------------------------------
1069 (define_insn_and_split "<optab><mode>2"
1070   [(set (match_operand:V_VLSI 0 "register_operand")
1071     (any_int_unop:V_VLSI
1072      (match_operand:V_VLSI 1 "register_operand")))]
1073   "TARGET_VECTOR && can_create_pseudo_p ()"
1074   "#"
1075   "&& 1"
1076   [(const_int 0)]
1077 {
1078   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1079   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
1080   DONE;
1081 }
1082 [(set_attr "type" "vialu")])
1083
1084 ;; -------------------------------------------------------------------------------
1085 ;; - [INT] ABS expansion to vneg and vmax.
1086 ;; -------------------------------------------------------------------------------
1087
1088 (define_expand "abs<mode>2"
1089   [(set (match_operand:V_VLSI 0 "register_operand")
1090     (smax:V_VLSI
1091      (match_dup 0)
1092      (neg:V_VLSI
1093        (match_operand:V_VLSI 1 "register_operand"))))]
1094   "TARGET_VECTOR"
1095 {
1096   DONE;
1097 })
1098
1099 ;; -------------------------------------------------------------------------------
1100 ;; ---- [FP] Unary operations
1101 ;; -------------------------------------------------------------------------------
1102 ;; Includes:
1103 ;; - vfneg.v/vfabs.v
1104 ;; -------------------------------------------------------------------------------
1105 (define_insn_and_split "<optab><mode>2"
1106   [(set (match_operand:V_VLSF 0 "register_operand")
1107     (any_float_unop_nofrm:V_VLSF
1108      (match_operand:V_VLSF 1 "register_operand")))]
1109   "TARGET_VECTOR && can_create_pseudo_p ()"
1110   "#"
1111   "&& 1"
1112   [(const_int 0)]
1113 {
1114   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1115   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
1116   DONE;
1117 }
1118 [(set_attr "type" "vector")])
1119
1120 ;; -------------------------------------------------------------------------------
1121 ;; - [FP] Square root
1122 ;; -------------------------------------------------------------------------------
1123 ;; Includes:
1124 ;; - vfsqrt.v
1125 ;; -------------------------------------------------------------------------------
1126 (define_insn_and_split "<optab><mode>2"
1127   [(set (match_operand:V_VLSF 0 "register_operand")
1128     (any_float_unop:V_VLSF
1129      (match_operand:V_VLSF 1 "register_operand")))]
1130   "TARGET_VECTOR && can_create_pseudo_p ()"
1131   "#"
1132   "&& 1"
1133   [(const_int 0)]
1134 {
1135   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1136   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP_FRM_DYN, operands);
1137   DONE;
1138 }
1139 [(set_attr "type" "vfsqrt")])
1140
1141 ;; =========================================================================
1142 ;; == Ternary arithmetic
1143 ;; =========================================================================
1144
1145 ;; -------------------------------------------------------------------------
1146 ;; ---- [INT] VMACC and VMADD
1147 ;; -------------------------------------------------------------------------
1148 ;; Includes:
1149 ;; - vmacc
1150 ;; - vmadd
1151 ;; -------------------------------------------------------------------------
1152
1153 (define_insn_and_split "fma<mode>4"
1154   [(set (match_operand:V_VLSI 0 "register_operand")
1155         (plus:V_VLSI
1156           (mult:V_VLSI
1157             (match_operand:V_VLSI 1 "register_operand")
1158             (match_operand:V_VLSI 2 "register_operand"))
1159           (match_operand:V_VLSI 3 "register_operand")))]
1160   "TARGET_VECTOR && can_create_pseudo_p ()"
1161   "#"
1162   "&& 1"
1163   [(const_int 0)]
1164   {
1165     rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
1166                  RVV_VUNDEF(<MODE>mode)};
1167     riscv_vector::emit_vlmax_insn (code_for_pred_mul_plus (<MODE>mode),
1168                                    riscv_vector::TERNARY_OP, ops);
1169     DONE;
1170   }
1171   [(set_attr "type" "vector")])
1172
1173 ;; -------------------------------------------------------------------------
1174 ;; ---- [INT] VNMSAC and VNMSUB
1175 ;; -------------------------------------------------------------------------
1176 ;; Includes:
1177 ;; - vnmsac
1178 ;; - vnmsub
1179 ;; -------------------------------------------------------------------------
1180
1181 (define_insn_and_split "fnma<mode>4"
1182   [(set (match_operand:V_VLSI 0 "register_operand")
1183         (minus:V_VLSI
1184           (match_operand:V_VLSI 3 "register_operand")
1185           (mult:V_VLSI
1186             (match_operand:V_VLSI 1 "register_operand")
1187             (match_operand:V_VLSI 2 "register_operand"))))]
1188   "TARGET_VECTOR && can_create_pseudo_p ()"
1189   "#"
1190   "&& 1"
1191   [(const_int 0)]
1192   {
1193     rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
1194                  RVV_VUNDEF(<MODE>mode)};
1195     riscv_vector::emit_vlmax_insn (code_for_pred_minus_mul (<MODE>mode),
1196                                    riscv_vector::TERNARY_OP, ops);
1197     DONE;
1198   }
1199   [(set_attr "type" "vector")])
1200
1201 ;; -------------------------------------------------------------------------
1202 ;; ---- [FP] VFMACC and VFMADD
1203 ;; -------------------------------------------------------------------------
1204 ;; Includes:
1205 ;; - vfmacc
1206 ;; - vfmadd
1207 ;; -------------------------------------------------------------------------
1208
1209 (define_insn_and_split "fma<mode>4"
1210   [(set (match_operand:V_VLSF 0 "register_operand")
1211         (plus:V_VLSF
1212           (mult:V_VLSF
1213             (match_operand:V_VLSF 1 "register_operand")
1214             (match_operand:V_VLSF 2 "register_operand"))
1215           (match_operand:V_VLSF 3 "register_operand")))]
1216   "TARGET_VECTOR && can_create_pseudo_p ()"
1217   "#"
1218   "&& 1"
1219   [(const_int 0)]
1220   {
1221     rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
1222                  RVV_VUNDEF(<MODE>mode)};
1223     riscv_vector::emit_vlmax_insn (code_for_pred_mul (PLUS, <MODE>mode),
1224                                    riscv_vector::TERNARY_OP_FRM_DYN, ops);
1225     DONE;
1226   }
1227   [(set_attr "type" "vector")])
1228
1229 ;; -------------------------------------------------------------------------
1230 ;; ---- [FP] VFNMSAC and VFNMSUB
1231 ;; -------------------------------------------------------------------------
1232 ;; Includes:
1233 ;; - vfnmsac
1234 ;; - vfnmsub
1235 ;; -------------------------------------------------------------------------
1236
1237 (define_insn_and_split "fnma<mode>4"
1238   [(set (match_operand:V_VLSF 0 "register_operand")
1239         (minus:V_VLSF
1240           (match_operand:V_VLSF 3 "register_operand")
1241           (mult:V_VLSF
1242             (match_operand:V_VLSF 1 "register_operand")
1243             (match_operand:V_VLSF 2 "register_operand"))))]
1244   "TARGET_VECTOR && can_create_pseudo_p ()"
1245   "#"
1246   "&& 1"
1247   [(const_int 0)]
1248   {
1249     rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
1250                  RVV_VUNDEF(<MODE>mode)};
1251     riscv_vector::emit_vlmax_insn (code_for_pred_mul_neg (PLUS, <MODE>mode),
1252                                    riscv_vector::TERNARY_OP_FRM_DYN, ops);
1253     DONE;
1254   }
1255   [(set_attr "type" "vector")])
1256
1257 ;; -------------------------------------------------------------------------
1258 ;; ---- [FP] VFMSAC and VFMSUB
1259 ;; -------------------------------------------------------------------------
1260 ;; Includes:
1261 ;; - vfmsac
1262 ;; - vfmsub
1263 ;; -------------------------------------------------------------------------
1264
1265 (define_insn_and_split "fms<mode>4"
1266   [(set (match_operand:V_VLSF 0 "register_operand")
1267         (minus:V_VLSF
1268           (mult:V_VLSF
1269             (match_operand:V_VLSF 1 "register_operand")
1270             (match_operand:V_VLSF 2 "register_operand"))
1271           (match_operand:V_VLSF 3 "register_operand")))]
1272   "TARGET_VECTOR && can_create_pseudo_p ()"
1273   "#"
1274   "&& 1"
1275   [(const_int 0)]
1276   {
1277     rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
1278                  RVV_VUNDEF(<MODE>mode)};
1279     riscv_vector::emit_vlmax_insn (code_for_pred_mul (MINUS, <MODE>mode),
1280                                    riscv_vector::TERNARY_OP_FRM_DYN, ops);
1281     DONE;
1282   }
1283   [(set_attr "type" "vector")])
1284
1285 ;; -------------------------------------------------------------------------
1286 ;; ---- [FP] VFNMACC and VFNMADD
1287 ;; -------------------------------------------------------------------------
1288 ;; Includes:
1289 ;; - vfnmacc
1290 ;; - vfnmadd
1291 ;; -------------------------------------------------------------------------
1292
1293 (define_insn_and_split "fnms<mode>4"
1294   [(set (match_operand:V_VLSF 0 "register_operand")
1295         (minus:V_VLSF
1296           (neg:V_VLSF
1297             (mult:V_VLSF
1298               (match_operand:V_VLSF 1 "register_operand")
1299               (match_operand:V_VLSF 2 "register_operand")))
1300           (match_operand:V_VLSF 3 "register_operand")))]
1301   "TARGET_VECTOR && can_create_pseudo_p ()"
1302   "#"
1303   "&& 1"
1304   [(const_int 0)]
1305   {
1306     rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
1307                  RVV_VUNDEF(<MODE>mode)};
1308     riscv_vector::emit_vlmax_insn (code_for_pred_mul_neg (MINUS, <MODE>mode),
1309                                    riscv_vector::TERNARY_OP_FRM_DYN, ops);
1310     DONE;
1311   }
1312   [(set_attr "type" "vector")])
1313
1314 ;; =========================================================================
1315 ;; == SELECT_VL
1316 ;; =========================================================================
1317
1318 (define_expand "select_vl<mode>"
1319   [(match_operand:P 0 "register_operand")
1320    (match_operand:P 1 "vector_length_operand")
1321    (match_operand:P 2 "")]
1322   "TARGET_VECTOR"
1323 {
1324   riscv_vector::expand_select_vl (operands);
1325   DONE;
1326 })
1327
1328 ;; -------------------------------------------------------------------------
1329 ;; ---- [INT,FP] Insert a vector element.
1330 ;; -------------------------------------------------------------------------
1331
1332 (define_expand "vec_set<mode>"
1333   [(match_operand:V_VLS 0 "register_operand")
1334    (match_operand:<VEL> 1 "register_operand")
1335    (match_operand       2 "nonmemory_operand")]
1336   "TARGET_VECTOR"
1337 {
1338   /* If we set the first element, emit an v(f)mv.s.[xf].  */
1339   if (operands[2] == const0_rtx)
1340     {
1341       rtx ops[] = {operands[0], operands[0], operands[1]};
1342       riscv_vector::emit_nonvlmax_insn (code_for_pred_broadcast (<MODE>mode),
1343                                         riscv_vector::SCALAR_MOVE_MERGED_OP_TU,
1344                                         ops, CONST1_RTX (Pmode));
1345     }
1346   else
1347     {
1348       /* Move the desired value into a vector register and insert
1349          it at the proper position using vslideup with an
1350          "effective length" of 1 i.e. a VL 1 past the offset.  */
1351
1352       /* Here we set VL = offset + 1.  */
1353       rtx length = gen_reg_rtx (Pmode);
1354       operands[2] = gen_lowpart (Pmode, operands[2]);
1355       if (CONST_INT_P (operands[2]))
1356           emit_move_insn (length, GEN_INT (INTVAL (operands[2]) + 1));
1357       else
1358         {
1359           rtx add = gen_rtx_PLUS (GET_MODE (operands[2]),
1360                                   operands[2], GEN_INT (1));
1361           emit_move_insn (length, add);
1362         }
1363
1364       /* Move operands[1] into a vector register via vmv.v.x using the same
1365          VL we need for the slide.  */
1366       rtx tmp = gen_reg_rtx (<MODE>mode);
1367       rtx ops1[] = {tmp, operands[1]};
1368       emit_nonvlmax_insn (code_for_pred_broadcast (<MODE>mode),
1369                            riscv_vector::UNARY_OP, ops1, length);
1370
1371       /* Slide exactly one element up leaving the tail elements
1372          unchanged.  */
1373       rtx ops2[] = {operands[0], operands[0], tmp, operands[2]};
1374       riscv_vector::emit_nonvlmax_insn
1375         (code_for_pred_slide (UNSPEC_VSLIDEUP, <MODE>mode), riscv_vector::BINARY_OP_TUMA, ops2, length);
1376     }
1377   DONE;
1378 })
1379
1380 ;; -------------------------------------------------------------------------
1381 ;; ---- [INT,FP] Extract a vector element.
1382 ;; -------------------------------------------------------------------------
1383 (define_expand "vec_extract<mode><vel>"
1384   [(set (match_operand:<VEL>      0 "register_operand")
1385      (vec_select:<VEL>
1386        (match_operand:V_VLS_ZVFH  1 "register_operand")
1387        (parallel
1388          [(match_operand          2 "nonmemory_operand")])))]
1389   "TARGET_VECTOR"
1390 {
1391   /* Element extraction can be done by sliding down the requested element
1392      to index 0 and then v(f)mv.[xf].s it to a scalar register.  */
1393
1394   /* When extracting any other than the first element we need to slide
1395      it down.  */
1396   rtx tmp = NULL_RTX;
1397   if (operands[2] != const0_rtx)
1398     {
1399       /* Properly convert a poly_int value and put the result into a
1400          register.  */
1401       if (CONST_POLY_INT_P (operands[2]))
1402         {
1403           rtx pos = gen_reg_rtx (Pmode);
1404           riscv_legitimize_poly_move (Pmode, pos, gen_reg_rtx (Pmode),
1405                                       operands[2]);
1406           operands[2] = pos;
1407         }
1408
1409     /* Emit the slide down to index 0 in a new vector.  */
1410     tmp = gen_reg_rtx (<MODE>mode);
1411     operands[2] = gen_lowpart (Pmode, operands[2]);
1412     rtx ops[] = {tmp, operands[1], operands[2]};
1413     riscv_vector::emit_vlmax_insn
1414       (code_for_pred_slide (UNSPEC_VSLIDEDOWN, <MODE>mode),
1415        riscv_vector::BINARY_OP, ops);
1416     }
1417
1418   /* Emit v(f)mv.[xf].s.  */
1419   emit_insn (gen_pred_extract_first (<MODE>mode, operands[0],
1420                                      tmp ? tmp : operands[1]));
1421   DONE;
1422 })
1423
1424 ;; -------------------------------------------------------------------------
1425 ;; This extracts a bit (via QImode) from a bitmask vector.
1426 ;; -------------------------------------------------------------------------
1427 (define_expand "vec_extract<mode>qi"
1428   [(set (match_operand:QI         0 "register_operand")
1429      (vec_select:QI
1430        (match_operand:VB_VLS      1 "register_operand")
1431        (parallel
1432          [(match_operand          2 "nonmemory_operand")])))]
1433   "TARGET_VECTOR"
1434 {
1435   /* Create an empty byte vector and set it to one under mask.  */
1436   machine_mode qimode = riscv_vector::get_vector_mode
1437       (QImode, GET_MODE_NUNITS (<MODE>mode)).require ();
1438
1439   rtx tmp1 = gen_reg_rtx (qimode);
1440   emit_move_insn (tmp1, gen_const_vec_duplicate (qimode, GEN_INT (0)));
1441   rtx ones = gen_const_vec_duplicate (qimode, GEN_INT (1));
1442
1443   rtx ops1[] = {tmp1, tmp1, ones, operands[1]};
1444   riscv_vector::emit_vlmax_insn (code_for_pred_merge (qimode),
1445                                  riscv_vector::MERGE_OP, ops1);
1446
1447   /* Extract from it.  */
1448   riscv_vector::emit_vec_extract (operands[0], tmp1, operands[2]);
1449   DONE;
1450 })
1451
1452 ;; Same for a BImode but still return a QImode.
1453 (define_expand "vec_extract<mode>bi"
1454   [(set (match_operand:QI         0 "register_operand")
1455      (vec_select:QI
1456        (match_operand:VB_VLS      1 "register_operand")
1457        (parallel
1458          [(match_operand          2 "nonmemory_operand")])))]
1459   "TARGET_VECTOR"
1460 {
1461   emit_insn (gen_vec_extract<mode>qi (operands[0], operands[1], operands[2]));
1462   DONE;
1463 })
1464
1465 ;; -------------------------------------------------------------------------
1466 ;; ---- [INT,FP] Extract a vector from a vector.
1467 ;; -------------------------------------------------------------------------
1468 ;; TODO: This can be extended to allow basically any extract mode.
1469 ;; For now this helps optimize VLS subregs like (subreg:V2DI (reg:V4DI) 16)
1470 ;; that would otherwise need to go via memory.
1471
1472 (define_expand "vec_extract<mode><vls_half>"
1473   [(set (match_operand:<VLS_HALF>        0 "nonimmediate_operand")
1474      (vec_select:<VLS_HALF>
1475        (match_operand:VLS_HAS_HALF       1 "register_operand")
1476        (parallel
1477          [(match_operand                 2 "immediate_operand")])))]
1478   "TARGET_VECTOR"
1479 {
1480   int sz = GET_MODE_NUNITS (<VLS_HALF>mode).to_constant ();
1481   int part = INTVAL (operands[2]);
1482
1483   rtx start = GEN_INT (part * sz);
1484   rtx tmp = operands[1];
1485
1486   if (part != 0)
1487     {
1488       tmp = gen_reg_rtx (<MODE>mode);
1489
1490       rtx ops[] = {tmp, operands[1], start};
1491       riscv_vector::emit_vlmax_insn
1492         (code_for_pred_slide (UNSPEC_VSLIDEDOWN, <MODE>mode),
1493          riscv_vector::BINARY_OP, ops);
1494     }
1495
1496   emit_move_insn (operands[0], gen_lowpart (<VLS_HALF>mode, tmp));
1497   DONE;
1498 })
1499
1500 (define_expand "vec_extract<mode><vls_quarter>"
1501   [(set (match_operand:<VLS_QUARTER>     0 "nonimmediate_operand")
1502      (vec_select:<VLS_QUARTER>
1503        (match_operand:VLS_HAS_QUARTER    1 "register_operand")
1504        (parallel
1505          [(match_operand                 2 "immediate_operand")])))]
1506   "TARGET_VECTOR"
1507 {
1508   int sz = GET_MODE_NUNITS (<VLS_QUARTER>mode).to_constant ();
1509   int part = INTVAL (operands[2]);
1510
1511   rtx start = GEN_INT (part * sz);
1512   rtx tmp = operands[1];
1513
1514   if (part != 0)
1515     {
1516       tmp = gen_reg_rtx (<MODE>mode);
1517
1518       rtx ops[] = {tmp, operands[1], start};
1519       riscv_vector::emit_vlmax_insn
1520         (code_for_pred_slide (UNSPEC_VSLIDEDOWN, <MODE>mode),
1521          riscv_vector::BINARY_OP, ops);
1522     }
1523
1524   emit_move_insn (operands[0], gen_lowpart (<VLS_QUARTER>mode, tmp));
1525   DONE;
1526 })
1527
1528 ;; -------------------------------------------------------------------------
1529 ;; ---- [FP] Binary operations
1530 ;; -------------------------------------------------------------------------
1531 ;; Includes:
1532 ;; - vfadd.vv/vfsub.vv/...
1533 ;; - vfadd.vf/vfsub.vf/...
1534 ;; -------------------------------------------------------------------------
1535 (define_insn_and_split "<optab><mode>3"
1536   [(set (match_operand:V_VLSF 0 "register_operand")
1537         (any_float_binop:V_VLSF
1538           (match_operand:V_VLSF 1 "register_operand")
1539           (match_operand:V_VLSF 2 "register_operand")))]
1540   "TARGET_VECTOR && can_create_pseudo_p ()"
1541   "#"
1542   "&& 1"
1543   [(const_int 0)]
1544 {
1545   riscv_vector::emit_vlmax_insn (code_for_pred (<CODE>, <MODE>mode),
1546                                     riscv_vector::BINARY_OP_FRM_DYN, operands);
1547   DONE;
1548 }
1549 [(set_attr "type" "vfalu")])
1550
1551 ;; -------------------------------------------------------------------------
1552 ;; Includes:
1553 ;; - vfmin.vv/vfmax.vv
1554 ;; - vfmin.vf/vfmax.vf
1555 ;; -------------------------------------------------------------------------
1556 (define_insn_and_split "<optab><mode>3"
1557   [(set (match_operand:V_VLSF 0 "register_operand")
1558         (any_float_binop_nofrm:V_VLSF
1559           (match_operand:V_VLSF 1 "register_operand")
1560           (match_operand:V_VLSF 2 "register_operand")))]
1561   "TARGET_VECTOR && can_create_pseudo_p ()"
1562   "#"
1563   "&& 1"
1564   [(const_int 0)]
1565 {
1566   riscv_vector::emit_vlmax_insn (code_for_pred (<CODE>, <MODE>mode),
1567                                   riscv_vector::BINARY_OP, operands);
1568   DONE;
1569 }
1570 [(set_attr "type" "vfminmax")])
1571
1572 (define_insn_and_split "<ieee_fmaxmin_op><mode>3"
1573   [(set (match_operand:V_VLSF 0 "register_operand")
1574       (unspec:V_VLSF
1575         [(match_operand:V_VLSF 1 "register_operand")
1576          (match_operand:V_VLSF 2 "register_operand")] UNSPEC_VFMAXMIN))]
1577   "TARGET_VECTOR && !HONOR_SNANS (<MODE>mode) && can_create_pseudo_p ()"
1578   "#"
1579   "&& 1"
1580   [(const_int 0)]
1581 {
1582   riscv_vector::emit_vlmax_insn (code_for_pred (<IEEE_FMAXMIN_OP>, <MODE>mode),
1583                                  riscv_vector::BINARY_OP, operands);
1584   DONE;
1585 }
1586 [(set_attr "type" "vfminmax")])
1587
1588 ;; -------------------------------------------------------------------------------
1589 ;; ---- [FP] Sign copying
1590 ;; -------------------------------------------------------------------------------
1591 ;; Includes:
1592 ;; - vfsgnj.vv/vfsgnjn.vv
1593 ;; - vfsgnj.vf/vfsgnjn.vf
1594 ;; -------------------------------------------------------------------------------
1595
1596 ;; Leave the pattern like this as to still allow combine to match
1597 ;; a negated copysign (see vector.md) before adding the UNSPEC_VPREDICATE later.
1598 (define_insn_and_split "copysign<mode>3"
1599   [(set (match_operand:V_VLSF 0 "register_operand"  "=vd, vd, vr, vr")
1600     (unspec:V_VLSF
1601      [(match_operand:V_VLSF 1 "register_operand"        " vr, vr, vr, vr")
1602      (match_operand:V_VLSF 2 "register_operand"         " vr, vr, vr, vr")] UNSPEC_VCOPYSIGN))]
1603   "TARGET_VECTOR && can_create_pseudo_p ()"
1604   "#"
1605   "&& 1"
1606   [(const_int 0)]
1607 {
1608   riscv_vector::emit_vlmax_insn (code_for_pred (UNSPEC_VCOPYSIGN, <MODE>mode),
1609                                   riscv_vector::BINARY_OP, operands);
1610   DONE;
1611 }
1612   [(set_attr "type" "vfsgnj")
1613    (set_attr "mode" "<MODE>")])
1614
1615 ;; -------------------------------------------------------------------------------
1616 ;; Includes:
1617 ;; - vfsgnjx.vv
1618 ;; - vfsgnjx.vf
1619 ;; -------------------------------------------------------------------------------
1620 (define_expand "xorsign<mode>3"
1621   [(match_operand:V_VLSF 0 "register_operand")
1622     (match_operand:V_VLSF 1 "register_operand")
1623     (match_operand:V_VLSF 2 "register_operand")]
1624   "TARGET_VECTOR"
1625 {
1626   riscv_vector::emit_vlmax_insn (code_for_pred (UNSPEC_VXORSIGN, <MODE>mode),
1627                                   riscv_vector::BINARY_OP, operands);
1628   DONE;
1629 })
1630
1631 ;; -------------------------------------------------------------------------------
1632 ;; - [INT] POPCOUNT, CTZ and CLZ.
1633 ;; -------------------------------------------------------------------------------
1634
1635 (define_expand "popcount<mode>2"
1636   [(match_operand:V_VLSI 0 "register_operand")
1637    (match_operand:V_VLSI 1 "register_operand")]
1638   "TARGET_VECTOR"
1639 {
1640   if (!TARGET_ZVBB)
1641     riscv_vector::expand_popcount (operands);
1642   else
1643     {
1644       riscv_vector::emit_vlmax_insn (code_for_pred_v (POPCOUNT, <MODE>mode),
1645                                      riscv_vector::CPOP_OP, operands);
1646     }
1647   DONE;
1648 })
1649
1650 (define_expand "ctz<mode>2"
1651   [(match_operand:V_VLSI 0 "register_operand")
1652    (match_operand:V_VLSI 1 "register_operand")]
1653   "TARGET_ZVBB"
1654   {
1655     riscv_vector::emit_vlmax_insn (code_for_pred_v (CTZ, <MODE>mode),
1656                                    riscv_vector::CPOP_OP, operands);
1657     DONE;
1658 })
1659
1660 (define_expand "clz<mode>2"
1661   [(match_operand:V_VLSI 0 "register_operand")
1662    (match_operand:V_VLSI 1 "register_operand")]
1663   "TARGET_ZVBB"
1664   {
1665     riscv_vector::emit_vlmax_insn (code_for_pred_v (CLZ, <MODE>mode),
1666                                    riscv_vector::CPOP_OP, operands);
1667     DONE;
1668 })
1669
1670
1671 ;; -------------------------------------------------------------------------
1672 ;; ---- [INT] Highpart multiplication
1673 ;; -------------------------------------------------------------------------
1674 ;; Includes:
1675 ;; - vmulh.vv
1676 ;; - vmulhu.vv
1677 ;; -------------------------------------------------------------------------
1678
1679 (define_insn_and_split "<mulh_table><mode>3_highpart"
1680   [(set (match_operand:VFULLI 0 "register_operand")
1681         (mulh:VFULLI
1682           (match_operand:VFULLI 1 "register_operand")
1683           (match_operand:VFULLI 2 "register_operand")))]
1684   "TARGET_VECTOR && can_create_pseudo_p ()"
1685   "#"
1686   "&& 1"
1687   [(const_int 0)]
1688 {
1689   insn_code icode = code_for_pred_mulh (<MULH_UNSPEC>, <MODE>mode);
1690   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, operands);
1691   DONE;
1692 }
1693 [(set_attr "type" "vimul")])
1694
1695 ;; -------------------------------------------------------------------------
1696 ;; ---- [INT] Conditional unary operations
1697 ;; -------------------------------------------------------------------------
1698 ;; Includes:
1699 ;; - vneg/vnot
1700 ;; -------------------------------------------------------------------------
1701
1702 (define_expand "cond_<optab><mode>"
1703   [(match_operand:V_VLSI 0 "register_operand")
1704    (match_operand:<VM> 1 "vector_mask_operand")
1705    (any_int_unop:V_VLSI
1706      (match_operand:V_VLSI 2 "register_operand"))
1707    (match_operand:V_VLSI 3 "autovec_else_operand")]
1708   "TARGET_VECTOR"
1709 {
1710   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1711   riscv_vector::expand_cond_unop (icode, operands);
1712   DONE;
1713 })
1714
1715 (define_expand "cond_len_<optab><mode>"
1716   [(match_operand:VI 0 "register_operand")
1717    (match_operand:<VM> 1 "vector_mask_operand")
1718    (any_int_unop:VI
1719      (match_operand:VI 2 "register_operand"))
1720    (match_operand:VI 3 "autovec_else_operand")
1721    (match_operand 4 "autovec_length_operand")
1722    (match_operand 5 "const_0_operand")]
1723   "TARGET_VECTOR"
1724 {
1725   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1726   riscv_vector::expand_cond_len_unop (icode, operands);
1727   DONE;
1728 })
1729
1730 ;; -------------------------------------------------------------------------
1731 ;; ---- [FP] Conditional unary operations
1732 ;; -------------------------------------------------------------------------
1733 ;; Includes:
1734 ;; - vfneg/vfabs
1735 ;; -------------------------------------------------------------------------
1736
1737 (define_expand "cond_<optab><mode>"
1738   [(match_operand:V_VLSF 0 "register_operand")
1739    (match_operand:<VM> 1 "vector_mask_operand")
1740    (any_float_unop_nofrm:V_VLSF
1741      (match_operand:V_VLSF 2 "register_operand"))
1742    (match_operand:V_VLSF 3 "autovec_else_operand")]
1743   "TARGET_VECTOR"
1744 {
1745   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1746   riscv_vector::expand_cond_unop (icode, operands);
1747   DONE;
1748 })
1749
1750 (define_expand "cond_len_<optab><mode>"
1751   [(match_operand:VF 0 "register_operand")
1752    (match_operand:<VM> 1 "vector_mask_operand")
1753    (any_float_unop_nofrm:VF
1754      (match_operand:VF 2 "register_operand"))
1755    (match_operand:VF 3 "autovec_else_operand")
1756    (match_operand 4 "autovec_length_operand")
1757    (match_operand 5 "const_0_operand")]
1758   "TARGET_VECTOR"
1759 {
1760   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1761   riscv_vector::expand_cond_len_unop (icode, operands);
1762   DONE;
1763 })
1764
1765 ;; -------------------------------------------------------------------------
1766 ;; ---- [INT] Conditional binary operations
1767 ;; -------------------------------------------------------------------------
1768 ;; Includes:
1769 ;; - vsra/vsrl/vsll
1770 ;; -------------------------------------------------------------------------
1771
1772 (define_expand "cond_<optab><mode>"
1773   [(match_operand:V_VLSI 0 "register_operand")
1774    (match_operand:<VM> 1 "vector_mask_operand")
1775    (any_shift:V_VLSI
1776      (match_operand:V_VLSI 2 "register_operand")
1777      (match_operand:V_VLSI 3 "vector_shift_operand"))
1778    (match_operand:V_VLSI 4 "autovec_else_operand")]
1779   "TARGET_VECTOR"
1780 {
1781   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1782   riscv_vector::expand_cond_binop (icode, operands);
1783   DONE;
1784 })
1785
1786 (define_expand "cond_len_<optab><mode>"
1787   [(match_operand:VI 0 "register_operand")
1788    (match_operand:<VM> 1 "vector_mask_operand")
1789    (any_shift:VI
1790      (match_operand:VI 2 "register_operand")
1791      (match_operand:VI 3 "vector_shift_operand"))
1792    (match_operand:VI 4 "autovec_else_operand")
1793    (match_operand 5 "autovec_length_operand")
1794    (match_operand 6 "const_0_operand")]
1795   "TARGET_VECTOR"
1796 {
1797   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1798   riscv_vector::expand_cond_len_binop (icode, operands);
1799   DONE;
1800 })
1801
1802 ;; -------------------------------------------------------------------------
1803 ;; ---- [INT] Conditional binary operations
1804 ;; -------------------------------------------------------------------------
1805 ;; Includes:
1806 ;; - vadd.vv/vsub.vv/...
1807 ;; - vadd.vi/vsub.vi/...
1808 ;; -------------------------------------------------------------------------
1809
1810 (define_expand "cond_<optab><mode>"
1811   [(match_operand:V_VLSI 0 "register_operand")
1812    (match_operand:<VM> 1 "vector_mask_operand")
1813    (any_int_binop_no_shift:V_VLSI
1814      (match_operand:V_VLSI 2 "<binop_rhs1_predicate>")
1815      (match_operand:V_VLSI 3 "<binop_rhs2_predicate>"))
1816    (match_operand:V_VLSI 4 "autovec_else_operand")]
1817   "TARGET_VECTOR"
1818 {
1819   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1820   riscv_vector::expand_cond_binop (icode, operands);
1821   DONE;
1822 })
1823
1824 (define_expand "cond_len_<optab><mode>"
1825   [(match_operand:VI 0 "register_operand")
1826    (match_operand:<VM> 1 "vector_mask_operand")
1827    (any_int_binop_no_shift:VI
1828      (match_operand:VI 2 "<binop_rhs1_predicate>")
1829      (match_operand:VI 3 "<binop_rhs2_predicate>"))
1830    (match_operand:VI 4 "autovec_else_operand")
1831    (match_operand 5 "autovec_length_operand")
1832    (match_operand 6 "const_0_operand")]
1833   "TARGET_VECTOR"
1834 {
1835   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1836   riscv_vector::expand_cond_len_binop (icode, operands);
1837   DONE;
1838 })
1839
1840 ;; -------------------------------------------------------------------------
1841 ;; ---- [FP] Conditional binary operations
1842 ;; -------------------------------------------------------------------------
1843 ;; Includes:
1844 ;; - vfadd.vv/vfsub.vv/...
1845 ;; - vfadd.vf/vfsub.vf/...
1846 ;; -------------------------------------------------------------------------
1847
1848 (define_expand "cond_<optab><mode>"
1849   [(match_operand:V_VLSF 0 "register_operand")
1850    (match_operand:<VM> 1 "vector_mask_operand")
1851    (any_float_binop:V_VLSF
1852      (match_operand:V_VLSF 2 "register_operand")
1853      (match_operand:V_VLSF 3 "register_operand"))
1854    (match_operand:V_VLSF 4 "autovec_else_operand")]
1855   "TARGET_VECTOR"
1856 {
1857   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1858   riscv_vector::expand_cond_binop (icode, operands);
1859   DONE;
1860 })
1861
1862 (define_expand "cond_len_<optab><mode>"
1863   [(match_operand:VF 0 "register_operand")
1864    (match_operand:<VM> 1 "vector_mask_operand")
1865    (any_float_binop:VF
1866      (match_operand:VF 2 "register_operand")
1867      (match_operand:VF 3 "register_operand"))
1868    (match_operand:VF 4 "autovec_else_operand")
1869    (match_operand 5 "autovec_length_operand")
1870    (match_operand 6 "const_0_operand")]
1871   "TARGET_VECTOR"
1872 {
1873   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1874   riscv_vector::expand_cond_len_binop (icode, operands);
1875   DONE;
1876 })
1877
1878 ;; -------------------------------------------------------------------------
1879 ;; Includes:
1880 ;; - vfmin.vv/vfmax.vv
1881 ;; - vfmin.vf/vfmax.vf
1882 ;; -------------------------------------------------------------------------
1883
1884 (define_expand "cond_<optab><mode>"
1885   [(match_operand:V_VLSF 0 "register_operand")
1886    (match_operand:<VM> 1 "vector_mask_operand")
1887    (any_float_binop_nofrm:V_VLSF
1888      (match_operand:V_VLSF 2 "register_operand")
1889      (match_operand:V_VLSF 3 "register_operand"))
1890    (match_operand:V_VLSF 4 "autovec_else_operand")]
1891   "TARGET_VECTOR"
1892 {
1893   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1894   riscv_vector::expand_cond_binop (icode, operands);
1895   DONE;
1896 })
1897
1898 (define_expand "cond_len_<optab><mode>"
1899   [(match_operand:VF 0 "register_operand")
1900    (match_operand:<VM> 1 "vector_mask_operand")
1901    (any_float_binop_nofrm:VF
1902      (match_operand:VF 2 "register_operand")
1903      (match_operand:VF 3 "register_operand"))
1904    (match_operand:VF 4 "autovec_else_operand")
1905    (match_operand 5 "autovec_length_operand")
1906    (match_operand 6 "const_0_operand")]
1907   "TARGET_VECTOR"
1908 {
1909   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1910   riscv_vector::expand_cond_len_binop (icode, operands);
1911   DONE;
1912 })
1913
1914 (define_expand "cond_<ieee_fmaxmin_op><mode>"
1915   [(match_operand:V_VLSF 0 "register_operand")
1916    (match_operand:<VM> 1 "vector_mask_operand")
1917    (unspec:V_VLSF
1918      [(match_operand:V_VLSF 2 "register_operand")
1919       (match_operand:V_VLSF 3 "register_operand")] UNSPEC_VFMAXMIN)
1920    (match_operand:V_VLSF 4 "autovec_else_operand")]
1921   "TARGET_VECTOR && !HONOR_SNANS (<MODE>mode)"
1922 {
1923   insn_code icode = code_for_pred (<IEEE_FMAXMIN_OP>, <MODE>mode);
1924   riscv_vector::expand_cond_binop (icode, operands);
1925   DONE;
1926 })
1927
1928 (define_expand "cond_len_<ieee_fmaxmin_op><mode>"
1929   [(match_operand:VF 0 "register_operand")
1930    (match_operand:<VM> 1 "vector_mask_operand")
1931    (unspec:VF
1932      [(match_operand:VF 2 "register_operand")
1933       (match_operand:VF 3 "register_operand")] UNSPEC_VFMAXMIN)
1934    (match_operand:VF 4 "autovec_else_operand")
1935    (match_operand 5 "autovec_length_operand")
1936    (match_operand 6 "const_0_operand")]
1937   "TARGET_VECTOR && !HONOR_SNANS (<MODE>mode)"
1938 {
1939   insn_code icode = code_for_pred (<IEEE_FMAXMIN_OP>, <MODE>mode);
1940   riscv_vector::expand_cond_len_binop (icode, operands);
1941   DONE;
1942 })
1943
1944 ;; -------------------------------------------------------------------------
1945 ;; ---- [FP] Conditional copysign operations
1946 ;; -------------------------------------------------------------------------
1947 ;; Includes:
1948 ;; - vfsgnj
1949 ;; -------------------------------------------------------------------------
1950
1951 (define_expand "cond_copysign<mode>"
1952   [(match_operand:V_VLSF 0 "register_operand")
1953    (match_operand:<VM> 1 "register_operand")
1954    (match_operand:V_VLSF 2 "register_operand")
1955    (match_operand:V_VLSF 3 "register_operand")
1956    (match_operand:V_VLSF 4 "register_operand")]
1957   "TARGET_VECTOR"
1958 {
1959   insn_code icode = code_for_pred (UNSPEC_VCOPYSIGN, <MODE>mode);
1960   rtx ops[] = {operands[0], operands[1], operands[2], operands[3], operands[4],
1961                gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
1962   riscv_vector::expand_cond_len_binop (icode, ops);
1963   DONE;
1964 })
1965
1966 ;; -------------------------------------------------------------------------
1967 ;; ---- [INT] Conditional ternary operations
1968 ;; -------------------------------------------------------------------------
1969 ;; Includes:
1970 ;; - vmacc/...
1971 ;; -------------------------------------------------------------------------
1972
1973 (define_expand "cond_fma<mode>"
1974   [(match_operand:V_VLSI 0 "register_operand")
1975    (match_operand:<VM> 1 "vector_mask_operand")
1976    (match_operand:V_VLSI 2 "register_operand")
1977    (match_operand:V_VLSI 3 "register_operand")
1978    (match_operand:V_VLSI 4 "register_operand")
1979    (match_operand:V_VLSI 5 "autovec_else_operand")]
1980   "TARGET_VECTOR"
1981 {
1982   insn_code icode = code_for_pred_mul_plus (<MODE>mode);
1983   riscv_vector::expand_cond_ternop (icode, operands);
1984   DONE;
1985 })
1986
1987 (define_expand "cond_len_fma<mode>"
1988   [(match_operand:VI 0 "register_operand")
1989    (match_operand:<VM> 1 "vector_mask_operand")
1990    (match_operand:VI 2 "register_operand")
1991    (match_operand:VI 3 "register_operand")
1992    (match_operand:VI 4 "register_operand")
1993    (match_operand:VI 5 "autovec_else_operand")
1994    (match_operand 6 "autovec_length_operand")
1995    (match_operand 7 "const_0_operand")]
1996   "TARGET_VECTOR"
1997 {
1998   insn_code icode = code_for_pred_mul_plus (<MODE>mode);
1999   riscv_vector::expand_cond_len_ternop (icode, operands);
2000   DONE;
2001 })
2002
2003 (define_expand "cond_fnma<mode>"
2004   [(match_operand:V_VLSI 0 "register_operand")
2005    (match_operand:<VM> 1 "vector_mask_operand")
2006    (match_operand:V_VLSI 2 "register_operand")
2007    (match_operand:V_VLSI 3 "register_operand")
2008    (match_operand:V_VLSI 4 "register_operand")
2009    (match_operand:V_VLSI 5 "autovec_else_operand")]
2010   "TARGET_VECTOR"
2011 {
2012   insn_code icode = code_for_pred_minus_mul (<MODE>mode);
2013   riscv_vector::expand_cond_ternop (icode, operands);
2014   DONE;
2015 })
2016
2017 (define_expand "cond_len_fnma<mode>"
2018   [(match_operand:VI 0 "register_operand")
2019    (match_operand:<VM> 1 "vector_mask_operand")
2020    (match_operand:VI 2 "register_operand")
2021    (match_operand:VI 3 "register_operand")
2022    (match_operand:VI 4 "register_operand")
2023    (match_operand:VI 5 "autovec_else_operand")
2024    (match_operand 6 "autovec_length_operand")
2025    (match_operand 7 "const_0_operand")]
2026   "TARGET_VECTOR"
2027 {
2028   insn_code icode = code_for_pred_minus_mul (<MODE>mode);
2029   riscv_vector::expand_cond_len_ternop (icode, operands);
2030   DONE;
2031 })
2032
2033 ;; -------------------------------------------------------------------------
2034 ;; ---- [FP] Conditional ternary operations
2035 ;; -------------------------------------------------------------------------
2036 ;; Includes:
2037 ;; - vfmacc/...
2038 ;; -------------------------------------------------------------------------
2039
2040 (define_expand "cond_fma<mode>"
2041   [(match_operand:V_VLSF 0 "register_operand")
2042    (match_operand:<VM> 1 "vector_mask_operand")
2043    (match_operand:V_VLSF 2 "register_operand")
2044    (match_operand:V_VLSF 3 "register_operand")
2045    (match_operand:V_VLSF 4 "register_operand")
2046    (match_operand:V_VLSF 5 "autovec_else_operand")]
2047   "TARGET_VECTOR"
2048 {
2049   insn_code icode = code_for_pred_mul (PLUS, <MODE>mode);
2050   riscv_vector::expand_cond_ternop (icode, operands);
2051   DONE;
2052 })
2053
2054 (define_expand "cond_len_fma<mode>"
2055   [(match_operand:VF 0 "register_operand")
2056    (match_operand:<VM> 1 "vector_mask_operand")
2057    (match_operand:VF 2 "register_operand")
2058    (match_operand:VF 3 "register_operand")
2059    (match_operand:VF 4 "register_operand")
2060    (match_operand:VF 5 "autovec_else_operand")
2061    (match_operand 6 "autovec_length_operand")
2062    (match_operand 7 "const_0_operand")]
2063   "TARGET_VECTOR"
2064 {
2065   insn_code icode = code_for_pred_mul (PLUS, <MODE>mode);
2066   riscv_vector::expand_cond_len_ternop (icode, operands);
2067   DONE;
2068 })
2069
2070 (define_expand "cond_fnma<mode>"
2071   [(match_operand:V_VLSF 0 "register_operand")
2072    (match_operand:<VM> 1 "vector_mask_operand")
2073    (match_operand:V_VLSF 2 "register_operand")
2074    (match_operand:V_VLSF 3 "register_operand")
2075    (match_operand:V_VLSF 4 "register_operand")
2076    (match_operand:V_VLSF 5 "autovec_else_operand")]
2077   "TARGET_VECTOR"
2078 {
2079   insn_code icode = code_for_pred_mul_neg (PLUS, <MODE>mode);
2080   riscv_vector::expand_cond_ternop (icode, operands);
2081   DONE;
2082 })
2083
2084 (define_expand "cond_len_fnma<mode>"
2085   [(match_operand:VF 0 "register_operand")
2086    (match_operand:<VM> 1 "vector_mask_operand")
2087    (match_operand:VF 2 "register_operand")
2088    (match_operand:VF 3 "register_operand")
2089    (match_operand:VF 4 "register_operand")
2090    (match_operand:VF 5 "autovec_else_operand")
2091    (match_operand 6 "autovec_length_operand")
2092    (match_operand 7 "const_0_operand")]
2093   "TARGET_VECTOR"
2094 {
2095   insn_code icode = code_for_pred_mul_neg (PLUS, <MODE>mode);
2096   riscv_vector::expand_cond_len_ternop (icode, operands);
2097   DONE;
2098 })
2099
2100 (define_expand "cond_fms<mode>"
2101   [(match_operand:V_VLSF 0 "register_operand")
2102    (match_operand:<VM> 1 "vector_mask_operand")
2103    (match_operand:V_VLSF 2 "register_operand")
2104    (match_operand:V_VLSF 3 "register_operand")
2105    (match_operand:V_VLSF 4 "register_operand")
2106    (match_operand:V_VLSF 5 "autovec_else_operand")]
2107   "TARGET_VECTOR"
2108 {
2109   insn_code icode = code_for_pred_mul (MINUS, <MODE>mode);
2110   riscv_vector::expand_cond_ternop (icode, operands);
2111   DONE;
2112 })
2113
2114 (define_expand "cond_len_fms<mode>"
2115   [(match_operand:VF 0 "register_operand")
2116    (match_operand:<VM> 1 "vector_mask_operand")
2117    (match_operand:VF 2 "register_operand")
2118    (match_operand:VF 3 "register_operand")
2119    (match_operand:VF 4 "register_operand")
2120    (match_operand:VF 5 "autovec_else_operand")
2121    (match_operand 6 "autovec_length_operand")
2122    (match_operand 7 "const_0_operand")]
2123   "TARGET_VECTOR"
2124 {
2125   insn_code icode = code_for_pred_mul (MINUS, <MODE>mode);
2126   riscv_vector::expand_cond_len_ternop (icode, operands);
2127   DONE;
2128 })
2129
2130 (define_expand "cond_fnms<mode>"
2131   [(match_operand:V_VLSF 0 "register_operand")
2132    (match_operand:<VM> 1 "vector_mask_operand")
2133    (match_operand:V_VLSF 2 "register_operand")
2134    (match_operand:V_VLSF 3 "register_operand")
2135    (match_operand:V_VLSF 4 "register_operand")
2136    (match_operand:V_VLSF 5 "autovec_else_operand")]
2137   "TARGET_VECTOR"
2138 {
2139   insn_code icode = code_for_pred_mul_neg (MINUS, <MODE>mode);
2140   riscv_vector::expand_cond_ternop (icode, operands);
2141   DONE;
2142 })
2143
2144 (define_expand "cond_len_fnms<mode>"
2145   [(match_operand:VF 0 "register_operand")
2146    (match_operand:<VM> 1 "vector_mask_operand")
2147    (match_operand:VF 2 "register_operand")
2148    (match_operand:VF 3 "register_operand")
2149    (match_operand:VF 4 "register_operand")
2150    (match_operand:VF 5 "autovec_else_operand")
2151    (match_operand 6 "autovec_length_operand")
2152    (match_operand 7 "const_0_operand")]
2153   "TARGET_VECTOR"
2154 {
2155   insn_code icode = code_for_pred_mul_neg (MINUS, <MODE>mode);
2156   riscv_vector::expand_cond_len_ternop (icode, operands);
2157   DONE;
2158 })
2159
2160 ;; =========================================================================
2161 ;; == Reductions
2162 ;; =========================================================================
2163
2164 ;; -------------------------------------------------------------------------
2165 ;; ---- [INT] Tree reductions
2166 ;; -------------------------------------------------------------------------
2167 ;; Includes:
2168 ;; - vredsum.vs
2169 ;; - vredmaxu.vs
2170 ;; - vredmax.vs
2171 ;; - vredminu.vs
2172 ;; - vredmin.vs
2173 ;; - vredand.vs
2174 ;; - vredor.vs
2175 ;; - vredxor.vs
2176 ;; -------------------------------------------------------------------------
2177
2178 (define_insn_and_split "reduc_plus_scal_<mode>"
2179   [(set (match_operand:<VEL> 0 "register_operand")
2180         (unspec:<VEL> [
2181              (match_operand:V_VLSI 1 "register_operand")
2182         ] UNSPEC_REDUC_SUM))]
2183   "TARGET_VECTOR && can_create_pseudo_p ()"
2184   "#"
2185   "&& 1"
2186   [(const_int 0)]
2187 {
2188   riscv_vector::expand_reduction (UNSPEC_REDUC_SUM,
2189                                   UNSPEC_REDUC_SUM_VL0_SAFE,
2190                                   riscv_vector::REDUCE_OP,
2191                                   operands, CONST0_RTX (<VEL>mode));
2192   DONE;
2193 }
2194 [(set_attr "type" "vector")])
2195
2196 (define_expand "reduc_smax_scal_<mode>"
2197   [(match_operand:<VEL> 0 "register_operand")
2198    (match_operand:V_VLSI 1 "register_operand")]
2199   "TARGET_VECTOR"
2200 {
2201   int prec = GET_MODE_PRECISION (<VEL>mode);
2202   rtx min = immed_wide_int_const (wi::min_value (prec, SIGNED), <VEL>mode);
2203   riscv_vector::expand_reduction (UNSPEC_REDUC_MAX,
2204                                   UNSPEC_REDUC_MAX_VL0_SAFE,
2205                                   riscv_vector::REDUCE_OP,
2206                                   operands, min);
2207   DONE;
2208 })
2209
2210 (define_expand "reduc_umax_scal_<mode>"
2211   [(match_operand:<VEL> 0 "register_operand")
2212    (match_operand:V_VLSI 1 "register_operand")]
2213   "TARGET_VECTOR"
2214 {
2215   riscv_vector::expand_reduction (UNSPEC_REDUC_MAXU,
2216                                   UNSPEC_REDUC_MAXU_VL0_SAFE,
2217                                   riscv_vector::REDUCE_OP,
2218                                   operands, CONST0_RTX (<VEL>mode));
2219   DONE;
2220 })
2221
2222 (define_expand "reduc_smin_scal_<mode>"
2223   [(match_operand:<VEL> 0 "register_operand")
2224    (match_operand:V_VLSI 1 "register_operand")]
2225   "TARGET_VECTOR"
2226 {
2227   int prec = GET_MODE_PRECISION (<VEL>mode);
2228   rtx max = immed_wide_int_const (wi::max_value (prec, SIGNED), <VEL>mode);
2229   riscv_vector::expand_reduction (UNSPEC_REDUC_MIN,
2230                                   UNSPEC_REDUC_MIN_VL0_SAFE,
2231                                   riscv_vector::REDUCE_OP,
2232                                   operands, max);
2233   DONE;
2234 })
2235
2236 (define_expand "reduc_umin_scal_<mode>"
2237   [(match_operand:<VEL> 0 "register_operand")
2238    (match_operand:V_VLSI 1 "register_operand")]
2239   "TARGET_VECTOR"
2240 {
2241   int prec = GET_MODE_PRECISION (<VEL>mode);
2242   rtx max = immed_wide_int_const (wi::max_value (prec, UNSIGNED), <VEL>mode);
2243   riscv_vector::expand_reduction (UNSPEC_REDUC_MINU,
2244                                   UNSPEC_REDUC_MINU_VL0_SAFE,
2245                                   riscv_vector::REDUCE_OP,
2246                                   operands, max);
2247   DONE;
2248 })
2249
2250 (define_expand "reduc_and_scal_<mode>"
2251   [(match_operand:<VEL> 0 "register_operand")
2252    (match_operand:V_VLSI 1 "register_operand")]
2253   "TARGET_VECTOR"
2254 {
2255   riscv_vector::expand_reduction (UNSPEC_REDUC_AND,
2256                                   UNSPEC_REDUC_AND_VL0_SAFE,
2257                                   riscv_vector::REDUCE_OP,
2258                                   operands, CONSTM1_RTX (<VEL>mode));
2259   DONE;
2260 })
2261
2262 (define_expand "reduc_ior_scal_<mode>"
2263   [(match_operand:<VEL> 0 "register_operand")
2264    (match_operand:V_VLSI 1 "register_operand")]
2265   "TARGET_VECTOR"
2266 {
2267   riscv_vector::expand_reduction (UNSPEC_REDUC_OR,
2268                                   UNSPEC_REDUC_OR_VL0_SAFE,
2269                                   riscv_vector::REDUCE_OP,
2270                                   operands, CONST0_RTX (<VEL>mode));
2271   DONE;
2272 })
2273
2274 (define_expand "reduc_xor_scal_<mode>"
2275   [(match_operand:<VEL> 0 "register_operand")
2276    (match_operand:V_VLSI 1 "register_operand")]
2277   "TARGET_VECTOR"
2278 {
2279   riscv_vector::expand_reduction (UNSPEC_REDUC_XOR,
2280                                   UNSPEC_REDUC_XOR_VL0_SAFE,
2281                                   riscv_vector::REDUCE_OP,
2282                                   operands, CONST0_RTX (<VEL>mode));
2283   DONE;
2284 })
2285
2286 ;; -------------------------------------------------------------------------
2287 ;; ---- [FP] Tree reductions
2288 ;; -------------------------------------------------------------------------
2289 ;; Includes:
2290 ;; - vfredusum.vs
2291 ;; - vfredmax.vs
2292 ;; - vfredmin.vs
2293 ;; -------------------------------------------------------------------------
2294
2295 (define_insn_and_split "reduc_plus_scal_<mode>"
2296   [(set (match_operand:<VEL> 0 "register_operand")
2297         (unspec:<VEL> [
2298              (match_operand:V_VLSF 1 "register_operand")
2299         ] UNSPEC_REDUC_SUM_UNORDERED))]
2300   "TARGET_VECTOR && can_create_pseudo_p ()"
2301   "#"
2302   "&& 1"
2303   [(const_int 0)]
2304 {
2305   riscv_vector::expand_reduction (UNSPEC_REDUC_SUM_UNORDERED,
2306                                   UNSPEC_REDUC_SUM_UNORDERED_VL0_SAFE,
2307                                   riscv_vector::REDUCE_OP_FRM_DYN,
2308                                   operands, CONST0_RTX (<VEL>mode));
2309   DONE;
2310 }
2311 [(set_attr "type" "vector")])
2312
2313 (define_expand "reduc_smax_scal_<mode>"
2314   [(match_operand:<VEL> 0 "register_operand")
2315    (match_operand:V_VLSF 1 "register_operand")]
2316   "TARGET_VECTOR"
2317 {
2318   REAL_VALUE_TYPE rv;
2319   real_inf (&rv, true);
2320   rtx f = const_double_from_real_value (rv, <VEL>mode);
2321   riscv_vector::expand_reduction (UNSPEC_REDUC_MAX,
2322                                   UNSPEC_REDUC_MAX_VL0_SAFE,
2323                                   riscv_vector::REDUCE_OP,
2324                                   operands, f);
2325   DONE;
2326 })
2327
2328 (define_expand "reduc_smin_scal_<mode>"
2329   [(match_operand:<VEL> 0 "register_operand")
2330    (match_operand:V_VLSF 1 "register_operand")]
2331   "TARGET_VECTOR"
2332 {
2333   REAL_VALUE_TYPE rv;
2334   real_inf (&rv, false);
2335   rtx f = const_double_from_real_value (rv, <VEL>mode);
2336   riscv_vector::expand_reduction (UNSPEC_REDUC_MIN,
2337                                   UNSPEC_REDUC_MIN_VL0_SAFE,
2338                                   riscv_vector::REDUCE_OP,
2339                                   operands, f);
2340   DONE;
2341 })
2342
2343 (define_expand "reduc_fmax_scal_<mode>"
2344   [(match_operand:<VEL> 0 "register_operand")
2345    (match_operand:V_VLSF 1 "register_operand")]
2346   "TARGET_VECTOR && !HONOR_SNANS (<MODE>mode)"
2347 {
2348   REAL_VALUE_TYPE rv;
2349   real_inf (&rv, true);
2350   rtx f = const_double_from_real_value (rv, <VEL>mode);
2351   riscv_vector::expand_reduction (UNSPEC_REDUC_MAX,
2352                                   UNSPEC_REDUC_MAX_VL0_SAFE,
2353                                   riscv_vector::REDUCE_OP,
2354                                   operands, f);
2355   DONE;
2356 })
2357
2358 (define_expand "reduc_fmin_scal_<mode>"
2359   [(match_operand:<VEL> 0 "register_operand")
2360    (match_operand:V_VLSF 1 "register_operand")]
2361   "TARGET_VECTOR && !HONOR_SNANS (<MODE>mode)"
2362 {
2363   REAL_VALUE_TYPE rv;
2364   real_inf (&rv, false);
2365   rtx f = const_double_from_real_value (rv, <VEL>mode);
2366   riscv_vector::expand_reduction (UNSPEC_REDUC_MIN,
2367                                   UNSPEC_REDUC_MIN_VL0_SAFE,
2368                                   riscv_vector::REDUCE_OP,
2369                                   operands, f);
2370   DONE;
2371 })
2372
2373 ;; -------------------------------------------------------------------------
2374 ;; ---- [FP] Left-to-right reductions
2375 ;; -------------------------------------------------------------------------
2376 ;; Includes:
2377 ;; - vfredosum.vs
2378 ;; -------------------------------------------------------------------------
2379
2380 ;; Unpredicated in-order FP reductions.
2381 (define_insn_and_split "fold_left_plus_<mode>"
2382   [(set (match_operand:<VEL> 0 "register_operand")
2383         (unspec:<VEL> [
2384              (match_operand:V_VLSF 2 "register_operand")
2385              (match_operand:<VEL> 1 "register_operand")
2386         ] UNSPEC_REDUC_SUM_ORDERED))]
2387   "TARGET_VECTOR && can_create_pseudo_p ()"
2388   "#"
2389   "&& 1"
2390   [(const_int 0)]
2391 {
2392   rtx ops[] = {operands[0], operands[2]};
2393   riscv_vector::expand_reduction (UNSPEC_REDUC_SUM_ORDERED,
2394                                   UNSPEC_REDUC_SUM_ORDERED_VL0_SAFE,
2395                                   riscv_vector::REDUCE_OP_FRM_DYN,
2396                                   ops, operands[1]);
2397   DONE;
2398 }
2399 [(set_attr "type" "vector")])
2400
2401 ;; Predicated in-order FP reductions.
2402 (define_insn_and_split "mask_len_fold_left_plus_<mode>"
2403   [(set (match_operand:<VEL> 0 "register_operand")
2404         (unspec:<VEL> [
2405           (match_operand:VF 2 "register_operand")
2406           (match_operand:<VEL> 1 "register_operand")
2407           (match_operand:<VM> 3 "vector_mask_operand")
2408           (match_operand 4 "autovec_length_operand")
2409           (match_operand 5 "const_0_operand")
2410         ] UNSPEC_REDUC_SUM_ORDERED))]
2411   "TARGET_VECTOR && can_create_pseudo_p ()"
2412   "#"
2413   "&& 1"
2414   [(const_int 0)]
2415 {
2416   if (rtx_equal_p (operands[4], const0_rtx))
2417     emit_move_insn (operands[0], operands[1]);
2418   else
2419     {
2420       rtx ops[] = {operands[0], operands[2], operands[3], operands[4]};
2421       riscv_vector::expand_reduction (UNSPEC_REDUC_SUM_ORDERED,
2422                                       UNSPEC_REDUC_SUM_ORDERED_VL0_SAFE,
2423                                       riscv_vector::REDUCE_OP_M_FRM_DYN,
2424                                       ops, operands[1]);
2425     }
2426   DONE;
2427 }
2428 [(set_attr "type" "vector")])
2429
2430 ;; -------------------------------------------------------------------------
2431 ;; ---- [INT,FP] Extract active element
2432 ;; -------------------------------------------------------------------------
2433 ;; Includes:
2434 ;; - vcompress.vm
2435 ;; - vcpop.m
2436 ;; - vslidedown.vx
2437 ;; - vmv.x.s
2438 ;; - vfmv.f.s
2439 ;; -------------------------------------------------------------------------
2440
2441 (define_expand "len_fold_extract_last_<mode>"
2442   [(match_operand:<VEL> 0 "register_operand")
2443    (match_operand:<VEL> 1 "register_operand")
2444    (match_operand:<VM> 2 "register_operand")
2445    (match_operand:V 3 "register_operand")
2446    (match_operand 4 "autovec_length_operand")
2447    (match_operand 5 "const_0_operand")]
2448   "TARGET_VECTOR"
2449   {
2450     riscv_vector::expand_fold_extract_last (operands);
2451     DONE;
2452   })
2453
2454 ;; -------------------------------------------------------------------------
2455 ;; ---- [INT] Average.
2456 ;; -------------------------------------------------------------------------
2457 ;; Implements the following "average" patterns:
2458 ;; floor:
2459 ;;  op[0] = (narrow) ((wide) op[1] + (wide) op[2]) >> 1;
2460 ;; ceil:
2461 ;;  op[0] = (narrow) ((wide) op[1] + (wide) op[2] + 1)) >> 1;
2462 ;; -------------------------------------------------------------------------
2463
2464 (define_expand "avg<v_double_trunc>3_floor"
2465  [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand")
2466    (truncate:<V_DOUBLE_TRUNC>
2467     (ashiftrt:VWEXTI
2468      (plus:VWEXTI
2469       (sign_extend:VWEXTI
2470        (match_operand:<V_DOUBLE_TRUNC> 1 "register_operand"))
2471       (sign_extend:VWEXTI
2472        (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))))))]
2473   "TARGET_VECTOR"
2474 {
2475   /* First emit a widening addition.  */
2476   rtx tmp1 = gen_reg_rtx (<MODE>mode);
2477   rtx ops1[] = {tmp1, operands[1], operands[2]};
2478   insn_code icode = code_for_pred_dual_widen (PLUS, SIGN_EXTEND, <MODE>mode);
2479   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops1);
2480
2481   /* Then a narrowing shift.  */
2482   rtx ops2[] = {operands[0], tmp1, const1_rtx};
2483   icode = code_for_pred_narrow_scalar (ASHIFTRT, <MODE>mode);
2484   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops2);
2485   DONE;
2486 })
2487
2488 (define_expand "avg<v_double_trunc>3_ceil"
2489  [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand")
2490    (truncate:<V_DOUBLE_TRUNC>
2491     (ashiftrt:VWEXTI
2492      (plus:VWEXTI
2493       (plus:VWEXTI
2494        (sign_extend:VWEXTI
2495         (match_operand:<V_DOUBLE_TRUNC> 1 "register_operand"))
2496        (sign_extend:VWEXTI
2497         (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand")))
2498       (const_int 1)))))]
2499   "TARGET_VECTOR"
2500 {
2501   /* First emit a widening addition.  */
2502   rtx tmp1 = gen_reg_rtx (<MODE>mode);
2503   rtx ops1[] = {tmp1, operands[1], operands[2]};
2504   insn_code icode = code_for_pred_dual_widen (PLUS, SIGN_EXTEND, <MODE>mode);
2505   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops1);
2506
2507   /* Then add 1.  */
2508   rtx tmp2 = gen_reg_rtx (<MODE>mode);
2509   rtx ops2[] = {tmp2, tmp1, const1_rtx};
2510   icode = code_for_pred_scalar (PLUS, <MODE>mode);
2511   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops2);
2512
2513   /* Finally, a narrowing shift.  */
2514   rtx ops3[] = {operands[0], tmp2, const1_rtx};
2515   icode = code_for_pred_narrow_scalar (ASHIFTRT, <MODE>mode);
2516   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops3);
2517   DONE;
2518 })
2519
2520 ;; csrwi vxrm, 2
2521 ;; vaaddu.vv vd, vs2, vs1
2522 (define_expand "uavg<mode>3_floor"
2523  [(match_operand:V_VLSI 0 "register_operand")
2524   (match_operand:V_VLSI 1 "register_operand")
2525   (match_operand:V_VLSI 2 "register_operand")]
2526   "TARGET_VECTOR"
2527 {
2528   insn_code icode = code_for_pred (UNSPEC_VAADDU, <MODE>mode);
2529   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP_VXRM_RDN, operands);
2530   DONE;
2531 })
2532
2533 ;; csrwi vxrm, 0
2534 ;; vaaddu.vv vd, vs2, vs1
2535 (define_expand "uavg<mode>3_ceil"
2536  [(match_operand:V_VLSI 0 "register_operand")
2537   (match_operand:V_VLSI 1 "register_operand")
2538   (match_operand:V_VLSI 2 "register_operand")]
2539   "TARGET_VECTOR"
2540 {
2541   insn_code icode = code_for_pred (UNSPEC_VAADDU, <MODE>mode);
2542   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP_VXRM_RNU, operands);
2543   DONE;
2544 })
2545
2546 ;; -------------------------------------------------------------------------
2547 ;; ---- [FP] Rounding.
2548 ;; -------------------------------------------------------------------------
2549 ;; Includes:
2550 ;; - ceil/ceilf
2551 ;; - floor/floorf
2552 ;; - nearbyint/nearbyintf
2553 ;; - rint/rintf
2554 ;; - round/roundf
2555 ;; - trunc/truncf
2556 ;; - roundeven/roundevenf
2557 ;; - lrint/lrintf
2558 ;; - irintf
2559 ;; - lceil/lceilf
2560 ;; - lfloor/lfloorf
2561 ;; -------------------------------------------------------------------------
2562 (define_expand "ceil<mode>2"
2563   [(match_operand:V_VLSF 0 "register_operand")
2564    (match_operand:V_VLSF 1 "register_operand")]
2565   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2566   {
2567     riscv_vector::expand_vec_ceil (operands[0], operands[1], <MODE>mode, <VCONVERT>mode);
2568     DONE;
2569   }
2570 )
2571
2572 (define_expand "floor<mode>2"
2573   [(match_operand:V_VLSF 0 "register_operand")
2574    (match_operand:V_VLSF 1 "register_operand")]
2575   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2576   {
2577     riscv_vector::expand_vec_floor (operands[0], operands[1], <MODE>mode, <VCONVERT>mode);
2578     DONE;
2579   }
2580 )
2581
2582 (define_expand "nearbyint<mode>2"
2583   [(match_operand:V_VLSF 0 "register_operand")
2584    (match_operand:V_VLSF 1 "register_operand")]
2585   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2586   {
2587     riscv_vector::expand_vec_nearbyint (operands[0], operands[1], <MODE>mode, <VCONVERT>mode);
2588     DONE;
2589   }
2590 )
2591
2592 (define_expand "rint<mode>2"
2593   [(match_operand:V_VLSF 0 "register_operand")
2594    (match_operand:V_VLSF 1 "register_operand")]
2595   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2596   {
2597     riscv_vector::expand_vec_rint (operands[0], operands[1], <MODE>mode, <VCONVERT>mode);
2598     DONE;
2599   }
2600 )
2601
2602 (define_expand "round<mode>2"
2603   [(match_operand:V_VLSF 0 "register_operand")
2604    (match_operand:V_VLSF 1 "register_operand")]
2605   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2606   {
2607     riscv_vector::expand_vec_round (operands[0], operands[1], <MODE>mode, <VCONVERT>mode);
2608     DONE;
2609   }
2610 )
2611
2612 (define_expand "btrunc<mode>2"
2613   [(match_operand:V_VLSF 0 "register_operand")
2614    (match_operand:V_VLSF 1 "register_operand")]
2615   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2616   {
2617     riscv_vector::expand_vec_trunc (operands[0], operands[1], <MODE>mode, <VCONVERT>mode);
2618     DONE;
2619   }
2620 )
2621
2622 (define_expand "roundeven<mode>2"
2623   [(match_operand:V_VLSF 0 "register_operand")
2624    (match_operand:V_VLSF 1 "register_operand")]
2625   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2626   {
2627     riscv_vector::expand_vec_roundeven (operands[0], operands[1], <MODE>mode, <VCONVERT>mode);
2628     DONE;
2629   }
2630 )
2631
2632 (define_expand "lrint<mode><v_f2si_convert>2"
2633   [(match_operand:<V_F2SI_CONVERT>   0 "register_operand")
2634    (match_operand:V_VLS_F_CONVERT_SI 1 "register_operand")]
2635   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2636   {
2637     riscv_vector::expand_vec_lrint (operands[0], operands[1], <MODE>mode,
2638                                     <V_F2SI_CONVERT>mode, VOIDmode);
2639     DONE;
2640   }
2641 )
2642
2643 (define_expand "lrint<mode><v_f2di_convert>2"
2644   [(match_operand:<V_F2DI_CONVERT>   0 "register_operand")
2645    (match_operand:V_VLS_F_CONVERT_DI 1 "register_operand")]
2646   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2647   {
2648     riscv_vector::expand_vec_lrint (operands[0], operands[1], <MODE>mode,
2649                                     <V_F2DI_CONVERT>mode,
2650                                     <V_F2DI_CONVERT_BRIDGE>mode);
2651     DONE;
2652   }
2653 )
2654
2655 (define_expand "lround<mode><v_f2si_convert>2"
2656   [(match_operand:<V_F2SI_CONVERT>   0 "register_operand")
2657    (match_operand:V_VLS_F_CONVERT_SI 1 "register_operand")]
2658   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2659   {
2660     riscv_vector::expand_vec_lround (operands[0], operands[1], <MODE>mode,
2661                                      <V_F2SI_CONVERT>mode, VOIDmode);
2662     DONE;
2663   }
2664 )
2665
2666 (define_expand "lround<mode><v_f2di_convert>2"
2667   [(match_operand:<V_F2DI_CONVERT>   0 "register_operand")
2668    (match_operand:V_VLS_F_CONVERT_DI 1 "register_operand")]
2669   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2670   {
2671     riscv_vector::expand_vec_lround (operands[0], operands[1], <MODE>mode,
2672                                      <V_F2DI_CONVERT>mode,
2673                                      <V_F2DI_CONVERT_BRIDGE>mode);
2674
2675     DONE;
2676   }
2677 )
2678
2679 (define_expand "lceil<mode><v_f2si_convert>2"
2680   [(match_operand:<V_F2SI_CONVERT>   0 "register_operand")
2681    (match_operand:V_VLS_F_CONVERT_SI 1 "register_operand")]
2682   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2683   {
2684     riscv_vector::expand_vec_lceil (operands[0], operands[1], <MODE>mode, <V_F2SI_CONVERT>mode);
2685     DONE;
2686   }
2687 )
2688
2689 (define_expand "lceil<mode><v_f2di_convert>2"
2690   [(match_operand:<V_F2DI_CONVERT>   0 "register_operand")
2691    (match_operand:V_VLS_F_CONVERT_DI 1 "register_operand")]
2692   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2693   {
2694     riscv_vector::expand_vec_lceil (operands[0], operands[1], <MODE>mode, <V_F2DI_CONVERT>mode);
2695     DONE;
2696   }
2697 )
2698
2699 (define_expand "lfloor<mode><v_f2si_convert>2"
2700   [(match_operand:<V_F2SI_CONVERT>   0 "register_operand")
2701    (match_operand:V_VLS_F_CONVERT_SI 1 "register_operand")]
2702   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2703   {
2704     riscv_vector::expand_vec_lfloor (operands[0], operands[1], <MODE>mode, <V_F2SI_CONVERT>mode);
2705     DONE;
2706   }
2707 )
2708
2709 (define_expand "lfloor<mode><v_f2di_convert>2"
2710   [(match_operand:<V_F2DI_CONVERT>   0 "register_operand")
2711    (match_operand:V_VLS_F_CONVERT_DI 1 "register_operand")]
2712   "TARGET_VECTOR && !flag_trapping_math && !flag_rounding_math"
2713   {
2714     riscv_vector::expand_vec_lfloor (operands[0], operands[1], <MODE>mode, <V_F2DI_CONVERT>mode);
2715     DONE;
2716   }
2717 )
2718
2719 ;; Implement rawmemchr[qi|si|hi].
2720 (define_expand "rawmemchr<ANYI:mode>"
2721   [(match_operand      0 "register_operand")
2722    (match_operand      1 "memory_operand")
2723    (match_operand:ANYI 2 "const_int_operand")]
2724   "TARGET_VECTOR && !TARGET_XTHEADVECTOR"
2725   {
2726     riscv_vector::expand_rawmemchr(<MODE>mode, operands[0], operands[1],
2727                                    operands[2]);
2728     DONE;
2729   }
2730 )
2731
2732 ;; =========================================================================
2733 ;; == [INT] Saturation ALU.
2734 ;; =========================================================================
2735 ;; Includes:
2736 ;; - add
2737 ;; - sub
2738 ;; - trunc
2739 ;; =========================================================================
2740 (define_expand "usadd<mode>3"
2741   [(match_operand:V_VLSI 0 "register_operand")
2742    (match_operand:V_VLSI 1 "register_operand")
2743    (match_operand:V_VLSI 2 "register_operand")]
2744   "TARGET_VECTOR"
2745   {
2746     riscv_vector::expand_vec_usadd (operands[0], operands[1], operands[2], <MODE>mode);
2747     DONE;
2748   }
2749 )
2750
2751 (define_expand "ssadd<mode>3"
2752   [(match_operand:V_VLSI 0 "register_operand")
2753    (match_operand:V_VLSI 1 "register_operand")
2754    (match_operand:V_VLSI 2 "register_operand")]
2755   "TARGET_VECTOR"
2756   {
2757     riscv_vector::expand_vec_ssadd (operands[0], operands[1], operands[2], <MODE>mode);
2758     DONE;
2759   }
2760 )
2761
2762 (define_expand "ussub<mode>3"
2763   [(match_operand:V_VLSI 0 "register_operand")
2764    (match_operand:V_VLSI 1 "register_operand")
2765    (match_operand:V_VLSI 2 "register_operand")]
2766   "TARGET_VECTOR"
2767   {
2768     riscv_vector::expand_vec_ussub (operands[0], operands[1], operands[2], <MODE>mode);
2769     DONE;
2770   }
2771 )
2772
2773 (define_expand "sssub<mode>3"
2774   [(match_operand:V_VLSI 0 "register_operand")
2775    (match_operand:V_VLSI 1 "register_operand")
2776    (match_operand:V_VLSI 2 "register_operand")]
2777   "TARGET_VECTOR"
2778   {
2779     riscv_vector::expand_vec_sssub (operands[0], operands[1], operands[2], <MODE>mode);
2780     DONE;
2781   }
2782 )
2783
2784 (define_expand "ustrunc<mode><v_double_trunc>2"
2785   [(match_operand:<V_DOUBLE_TRUNC> 0 "register_operand")
2786    (match_operand:VWEXTI           1 "register_operand")]
2787   "TARGET_VECTOR"
2788   {
2789     riscv_vector::expand_vec_double_ustrunc (operands[0], operands[1],
2790                                              <MODE>mode);
2791     DONE;
2792   }
2793 )
2794
2795 (define_expand "ustrunc<mode><v_quad_trunc>2"
2796   [(match_operand:<V_QUAD_TRUNC> 0 "register_operand")
2797    (match_operand:VQEXTI         1 "register_operand")]
2798   "TARGET_VECTOR"
2799   {
2800     riscv_vector::expand_vec_quad_ustrunc (operands[0], operands[1], <MODE>mode,
2801                                            <V_DOUBLE_TRUNC>mode);
2802     DONE;
2803   }
2804 )
2805
2806 (define_expand "ustrunc<mode><v_oct_trunc>2"
2807   [(match_operand:<V_OCT_TRUNC> 0 "register_operand")
2808    (match_operand:VOEXTI        1 "register_operand")]
2809   "TARGET_VECTOR"
2810   {
2811     riscv_vector::expand_vec_oct_ustrunc (operands[0], operands[1], <MODE>mode,
2812                                           <V_DOUBLE_TRUNC>mode,
2813                                           <V_QUAD_TRUNC>mode);
2814     DONE;
2815   }
2816 )
2817
2818 (define_expand "sstrunc<mode><v_double_trunc>2"
2819   [(match_operand:<V_DOUBLE_TRUNC> 0 "register_operand")
2820    (match_operand:VWEXTI           1 "register_operand")]
2821   "TARGET_VECTOR"
2822   {
2823     riscv_vector::expand_vec_double_sstrunc (operands[0], operands[1],
2824                                           <MODE>mode);
2825     DONE;
2826   }
2827 )
2828
2829 (define_expand "sstrunc<mode><v_quad_trunc>2"
2830   [(match_operand:<V_QUAD_TRUNC> 0 "register_operand")
2831    (match_operand:VQEXTI         1 "register_operand")]
2832   "TARGET_VECTOR"
2833   {
2834     riscv_vector::expand_vec_quad_sstrunc (operands[0], operands[1], <MODE>mode,
2835                                            <V_DOUBLE_TRUNC>mode);
2836     DONE;
2837   }
2838 )
2839
2840 (define_expand "sstrunc<mode><v_oct_trunc>2"
2841   [(match_operand:<V_OCT_TRUNC> 0 "register_operand")
2842    (match_operand:VOEXTI        1 "register_operand")]
2843   "TARGET_VECTOR"
2844   {
2845     riscv_vector::expand_vec_oct_sstrunc (operands[0], operands[1], <MODE>mode,
2846                                           <V_DOUBLE_TRUNC>mode,
2847                                           <V_QUAD_TRUNC>mode);
2848     DONE;
2849   }
2850 )
2851
2852 ;; =========================================================================
2853 ;; == Early break auto-vectorization patterns
2854 ;; =========================================================================
2855
2856 ;; vcond_mask_len (mask, 1s, 0s, len, bias)
2857 ;; => mask[i] = mask[i] && i < len ? 1 : 0
2858 (define_insn_and_split "vcond_mask_len_<mode>"
2859   [(set (match_operand:VB 0 "register_operand")
2860     (unspec: VB [
2861      (match_operand:VB 1 "register_operand")
2862      (match_operand:VB 2 "const_1_operand")
2863      (match_operand:VB 3 "const_0_operand")
2864      (match_operand 4 "autovec_length_operand")
2865      (match_operand 5 "const_0_operand")] UNSPEC_SELECT_MASK))]
2866   "TARGET_VECTOR
2867    && can_create_pseudo_p ()
2868    && riscv_vector::get_vector_mode (Pmode, GET_MODE_NUNITS (<MODE>mode)).exists ()"
2869   "#"
2870   "&& 1"
2871   [(const_int 0)]
2872   {
2873     machine_mode mode = riscv_vector::get_vector_mode (Pmode,
2874                         GET_MODE_NUNITS (<MODE>mode)).require ();
2875     rtx reg = gen_reg_rtx (mode);
2876     riscv_vector::expand_vec_series (reg, const0_rtx, const1_rtx);
2877     rtx dup_rtx = gen_rtx_VEC_DUPLICATE (mode, operands[4]);
2878     insn_code icode = code_for_pred_cmp_scalar (mode);
2879     rtx cmp = gen_rtx_fmt_ee (LTU, <MODE>mode, reg, dup_rtx);
2880     rtx ops[] = {operands[0], operands[1], operands[1], cmp, reg, operands[4]};
2881     emit_vlmax_insn (icode, riscv_vector::COMPARE_OP_MU, ops);
2882     DONE;
2883   }
2884   [(set_attr "type" "vector")])
2885
2886 ;; cbranch
2887 (define_expand "cbranch<mode>4"
2888   [(set (pc)
2889         (if_then_else
2890           (match_operator 0 "equality_operator"
2891             [(match_operand:VB_VLS 1 "register_operand")
2892              (match_operand:VB_VLS 2 "reg_or_0_operand")])
2893           (label_ref (match_operand 3 ""))
2894           (pc)))]
2895   "TARGET_VECTOR"
2896   {
2897     rtx pred;
2898     if (operands[2] == CONST0_RTX (<MODE>mode))
2899       pred = operands[1];
2900     else
2901       pred = expand_binop (<MODE>mode, xor_optab, operands[1],
2902                            operands[2], NULL_RTX, 0,
2903                            OPTAB_DIRECT);
2904     rtx reg = gen_reg_rtx (Pmode);
2905     rtx cpop_ops[] = {reg, pred};
2906     emit_vlmax_insn (code_for_pred_popcount (<MODE>mode, Pmode),
2907                      riscv_vector::CPOP_OP, cpop_ops);
2908     operands[1] = reg;
2909     operands[2] = const0_rtx;
2910   }
2911 )
2912
2913 ;; -------------------------------------------------------------------------
2914 ;; - vrol.vv vror.vv
2915 ;; -------------------------------------------------------------------------
2916 (define_expand "v<bitmanip_optab><mode>3"
2917   [(set (match_operand:VI 0 "register_operand")
2918         (bitmanip_rotate:VI
2919           (match_operand:VI 1 "register_operand")
2920           (match_operand:VI 2 "register_operand")))]
2921   "TARGET_ZVBB || TARGET_ZVKB"
2922   {
2923     riscv_vector::emit_vlmax_insn (code_for_pred_v (<CODE>, <MODE>mode),
2924                                    riscv_vector::BINARY_OP, operands);
2925     DONE;
2926   }
2927 )
2928
2929 ;; =========================================================================
2930 ;; == Strided Load/Store
2931 ;; =========================================================================
2932 (define_expand "mask_len_strided_load_<mode>"
2933   [(match_operand:V_VLS 0 "register_operand")
2934    (match_operand       1 "pmode_reg_or_0_operand")
2935    (match_operand       2 "pmode_reg_or_0_operand")
2936    (match_operand:<VM>  3 "vector_mask_operand")
2937    (match_operand       4 "maskload_else_operand")
2938    (match_operand       5 "autovec_length_operand")
2939    (match_operand       6 "const_0_operand")]
2940   "TARGET_VECTOR"
2941   {
2942     riscv_vector::expand_strided_load (<MODE>mode, operands);
2943     DONE;
2944   })
2945
2946 (define_expand "mask_len_strided_store_<mode>"
2947   [(match_operand       0 "pmode_reg_or_0_operand")
2948    (match_operand       1 "pmode_reg_or_0_operand")
2949    (match_operand:V_VLS 2 "register_operand")
2950    (match_operand:<VM>  3 "vector_mask_operand")
2951    (match_operand       4 "autovec_length_operand")
2952    (match_operand       5 "const_0_operand")]
2953   "TARGET_VECTOR"
2954   {
2955     riscv_vector::expand_strided_store (<MODE>mode, operands);
2956     DONE;
2957   })
2958
2959 ; ========
2960 ; == Absolute difference (not including sum)
2961 ; ========
2962 (define_expand "uabd<mode>3"
2963   [(match_operand:V_VLSI 0 "register_operand")
2964    (match_operand:V_VLSI 1 "register_operand")
2965    (match_operand:V_VLSI 2 "register_operand")]
2966   "TARGET_VECTOR"
2967   {
2968     rtx max = gen_reg_rtx (<MODE>mode);
2969     insn_code icode = code_for_pred (UMAX, <MODE>mode);
2970     rtx ops1[] = {max, operands[1], operands[2]};
2971     riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops1);
2972
2973     rtx min = gen_reg_rtx (<MODE>mode);
2974     icode = code_for_pred (UMIN, <MODE>mode);
2975     rtx ops2[] = {min, operands[1], operands[2]};
2976     riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops2);
2977
2978     icode = code_for_pred (MINUS, <MODE>mode);
2979     rtx ops3[] = {operands[0], max, min};
2980     riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops3);
2981
2982     DONE;
2983   });