[NFC][RemoveDIs] Prefer iterators over inst-pointers in InstCombine

[llvm-project.git] / llvm / test / CodeGen / AArch64 / aarch64-mops.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc %s -o - -mtriple=aarch64 -O0 -global-isel=1 -global-isel-abort=1                    | FileCheck %s --check-prefixes=GISel-WITHOUT-MOPS,GISel-WITHOUT-MOPS-O0
; RUN: llc %s -o - -mtriple=aarch64     -global-isel=1 -global-isel-abort=1                    | FileCheck %s --check-prefixes=GISel-WITHOUT-MOPS,GISel-WITHOUT-MOPS-O3
; RUN: llc %s -o - -mtriple=aarch64 -O0 -global-isel=1 -global-isel-abort=1 -mattr=+mops       | FileCheck %s --check-prefixes=GISel-MOPS,GISel-MOPS-O0
; RUN: llc %s -o - -mtriple=aarch64     -global-isel=1 -global-isel-abort=1 -mattr=+mops       | FileCheck %s --check-prefixes=GISel-MOPS,GISel-MOPS-O3
; RUN: llc %s -o - -mtriple=aarch64 -O2                    | FileCheck %s --check-prefix=SDAG-WITHOUT-MOPS-O2
; RUN: llc %s -o - -mtriple=aarch64 -O2 -mattr=+mops       | FileCheck %s --check-prefix=SDAG-MOPS-O2

declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)

declare void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1 immarg)

declare void @llvm.memcpy.inline.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1 immarg)

declare void @llvm.memmove.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1 immarg)

define void @memset_0_zeroval(ptr %dst) {
; GISel-WITHOUT-MOPS-LABEL: memset_0_zeroval:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memset_0_zeroval:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_0_zeroval:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_0_zeroval:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 0, i64 0, i1 false)
  ret void
}

define void @memset_0_zeroval_volatile(ptr %dst) {
; GISel-WITHOUT-MOPS-LABEL: memset_0_zeroval_volatile:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memset_0_zeroval_volatile:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_0_zeroval_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_0_zeroval_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 0, i64 0, i1 true)
  ret void
}

define void @memset_10_zeroval(ptr %dst) {
; GISel-WITHOUT-MOPS-LABEL: memset_10_zeroval:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    str xzr, [x0]
; GISel-WITHOUT-MOPS-NEXT:    strh wzr, [x0, #8]
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memset_10_zeroval:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    str xzr, [x0]
; GISel-MOPS-NEXT:    strh wzr, [x0, #8]
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_10_zeroval:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    strh wzr, [x0, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str xzr, [x0]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_10_zeroval:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    strh wzr, [x0, #8]
; SDAG-MOPS-O2-NEXT:    str xzr, [x0]
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 0, i64 10, i1 false)
  ret void
}

define void @memset_10_zeroval_volatile(ptr %dst) {
; GISel-WITHOUT-MOPS-O0-LABEL: memset_10_zeroval_volatile:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w8, #10 // =0xa
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w2, w8
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w1, wzr
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memset_10_zeroval_volatile:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w1, wzr
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w2, #10 // =0xa
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memset_10_zeroval_volatile:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov w8, #10 // =0xa
; GISel-MOPS-O0-NEXT:    // kill: def $x8 killed $w8
; GISel-MOPS-O0-NEXT:    mov x9, xzr
; GISel-MOPS-O0-NEXT:    setp [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    setm [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    sete [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memset_10_zeroval_volatile:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    mov w8, #10 // =0xa
; GISel-MOPS-O3-NEXT:    setp [x0]!, x8!, xzr
; GISel-MOPS-O3-NEXT:    setm [x0]!, x8!, xzr
; GISel-MOPS-O3-NEXT:    sete [x0]!, x8!, xzr
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_10_zeroval_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    strh wzr, [x0, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str xzr, [x0]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_10_zeroval_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    strh wzr, [x0, #8]
; SDAG-MOPS-O2-NEXT:    str xzr, [x0]
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 0, i64 10, i1 true)
  ret void
}

define void @memset_10000_zeroval(ptr %dst) {
; GISel-WITHOUT-MOPS-O0-LABEL: memset_10000_zeroval:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w8, #10000 // =0x2710
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w2, w8
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w1, wzr
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memset_10000_zeroval:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w1, wzr
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w2, #10000 // =0x2710
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memset_10000_zeroval:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov w8, #10000 // =0x2710
; GISel-MOPS-O0-NEXT:    // kill: def $x8 killed $w8
; GISel-MOPS-O0-NEXT:    mov x9, xzr
; GISel-MOPS-O0-NEXT:    setp [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    setm [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    sete [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memset_10000_zeroval:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    mov w8, #10000 // =0x2710
; GISel-MOPS-O3-NEXT:    setp [x0]!, x8!, xzr
; GISel-MOPS-O3-NEXT:    setm [x0]!, x8!, xzr
; GISel-MOPS-O3-NEXT:    sete [x0]!, x8!, xzr
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_10000_zeroval:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w1, wzr
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w2, #10000 // =0x2710
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memset
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_10000_zeroval:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    mov w8, #10000 // =0x2710
; SDAG-MOPS-O2-NEXT:    setp [x0]!, x8!, xzr
; SDAG-MOPS-O2-NEXT:    setm [x0]!, x8!, xzr
; SDAG-MOPS-O2-NEXT:    sete [x0]!, x8!, xzr
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 0, i64 10000, i1 false)
  ret void
}

define void @memset_10000_zeroval_volatile(ptr %dst) {
; GISel-WITHOUT-MOPS-O0-LABEL: memset_10000_zeroval_volatile:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w8, #10000 // =0x2710
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w2, w8
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w1, wzr
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memset_10000_zeroval_volatile:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w1, wzr
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w2, #10000 // =0x2710
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memset_10000_zeroval_volatile:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov w8, #10000 // =0x2710
; GISel-MOPS-O0-NEXT:    // kill: def $x8 killed $w8
; GISel-MOPS-O0-NEXT:    mov x9, xzr
; GISel-MOPS-O0-NEXT:    setp [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    setm [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    sete [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memset_10000_zeroval_volatile:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    mov w8, #10000 // =0x2710
; GISel-MOPS-O3-NEXT:    setp [x0]!, x8!, xzr
; GISel-MOPS-O3-NEXT:    setm [x0]!, x8!, xzr
; GISel-MOPS-O3-NEXT:    sete [x0]!, x8!, xzr
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_10000_zeroval_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w1, wzr
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w2, #10000 // =0x2710
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memset
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_10000_zeroval_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    mov w8, #10000 // =0x2710
; SDAG-MOPS-O2-NEXT:    setp [x0]!, x8!, xzr
; SDAG-MOPS-O2-NEXT:    setm [x0]!, x8!, xzr
; SDAG-MOPS-O2-NEXT:    sete [x0]!, x8!, xzr
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 0, i64 10000, i1 true)
  ret void
}

define void @memset_size_zeroval(ptr %dst, i64 %size) {
; GISel-WITHOUT-MOPS-LABEL: memset_size_zeroval:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-NEXT:    mov x2, x1
; GISel-WITHOUT-MOPS-NEXT:    mov w1, wzr
; GISel-WITHOUT-MOPS-NEXT:    bl memset
; GISel-WITHOUT-MOPS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memset_size_zeroval:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov x8, xzr
; GISel-MOPS-O0-NEXT:    setp [x0]!, x1!, x8
; GISel-MOPS-O0-NEXT:    setm [x0]!, x1!, x8
; GISel-MOPS-O0-NEXT:    sete [x0]!, x1!, x8
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memset_size_zeroval:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    setp [x0]!, x1!, xzr
; GISel-MOPS-O3-NEXT:    setm [x0]!, x1!, xzr
; GISel-MOPS-O3-NEXT:    sete [x0]!, x1!, xzr
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_size_zeroval:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov x2, x1
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w1, wzr
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memset
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_size_zeroval:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    setp [x0]!, x1!, xzr
; SDAG-MOPS-O2-NEXT:    setm [x0]!, x1!, xzr
; SDAG-MOPS-O2-NEXT:    sete [x0]!, x1!, xzr
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 0, i64 %size, i1 false)
  ret void
}

define void @memset_size_zeroval_volatile(ptr %dst, i64 %size) {
; GISel-WITHOUT-MOPS-LABEL: memset_size_zeroval_volatile:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-NEXT:    mov x2, x1
; GISel-WITHOUT-MOPS-NEXT:    mov w1, wzr
; GISel-WITHOUT-MOPS-NEXT:    bl memset
; GISel-WITHOUT-MOPS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memset_size_zeroval_volatile:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov x8, xzr
; GISel-MOPS-O0-NEXT:    setp [x0]!, x1!, x8
; GISel-MOPS-O0-NEXT:    setm [x0]!, x1!, x8
; GISel-MOPS-O0-NEXT:    sete [x0]!, x1!, x8
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memset_size_zeroval_volatile:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    setp [x0]!, x1!, xzr
; GISel-MOPS-O3-NEXT:    setm [x0]!, x1!, xzr
; GISel-MOPS-O3-NEXT:    sete [x0]!, x1!, xzr
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_size_zeroval_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov x2, x1
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w1, wzr
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memset
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_size_zeroval_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    setp [x0]!, x1!, xzr
; SDAG-MOPS-O2-NEXT:    setm [x0]!, x1!, xzr
; SDAG-MOPS-O2-NEXT:    sete [x0]!, x1!, xzr
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 0, i64 %size, i1 true)
  ret void
}


define void @memset_0(ptr %dst, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memset_0:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memset_0:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_0:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_0:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ret
entry:
  %value_trunc = trunc i32 %value to i8
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 0, i64 0, i1 false)
  ret void
}

define void @memset_0_volatile(ptr %dst, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memset_0_volatile:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memset_0_volatile:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_0_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_0_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ret
entry:
  %value_trunc = trunc i32 %value to i8
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 0, i64 0, i1 true)
  ret void
}

define void @memset_10(ptr %dst, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memset_10:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    // implicit-def: $x8
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w8, w1
; GISel-WITHOUT-MOPS-O0-NEXT:    and x8, x8, #0xff
; GISel-WITHOUT-MOPS-O0-NEXT:    mov x9, #72340172838076673 // =0x101010101010101
; GISel-WITHOUT-MOPS-O0-NEXT:    mul x8, x8, x9
; GISel-WITHOUT-MOPS-O0-NEXT:    str x8, [x0]
; GISel-WITHOUT-MOPS-O0-NEXT:    // kill: def $w8 killed $w8 killed $x8
; GISel-WITHOUT-MOPS-O0-NEXT:    strh w8, [x0, #8]
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memset_10:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    // kill: def $w1 killed $w1 def $x1
; GISel-WITHOUT-MOPS-O3-NEXT:    mov x8, #72340172838076673 // =0x101010101010101
; GISel-WITHOUT-MOPS-O3-NEXT:    and x9, x1, #0xff
; GISel-WITHOUT-MOPS-O3-NEXT:    mul x8, x9, x8
; GISel-WITHOUT-MOPS-O3-NEXT:    str x8, [x0]
; GISel-WITHOUT-MOPS-O3-NEXT:    strh w8, [x0, #8]
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memset_10:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    // implicit-def: $x8
; GISel-MOPS-O0-NEXT:    mov w8, w1
; GISel-MOPS-O0-NEXT:    and x8, x8, #0xff
; GISel-MOPS-O0-NEXT:    mov x9, #72340172838076673 // =0x101010101010101
; GISel-MOPS-O0-NEXT:    mul x8, x8, x9
; GISel-MOPS-O0-NEXT:    str x8, [x0]
; GISel-MOPS-O0-NEXT:    // kill: def $w8 killed $w8 killed $x8
; GISel-MOPS-O0-NEXT:    strh w8, [x0, #8]
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memset_10:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    // kill: def $w1 killed $w1 def $x1
; GISel-MOPS-O3-NEXT:    mov x8, #72340172838076673 // =0x101010101010101
; GISel-MOPS-O3-NEXT:    and x9, x1, #0xff
; GISel-MOPS-O3-NEXT:    mul x8, x9, x8
; GISel-MOPS-O3-NEXT:    str x8, [x0]
; GISel-MOPS-O3-NEXT:    strh w8, [x0, #8]
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_10:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    // kill: def $w1 killed $w1 def $x1
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov x8, #72340172838076673 // =0x101010101010101
; SDAG-WITHOUT-MOPS-O2-NEXT:    and x9, x1, #0xff
; SDAG-WITHOUT-MOPS-O2-NEXT:    mul x8, x9, x8
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x8, [x0]
; SDAG-WITHOUT-MOPS-O2-NEXT:    strh w8, [x0, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_10:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    // kill: def $w1 killed $w1 def $x1
; SDAG-MOPS-O2-NEXT:    mov x8, #72340172838076673 // =0x101010101010101
; SDAG-MOPS-O2-NEXT:    and x9, x1, #0xff
; SDAG-MOPS-O2-NEXT:    mul x8, x9, x8
; SDAG-MOPS-O2-NEXT:    str x8, [x0]
; SDAG-MOPS-O2-NEXT:    strh w8, [x0, #8]
; SDAG-MOPS-O2-NEXT:    ret
entry:
  %value_trunc = trunc i32 %value to i8
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 %value_trunc, i64 10, i1 false)
  ret void
}

define void @memset_10_volatile(ptr %dst, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memset_10_volatile:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w8, #10 // =0xa
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w2, w8
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memset_10_volatile:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w2, #10 // =0xa
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memset_10_volatile:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov w8, #10 // =0xa
; GISel-MOPS-O0-NEXT:    // kill: def $x8 killed $w8
; GISel-MOPS-O0-NEXT:    // implicit-def: $x9
; GISel-MOPS-O0-NEXT:    mov w9, w1
; GISel-MOPS-O0-NEXT:    setp [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    setm [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    sete [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memset_10_volatile:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    mov w8, #10 // =0xa
; GISel-MOPS-O3-NEXT:    // kill: def $w1 killed $w1 def $x1
; GISel-MOPS-O3-NEXT:    setp [x0]!, x8!, x1
; GISel-MOPS-O3-NEXT:    setm [x0]!, x8!, x1
; GISel-MOPS-O3-NEXT:    sete [x0]!, x8!, x1
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_10_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    // kill: def $w1 killed $w1 def $x1
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov x8, #72340172838076673 // =0x101010101010101
; SDAG-WITHOUT-MOPS-O2-NEXT:    and x9, x1, #0xff
; SDAG-WITHOUT-MOPS-O2-NEXT:    mul x8, x9, x8
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x8, [x0]
; SDAG-WITHOUT-MOPS-O2-NEXT:    strh w8, [x0, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_10_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    // kill: def $w1 killed $w1 def $x1
; SDAG-MOPS-O2-NEXT:    mov x8, #72340172838076673 // =0x101010101010101
; SDAG-MOPS-O2-NEXT:    and x9, x1, #0xff
; SDAG-MOPS-O2-NEXT:    mul x8, x9, x8
; SDAG-MOPS-O2-NEXT:    str x8, [x0]
; SDAG-MOPS-O2-NEXT:    strh w8, [x0, #8]
; SDAG-MOPS-O2-NEXT:    ret
entry:
  %value_trunc = trunc i32 %value to i8
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 %value_trunc, i64 10, i1 true)
  ret void
}

define void @memset_10000(ptr %dst, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memset_10000:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w8, #10000 // =0x2710
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w2, w8
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memset_10000:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w2, #10000 // =0x2710
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memset_10000:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov w8, #10000 // =0x2710
; GISel-MOPS-O0-NEXT:    // kill: def $x8 killed $w8
; GISel-MOPS-O0-NEXT:    // implicit-def: $x9
; GISel-MOPS-O0-NEXT:    mov w9, w1
; GISel-MOPS-O0-NEXT:    setp [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    setm [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    sete [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memset_10000:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    mov w8, #10000 // =0x2710
; GISel-MOPS-O3-NEXT:    // kill: def $w1 killed $w1 def $x1
; GISel-MOPS-O3-NEXT:    setp [x0]!, x8!, x1
; GISel-MOPS-O3-NEXT:    setm [x0]!, x8!, x1
; GISel-MOPS-O3-NEXT:    sete [x0]!, x8!, x1
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_10000:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w2, #10000 // =0x2710
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memset
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_10000:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    mov w8, #10000 // =0x2710
; SDAG-MOPS-O2-NEXT:    // kill: def $w1 killed $w1 def $x1
; SDAG-MOPS-O2-NEXT:    setp [x0]!, x8!, x1
; SDAG-MOPS-O2-NEXT:    setm [x0]!, x8!, x1
; SDAG-MOPS-O2-NEXT:    sete [x0]!, x8!, x1
; SDAG-MOPS-O2-NEXT:    ret
entry:
  %value_trunc = trunc i32 %value to i8
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 %value_trunc, i64 10000, i1 false)
  ret void
}

define void @memset_10000_volatile(ptr %dst, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memset_10000_volatile:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w8, #10000 // =0x2710
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w2, w8
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memset_10000_volatile:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w2, #10000 // =0x2710
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memset_10000_volatile:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov w8, #10000 // =0x2710
; GISel-MOPS-O0-NEXT:    // kill: def $x8 killed $w8
; GISel-MOPS-O0-NEXT:    // implicit-def: $x9
; GISel-MOPS-O0-NEXT:    mov w9, w1
; GISel-MOPS-O0-NEXT:    setp [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    setm [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    sete [x0]!, x8!, x9
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memset_10000_volatile:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    mov w8, #10000 // =0x2710
; GISel-MOPS-O3-NEXT:    // kill: def $w1 killed $w1 def $x1
; GISel-MOPS-O3-NEXT:    setp [x0]!, x8!, x1
; GISel-MOPS-O3-NEXT:    setm [x0]!, x8!, x1
; GISel-MOPS-O3-NEXT:    sete [x0]!, x8!, x1
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_10000_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w2, #10000 // =0x2710
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memset
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_10000_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    mov w8, #10000 // =0x2710
; SDAG-MOPS-O2-NEXT:    // kill: def $w1 killed $w1 def $x1
; SDAG-MOPS-O2-NEXT:    setp [x0]!, x8!, x1
; SDAG-MOPS-O2-NEXT:    setm [x0]!, x8!, x1
; SDAG-MOPS-O2-NEXT:    sete [x0]!, x8!, x1
; SDAG-MOPS-O2-NEXT:    ret
entry:
  %value_trunc = trunc i32 %value to i8
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 %value_trunc, i64 10000, i1 true)
  ret void
}

define void @memset_size(ptr %dst, i64 %size, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memset_size:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    sub sp, sp, #32
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 32
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    str x1, [sp, #8] // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w1, w2
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x2, [sp, #8] // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    add sp, sp, #32
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memset_size:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov x3, x1
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w1, w2
; GISel-WITHOUT-MOPS-O3-NEXT:    mov x2, x3
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memset_size:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    // implicit-def: $x8
; GISel-MOPS-O0-NEXT:    mov w8, w2
; GISel-MOPS-O0-NEXT:    setp [x0]!, x1!, x8
; GISel-MOPS-O0-NEXT:    setm [x0]!, x1!, x8
; GISel-MOPS-O0-NEXT:    sete [x0]!, x1!, x8
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memset_size:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    // kill: def $w2 killed $w2 def $x2
; GISel-MOPS-O3-NEXT:    setp [x0]!, x1!, x2
; GISel-MOPS-O3-NEXT:    setm [x0]!, x1!, x2
; GISel-MOPS-O3-NEXT:    sete [x0]!, x1!, x2
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_size:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov x8, x1
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w1, w2
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov x2, x8
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memset
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_size:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    // kill: def $w2 killed $w2 def $x2
; SDAG-MOPS-O2-NEXT:    setp [x0]!, x1!, x2
; SDAG-MOPS-O2-NEXT:    setm [x0]!, x1!, x2
; SDAG-MOPS-O2-NEXT:    sete [x0]!, x1!, x2
; SDAG-MOPS-O2-NEXT:    ret
entry:
  %value_trunc = trunc i32 %value to i8
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 %value_trunc, i64 %size, i1 false)
  ret void
}

define void @memset_size_volatile(ptr %dst, i64 %size, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memset_size_volatile:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    sub sp, sp, #32
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 32
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    str x1, [sp, #8] // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w1, w2
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x2, [sp, #8] // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    add sp, sp, #32
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memset_size_volatile:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov x3, x1
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w1, w2
; GISel-WITHOUT-MOPS-O3-NEXT:    mov x2, x3
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memset
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memset_size_volatile:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    // implicit-def: $x8
; GISel-MOPS-O0-NEXT:    mov w8, w2
; GISel-MOPS-O0-NEXT:    setp [x0]!, x1!, x8
; GISel-MOPS-O0-NEXT:    setm [x0]!, x1!, x8
; GISel-MOPS-O0-NEXT:    sete [x0]!, x1!, x8
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memset_size_volatile:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    // kill: def $w2 killed $w2 def $x2
; GISel-MOPS-O3-NEXT:    setp [x0]!, x1!, x2
; GISel-MOPS-O3-NEXT:    setm [x0]!, x1!, x2
; GISel-MOPS-O3-NEXT:    sete [x0]!, x1!, x2
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memset_size_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov x8, x1
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w1, w2
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov x2, x8
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memset
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memset_size_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    // kill: def $w2 killed $w2 def $x2
; SDAG-MOPS-O2-NEXT:    setp [x0]!, x1!, x2
; SDAG-MOPS-O2-NEXT:    setm [x0]!, x1!, x2
; SDAG-MOPS-O2-NEXT:    sete [x0]!, x1!, x2
; SDAG-MOPS-O2-NEXT:    ret
entry:
  %value_trunc = trunc i32 %value to i8
  call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 %value_trunc, i64 %size, i1 true)
  ret void
}


define void @memcpy_0(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memcpy_0:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memcpy_0:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_0:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_0:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 0, i1 false)
  ret void
}

define void @memcpy_0_volatile(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memcpy_0_volatile:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memcpy_0_volatile:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_0_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_0_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 0, i1 true)
  ret void
}

define void @memcpy_10(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memcpy_10:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ldr x8, [x1]
; GISel-WITHOUT-MOPS-NEXT:    str x8, [x0]
; GISel-WITHOUT-MOPS-NEXT:    ldrh w8, [x1, #8]
; GISel-WITHOUT-MOPS-NEXT:    strh w8, [x0, #8]
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memcpy_10:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ldr x8, [x1]
; GISel-MOPS-NEXT:    str x8, [x0]
; GISel-MOPS-NEXT:    ldrh w8, [x1, #8]
; GISel-MOPS-NEXT:    strh w8, [x0, #8]
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_10:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldrh w8, [x1, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x9, [x1]
; SDAG-WITHOUT-MOPS-O2-NEXT:    strh w8, [x0, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x9, [x0]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_10:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ldrh w8, [x1, #8]
; SDAG-MOPS-O2-NEXT:    ldr x9, [x1]
; SDAG-MOPS-O2-NEXT:    strh w8, [x0, #8]
; SDAG-MOPS-O2-NEXT:    str x9, [x0]
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 10, i1 false)
  ret void
}

define void @memcpy_10_volatile(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memcpy_10_volatile:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w8, #10 // =0xa
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w2, w8
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memcpy
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memcpy_10_volatile:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w2, #10 // =0xa
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memcpy
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memcpy_10_volatile:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov w8, #10 // =0xa
; GISel-MOPS-O0-NEXT:    // kill: def $x8 killed $w8
; GISel-MOPS-O0-NEXT:    cpyfp [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    cpyfm [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    cpyfe [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memcpy_10_volatile:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    mov w8, #10 // =0xa
; GISel-MOPS-O3-NEXT:    cpyfp [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    cpyfm [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    cpyfe [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_10_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x8, [x1]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldrh w9, [x1, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    strh w9, [x0, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x8, [x0]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_10_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ldr x8, [x1]
; SDAG-MOPS-O2-NEXT:    ldrh w9, [x1, #8]
; SDAG-MOPS-O2-NEXT:    strh w9, [x0, #8]
; SDAG-MOPS-O2-NEXT:    str x8, [x0]
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 10, i1 true)
  ret void
}

define void @memcpy_1000(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memcpy_1000:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w8, #1000 // =0x3e8
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w2, w8
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memcpy
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memcpy_1000:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w2, #1000 // =0x3e8
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memcpy
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memcpy_1000:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov w8, #1000 // =0x3e8
; GISel-MOPS-O0-NEXT:    // kill: def $x8 killed $w8
; GISel-MOPS-O0-NEXT:    cpyfp [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    cpyfm [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    cpyfe [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memcpy_1000:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    mov w8, #1000 // =0x3e8
; GISel-MOPS-O3-NEXT:    cpyfp [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    cpyfm [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    cpyfe [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_1000:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w2, #1000 // =0x3e8
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memcpy
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_1000:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    mov w8, #1000 // =0x3e8
; SDAG-MOPS-O2-NEXT:    cpyfp [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    cpyfm [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    cpyfe [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 1000, i1 false)
  ret void
}

define void @memcpy_1000_volatile(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memcpy_1000_volatile:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w8, #1000 // =0x3e8
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w2, w8
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memcpy
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memcpy_1000_volatile:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w2, #1000 // =0x3e8
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memcpy
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memcpy_1000_volatile:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov w8, #1000 // =0x3e8
; GISel-MOPS-O0-NEXT:    // kill: def $x8 killed $w8
; GISel-MOPS-O0-NEXT:    cpyfp [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    cpyfm [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    cpyfe [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memcpy_1000_volatile:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    mov w8, #1000 // =0x3e8
; GISel-MOPS-O3-NEXT:    cpyfp [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    cpyfm [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    cpyfe [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_1000_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w2, #1000 // =0x3e8
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memcpy
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_1000_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    mov w8, #1000 // =0x3e8
; SDAG-MOPS-O2-NEXT:    cpyfp [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    cpyfm [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    cpyfe [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 1000, i1 true)
  ret void
}

define void @memcpy_n(ptr %dst, ptr %src, i64 %size, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memcpy_n:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-NEXT:    bl memcpy
; GISel-WITHOUT-MOPS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memcpy_n:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    cpyfp [x0]!, [x1]!, x2!
; GISel-MOPS-NEXT:    cpyfm [x0]!, [x1]!, x2!
; GISel-MOPS-NEXT:    cpyfe [x0]!, [x1]!, x2!
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_n:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memcpy
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_n:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    cpyfp [x0]!, [x1]!, x2!
; SDAG-MOPS-O2-NEXT:    cpyfm [x0]!, [x1]!, x2!
; SDAG-MOPS-O2-NEXT:    cpyfe [x0]!, [x1]!, x2!
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 %size, i1 false)
  ret void
}

define void @memcpy_n_volatile(ptr %dst, ptr %src, i64 %size, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memcpy_n_volatile:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-NEXT:    bl memcpy
; GISel-WITHOUT-MOPS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memcpy_n_volatile:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    cpyfp [x0]!, [x1]!, x2!
; GISel-MOPS-NEXT:    cpyfm [x0]!, [x1]!, x2!
; GISel-MOPS-NEXT:    cpyfe [x0]!, [x1]!, x2!
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_n_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memcpy
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_n_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    cpyfp [x0]!, [x1]!, x2!
; SDAG-MOPS-O2-NEXT:    cpyfm [x0]!, [x1]!, x2!
; SDAG-MOPS-O2-NEXT:    cpyfe [x0]!, [x1]!, x2!
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 %size, i1 true)
  ret void
}


define void @memcpy_inline_0(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memcpy_inline_0:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memcpy_inline_0:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_inline_0:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_inline_0:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 0, i1 false)
  ret void
}

define void @memcpy_inline_0_volatile(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memcpy_inline_0_volatile:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memcpy_inline_0_volatile:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_inline_0_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_inline_0_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 0, i1 true)
  ret void
}

define void @memcpy_inline_10(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memcpy_inline_10:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ldr x8, [x1]
; GISel-WITHOUT-MOPS-NEXT:    str x8, [x0]
; GISel-WITHOUT-MOPS-NEXT:    ldrh w8, [x1, #8]
; GISel-WITHOUT-MOPS-NEXT:    strh w8, [x0, #8]
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memcpy_inline_10:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ldr x8, [x1]
; GISel-MOPS-NEXT:    str x8, [x0]
; GISel-MOPS-NEXT:    ldrh w8, [x1, #8]
; GISel-MOPS-NEXT:    strh w8, [x0, #8]
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_inline_10:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldrh w8, [x1, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x9, [x1]
; SDAG-WITHOUT-MOPS-O2-NEXT:    strh w8, [x0, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x9, [x0]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_inline_10:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ldrh w8, [x1, #8]
; SDAG-MOPS-O2-NEXT:    ldr x9, [x1]
; SDAG-MOPS-O2-NEXT:    strh w8, [x0, #8]
; SDAG-MOPS-O2-NEXT:    str x9, [x0]
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 10, i1 false)
  ret void
}

define void @memcpy_inline_10_volatile(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memcpy_inline_10_volatile:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ldr x8, [x1]
; GISel-WITHOUT-MOPS-NEXT:    str x8, [x0]
; GISel-WITHOUT-MOPS-NEXT:    ldrh w8, [x1, #8]
; GISel-WITHOUT-MOPS-NEXT:    strh w8, [x0, #8]
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memcpy_inline_10_volatile:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ldr x8, [x1]
; GISel-MOPS-NEXT:    str x8, [x0]
; GISel-MOPS-NEXT:    ldrh w8, [x1, #8]
; GISel-MOPS-NEXT:    strh w8, [x0, #8]
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_inline_10_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x8, [x1]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldrh w9, [x1, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    strh w9, [x0, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x8, [x0]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_inline_10_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ldr x8, [x1]
; SDAG-MOPS-O2-NEXT:    ldrh w9, [x1, #8]
; SDAG-MOPS-O2-NEXT:    strh w9, [x0, #8]
; SDAG-MOPS-O2-NEXT:    str x8, [x0]
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 10, i1 true)
  ret void
}

define void @memcpy_inline_300(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memcpy_inline_300:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #16]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #16]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #32]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #32]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #48]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #48]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #64]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #64]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #80]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #80]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #96]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #96]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #112]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #112]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #128]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #128]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #144]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #144]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #160]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #160]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #176]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #176]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #192]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #192]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #208]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #208]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #224]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #224]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #240]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #240]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #256]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #256]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x1, #272]
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x0, #272]
; GISel-WITHOUT-MOPS-O0-NEXT:    add x8, x1, #284
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr q0, [x8]
; GISel-WITHOUT-MOPS-O0-NEXT:    add x8, x0, #284
; GISel-WITHOUT-MOPS-O0-NEXT:    str q0, [x8]
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memcpy_inline_300:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1]
; GISel-WITHOUT-MOPS-O3-NEXT:    add x8, x1, #284
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #16]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #16]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #32]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #32]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #48]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #48]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #64]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #64]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #80]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #80]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #96]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #96]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #112]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #112]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #128]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #128]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #144]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #144]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #160]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #160]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #176]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #176]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #192]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #192]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #208]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #208]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #224]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #224]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #240]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #240]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #256]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #256]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x1, #272]
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x0, #272]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr q0, [x8]
; GISel-WITHOUT-MOPS-O3-NEXT:    add x8, x0, #284
; GISel-WITHOUT-MOPS-O3-NEXT:    str q0, [x8]
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memcpy_inline_300:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    ldr q0, [x1]
; GISel-MOPS-O0-NEXT:    str q0, [x0]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #16]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #16]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #32]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #32]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #48]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #48]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #64]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #64]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #80]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #80]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #96]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #96]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #112]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #112]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #128]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #128]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #144]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #144]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #160]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #160]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #176]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #176]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #192]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #192]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #208]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #208]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #224]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #224]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #240]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #240]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #256]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #256]
; GISel-MOPS-O0-NEXT:    ldr q0, [x1, #272]
; GISel-MOPS-O0-NEXT:    str q0, [x0, #272]
; GISel-MOPS-O0-NEXT:    add x8, x1, #284
; GISel-MOPS-O0-NEXT:    ldr q0, [x8]
; GISel-MOPS-O0-NEXT:    add x8, x0, #284
; GISel-MOPS-O0-NEXT:    str q0, [x8]
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memcpy_inline_300:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    ldr q0, [x1]
; GISel-MOPS-O3-NEXT:    add x8, x1, #284
; GISel-MOPS-O3-NEXT:    str q0, [x0]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #16]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #16]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #32]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #32]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #48]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #48]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #64]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #64]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #80]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #80]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #96]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #96]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #112]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #112]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #128]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #128]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #144]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #144]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #160]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #160]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #176]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #176]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #192]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #192]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #208]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #208]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #224]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #224]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #240]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #240]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #256]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #256]
; GISel-MOPS-O3-NEXT:    ldr q0, [x1, #272]
; GISel-MOPS-O3-NEXT:    str q0, [x0, #272]
; GISel-MOPS-O3-NEXT:    ldr q0, [x8]
; GISel-MOPS-O3-NEXT:    add x8, x0, #284
; GISel-MOPS-O3-NEXT:    str q0, [x8]
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_inline_300:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldp q1, q0, [x1, #16]
; SDAG-WITHOUT-MOPS-O2-NEXT:    add x8, x1, #284
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q2, [x1]
; SDAG-WITHOUT-MOPS-O2-NEXT:    stp q1, q0, [x0, #16]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q2, [x0]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldp q1, q0, [x1, #80]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldp q2, q3, [x1, #48]
; SDAG-WITHOUT-MOPS-O2-NEXT:    stp q1, q0, [x0, #80]
; SDAG-WITHOUT-MOPS-O2-NEXT:    stp q2, q3, [x0, #48]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldp q1, q0, [x1, #144]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldp q2, q3, [x1, #112]
; SDAG-WITHOUT-MOPS-O2-NEXT:    stp q1, q0, [x0, #144]
; SDAG-WITHOUT-MOPS-O2-NEXT:    stp q2, q3, [x0, #112]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldp q1, q0, [x1, #208]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldp q2, q3, [x1, #176]
; SDAG-WITHOUT-MOPS-O2-NEXT:    stp q1, q0, [x0, #208]
; SDAG-WITHOUT-MOPS-O2-NEXT:    stp q2, q3, [x0, #176]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldp q3, q1, [x1, #256]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q0, [x8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q2, [x1, #240]
; SDAG-WITHOUT-MOPS-O2-NEXT:    add x8, x0, #284
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q0, [x8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    stp q3, q1, [x0, #256]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q2, [x0, #240]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_inline_300:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    mov w8, #300 // =0x12c
; SDAG-MOPS-O2-NEXT:    cpyfp [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    cpyfm [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    cpyfe [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 300, i1 false)
  ret void
}

define void @memcpy_inline_300_volatile(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memcpy_inline_300_volatile:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #16]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #16]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #32]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #32]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #48]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #48]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #64]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #64]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #80]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #80]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #96]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #96]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #112]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #112]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #128]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #128]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #144]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #144]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #160]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #160]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #176]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #176]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #192]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #192]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #208]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #208]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #224]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #224]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #240]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #240]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #256]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #256]
; GISel-WITHOUT-MOPS-NEXT:    ldr q0, [x1, #272]
; GISel-WITHOUT-MOPS-NEXT:    str q0, [x0, #272]
; GISel-WITHOUT-MOPS-NEXT:    ldr x8, [x1, #288]
; GISel-WITHOUT-MOPS-NEXT:    str x8, [x0, #288]
; GISel-WITHOUT-MOPS-NEXT:    ldr w8, [x1, #296]
; GISel-WITHOUT-MOPS-NEXT:    str w8, [x0, #296]
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memcpy_inline_300_volatile:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ldr q0, [x1]
; GISel-MOPS-NEXT:    str q0, [x0]
; GISel-MOPS-NEXT:    ldr q0, [x1, #16]
; GISel-MOPS-NEXT:    str q0, [x0, #16]
; GISel-MOPS-NEXT:    ldr q0, [x1, #32]
; GISel-MOPS-NEXT:    str q0, [x0, #32]
; GISel-MOPS-NEXT:    ldr q0, [x1, #48]
; GISel-MOPS-NEXT:    str q0, [x0, #48]
; GISel-MOPS-NEXT:    ldr q0, [x1, #64]
; GISel-MOPS-NEXT:    str q0, [x0, #64]
; GISel-MOPS-NEXT:    ldr q0, [x1, #80]
; GISel-MOPS-NEXT:    str q0, [x0, #80]
; GISel-MOPS-NEXT:    ldr q0, [x1, #96]
; GISel-MOPS-NEXT:    str q0, [x0, #96]
; GISel-MOPS-NEXT:    ldr q0, [x1, #112]
; GISel-MOPS-NEXT:    str q0, [x0, #112]
; GISel-MOPS-NEXT:    ldr q0, [x1, #128]
; GISel-MOPS-NEXT:    str q0, [x0, #128]
; GISel-MOPS-NEXT:    ldr q0, [x1, #144]
; GISel-MOPS-NEXT:    str q0, [x0, #144]
; GISel-MOPS-NEXT:    ldr q0, [x1, #160]
; GISel-MOPS-NEXT:    str q0, [x0, #160]
; GISel-MOPS-NEXT:    ldr q0, [x1, #176]
; GISel-MOPS-NEXT:    str q0, [x0, #176]
; GISel-MOPS-NEXT:    ldr q0, [x1, #192]
; GISel-MOPS-NEXT:    str q0, [x0, #192]
; GISel-MOPS-NEXT:    ldr q0, [x1, #208]
; GISel-MOPS-NEXT:    str q0, [x0, #208]
; GISel-MOPS-NEXT:    ldr q0, [x1, #224]
; GISel-MOPS-NEXT:    str q0, [x0, #224]
; GISel-MOPS-NEXT:    ldr q0, [x1, #240]
; GISel-MOPS-NEXT:    str q0, [x0, #240]
; GISel-MOPS-NEXT:    ldr q0, [x1, #256]
; GISel-MOPS-NEXT:    str q0, [x0, #256]
; GISel-MOPS-NEXT:    ldr q0, [x1, #272]
; GISel-MOPS-NEXT:    str q0, [x0, #272]
; GISel-MOPS-NEXT:    ldr x8, [x1, #288]
; GISel-MOPS-NEXT:    str x8, [x0, #288]
; GISel-MOPS-NEXT:    ldr w8, [x1, #296]
; GISel-MOPS-NEXT:    str w8, [x0, #296]
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memcpy_inline_300_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q0, [x1]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q1, [x1, #16]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q2, [x1, #32]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q3, [x1, #48]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q3, [x0, #48]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q2, [x0, #32]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q1, [x0, #16]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q0, [x0]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q0, [x1, #64]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q1, [x1, #80]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q2, [x1, #96]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q3, [x1, #112]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q3, [x0, #112]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q2, [x0, #96]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q1, [x0, #80]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q0, [x0, #64]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q0, [x1, #128]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q1, [x1, #144]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q2, [x1, #160]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q3, [x1, #176]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q3, [x0, #176]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q2, [x0, #160]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q1, [x0, #144]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q0, [x0, #128]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q0, [x1, #192]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q1, [x1, #208]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q2, [x1, #224]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q3, [x1, #240]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q3, [x0, #240]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q2, [x0, #224]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q1, [x0, #208]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q0, [x0, #192]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q0, [x1, #256]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr q1, [x1, #272]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x8, [x1, #288]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr w9, [x1, #296]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str w9, [x0, #296]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x8, [x0, #288]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q1, [x0, #272]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str q0, [x0, #256]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memcpy_inline_300_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    mov w8, #300 // =0x12c
; SDAG-MOPS-O2-NEXT:    cpyfp [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    cpyfm [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    cpyfe [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 300, i1 true)
  ret void
}

define void @memmove_0(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memmove_0:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memmove_0:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memmove_0:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memmove_0:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memmove.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 0, i1 false)
  ret void
}

define void @memmove_0_volatile(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memmove_0_volatile:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memmove_0_volatile:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memmove_0_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memmove_0_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memmove.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 0, i1 true)
  ret void
}

define void @memmove_10(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memmove_10:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x9, [x1]
; GISel-WITHOUT-MOPS-O0-NEXT:    ldrh w8, [x1, #8]
; GISel-WITHOUT-MOPS-O0-NEXT:    str x9, [x0]
; GISel-WITHOUT-MOPS-O0-NEXT:    strh w8, [x0, #8]
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memmove_10:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x8, [x1]
; GISel-WITHOUT-MOPS-O3-NEXT:    ldrh w9, [x1, #8]
; GISel-WITHOUT-MOPS-O3-NEXT:    str x8, [x0]
; GISel-WITHOUT-MOPS-O3-NEXT:    strh w9, [x0, #8]
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memmove_10:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    ldr x9, [x1]
; GISel-MOPS-O0-NEXT:    ldrh w8, [x1, #8]
; GISel-MOPS-O0-NEXT:    str x9, [x0]
; GISel-MOPS-O0-NEXT:    strh w8, [x0, #8]
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memmove_10:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    ldr x8, [x1]
; GISel-MOPS-O3-NEXT:    ldrh w9, [x1, #8]
; GISel-MOPS-O3-NEXT:    str x8, [x0]
; GISel-MOPS-O3-NEXT:    strh w9, [x0, #8]
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memmove_10:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldrh w8, [x1, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x9, [x1]
; SDAG-WITHOUT-MOPS-O2-NEXT:    strh w8, [x0, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x9, [x0]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memmove_10:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ldrh w8, [x1, #8]
; SDAG-MOPS-O2-NEXT:    ldr x9, [x1]
; SDAG-MOPS-O2-NEXT:    strh w8, [x0, #8]
; SDAG-MOPS-O2-NEXT:    str x9, [x0]
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memmove.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 10, i1 false)
  ret void
}

define void @memmove_10_volatile(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memmove_10_volatile:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w8, #10 // =0xa
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w2, w8
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memmove
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memmove_10_volatile:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w2, #10 // =0xa
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memmove
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memmove_10_volatile:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov w8, #10 // =0xa
; GISel-MOPS-O0-NEXT:    // kill: def $x8 killed $w8
; GISel-MOPS-O0-NEXT:    cpyp [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    cpym [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    cpye [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memmove_10_volatile:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    mov w8, #10 // =0xa
; GISel-MOPS-O3-NEXT:    cpyp [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    cpym [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    cpye [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memmove_10_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x8, [x1]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldrh w9, [x1, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    strh w9, [x0, #8]
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x8, [x0]
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memmove_10_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    ldr x8, [x1]
; SDAG-MOPS-O2-NEXT:    ldrh w9, [x1, #8]
; SDAG-MOPS-O2-NEXT:    strh w9, [x0, #8]
; SDAG-MOPS-O2-NEXT:    str x8, [x0]
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memmove.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 10, i1 true)
  ret void
}

define void @memmove_1000(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memmove_1000:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w8, #1000 // =0x3e8
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w2, w8
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memmove
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memmove_1000:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w2, #1000 // =0x3e8
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memmove
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memmove_1000:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov w8, #1000 // =0x3e8
; GISel-MOPS-O0-NEXT:    // kill: def $x8 killed $w8
; GISel-MOPS-O0-NEXT:    cpyp [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    cpym [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    cpye [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memmove_1000:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    mov w8, #1000 // =0x3e8
; GISel-MOPS-O3-NEXT:    cpyp [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    cpym [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    cpye [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memmove_1000:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w2, #1000 // =0x3e8
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memmove
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memmove_1000:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    mov w8, #1000 // =0x3e8
; SDAG-MOPS-O2-NEXT:    cpyp [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    cpym [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    cpye [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memmove.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 1000, i1 false)
  ret void
}

define void @memmove_1000_volatile(ptr %dst, ptr %src, i32 %value) {
; GISel-WITHOUT-MOPS-O0-LABEL: memmove_1000_volatile:
; GISel-WITHOUT-MOPS-O0:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O0-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w8, #1000 // =0x3e8
; GISel-WITHOUT-MOPS-O0-NEXT:    mov w2, w8
; GISel-WITHOUT-MOPS-O0-NEXT:    bl memmove
; GISel-WITHOUT-MOPS-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O0-NEXT:    ret
;
; GISel-WITHOUT-MOPS-O3-LABEL: memmove_1000_volatile:
; GISel-WITHOUT-MOPS-O3:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-O3-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-O3-NEXT:    mov w2, #1000 // =0x3e8
; GISel-WITHOUT-MOPS-O3-NEXT:    bl memmove
; GISel-WITHOUT-MOPS-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-O3-NEXT:    ret
;
; GISel-MOPS-O0-LABEL: memmove_1000_volatile:
; GISel-MOPS-O0:       // %bb.0: // %entry
; GISel-MOPS-O0-NEXT:    mov w8, #1000 // =0x3e8
; GISel-MOPS-O0-NEXT:    // kill: def $x8 killed $w8
; GISel-MOPS-O0-NEXT:    cpyp [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    cpym [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    cpye [x0]!, [x1]!, x8!
; GISel-MOPS-O0-NEXT:    ret
;
; GISel-MOPS-O3-LABEL: memmove_1000_volatile:
; GISel-MOPS-O3:       // %bb.0: // %entry
; GISel-MOPS-O3-NEXT:    mov w8, #1000 // =0x3e8
; GISel-MOPS-O3-NEXT:    cpyp [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    cpym [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    cpye [x0]!, [x1]!, x8!
; GISel-MOPS-O3-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memmove_1000_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    mov w2, #1000 // =0x3e8
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memmove
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memmove_1000_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    mov w8, #1000 // =0x3e8
; SDAG-MOPS-O2-NEXT:    cpyp [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    cpym [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    cpye [x0]!, [x1]!, x8!
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memmove.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 1000, i1 true)
  ret void
}

define void @memmove_n(ptr %dst, ptr %src, i64 %size, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memmove_n:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-NEXT:    bl memmove
; GISel-WITHOUT-MOPS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memmove_n:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    cpyp [x0]!, [x1]!, x2!
; GISel-MOPS-NEXT:    cpym [x0]!, [x1]!, x2!
; GISel-MOPS-NEXT:    cpye [x0]!, [x1]!, x2!
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memmove_n:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memmove
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memmove_n:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    cpyp [x0]!, [x1]!, x2!
; SDAG-MOPS-O2-NEXT:    cpym [x0]!, [x1]!, x2!
; SDAG-MOPS-O2-NEXT:    cpye [x0]!, [x1]!, x2!
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memmove.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 %size, i1 false)
  ret void
}

define void @memmove_n_volatile(ptr %dst, ptr %src, i64 %size, i32 %value) {
; GISel-WITHOUT-MOPS-LABEL: memmove_n_volatile:
; GISel-WITHOUT-MOPS:       // %bb.0: // %entry
; GISel-WITHOUT-MOPS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; GISel-WITHOUT-MOPS-NEXT:    .cfi_def_cfa_offset 16
; GISel-WITHOUT-MOPS-NEXT:    .cfi_offset w30, -16
; GISel-WITHOUT-MOPS-NEXT:    bl memmove
; GISel-WITHOUT-MOPS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; GISel-WITHOUT-MOPS-NEXT:    ret
;
; GISel-MOPS-LABEL: memmove_n_volatile:
; GISel-MOPS:       // %bb.0: // %entry
; GISel-MOPS-NEXT:    cpyp [x0]!, [x1]!, x2!
; GISel-MOPS-NEXT:    cpym [x0]!, [x1]!, x2!
; GISel-MOPS-NEXT:    cpye [x0]!, [x1]!, x2!
; GISel-MOPS-NEXT:    ret
;
; SDAG-WITHOUT-MOPS-O2-LABEL: memmove_n_volatile:
; SDAG-WITHOUT-MOPS-O2:       // %bb.0: // %entry
; SDAG-WITHOUT-MOPS-O2-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_def_cfa_offset 16
; SDAG-WITHOUT-MOPS-O2-NEXT:    .cfi_offset w30, -16
; SDAG-WITHOUT-MOPS-O2-NEXT:    bl memmove
; SDAG-WITHOUT-MOPS-O2-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; SDAG-WITHOUT-MOPS-O2-NEXT:    ret
;
; SDAG-MOPS-O2-LABEL: memmove_n_volatile:
; SDAG-MOPS-O2:       // %bb.0: // %entry
; SDAG-MOPS-O2-NEXT:    cpyp [x0]!, [x1]!, x2!
; SDAG-MOPS-O2-NEXT:    cpym [x0]!, [x1]!, x2!
; SDAG-MOPS-O2-NEXT:    cpye [x0]!, [x1]!, x2!
; SDAG-MOPS-O2-NEXT:    ret
entry:
  call void @llvm.memmove.p0.p0.i64(ptr align 1 %dst, ptr align 1 %src, i64 %size, i1 true)
  ret void
}