Run DCE after a LoopFlatten test to reduce spurious output [nfc]

[llvm-project.git] / llvm / test / CodeGen / X86 / wide-scalar-shift-legalization.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-NO-BMI2,X64-NO-SHLD,X64-NO-BMI2-NO-SHLD
; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-NO-BMI2,X64-SHLD,X64-NO-BMI2-HAVE-SHLD
; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-BMI2,X64-NO-SHLD,X64-HAVE-BMI2-NO-SHLD
; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-BMI2,X64-SHLD,X64-HAVE-BMI2-HAVE-SHLD
; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-NO-BMI2,X86-NO-SHLD,X86-NO-BMI2-NO-SHLD
; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-NO-BMI2,X86-SHLD,X86-NO-BMI2-HAVE-SHLD
; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-BMI2,X86-NO-SHLD,X86-HAVE-BMI2-NO-SHLD
; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-BMI2,X86-SHLD,X86-HAVE-BMI2-HAVE-SHLD

define void @lshr_4bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-LABEL: lshr_4bytes:
; X64-NO-BMI2:       # %bb.0:
; X64-NO-BMI2-NEXT:    movl (%rdi), %eax
; X64-NO-BMI2-NEXT:    movzbl (%rsi), %ecx
; X64-NO-BMI2-NEXT:    shrl %cl, %eax
; X64-NO-BMI2-NEXT:    movl %eax, (%rdx)
; X64-NO-BMI2-NEXT:    retq
;
; X64-BMI2-LABEL: lshr_4bytes:
; X64-BMI2:       # %bb.0:
; X64-BMI2-NEXT:    movzbl (%rsi), %eax
; X64-BMI2-NEXT:    shrxl %eax, (%rdi), %eax
; X64-BMI2-NEXT:    movl %eax, (%rdx)
; X64-BMI2-NEXT:    retq
;
; X86-NO-BMI2-LABEL: lshr_4bytes:
; X86-NO-BMI2:       # %bb.0:
; X86-NO-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-NEXT:    movl (%edx), %edx
; X86-NO-BMI2-NEXT:    movzbl (%ecx), %ecx
; X86-NO-BMI2-NEXT:    shrl %cl, %edx
; X86-NO-BMI2-NEXT:    movl %edx, (%eax)
; X86-NO-BMI2-NEXT:    retl
;
; X86-BMI2-LABEL: lshr_4bytes:
; X86-BMI2:       # %bb.0:
; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-BMI2-NEXT:    movzbl (%edx), %edx
; X86-BMI2-NEXT:    shrxl %edx, (%ecx), %ecx
; X86-BMI2-NEXT:    movl %ecx, (%eax)
; X86-BMI2-NEXT:    retl
  %src = load i32, ptr %src.ptr, align 1
  %bitOff = load i32, ptr %bitOff.ptr, align 1
  %res = lshr i32 %src, %bitOff
  store i32 %res, ptr %dst, align 1
  ret void
}
define void @shl_4bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-LABEL: shl_4bytes:
; X64-NO-BMI2:       # %bb.0:
; X64-NO-BMI2-NEXT:    movl (%rdi), %eax
; X64-NO-BMI2-NEXT:    movzbl (%rsi), %ecx
; X64-NO-BMI2-NEXT:    shll %cl, %eax
; X64-NO-BMI2-NEXT:    movl %eax, (%rdx)
; X64-NO-BMI2-NEXT:    retq
;
; X64-BMI2-LABEL: shl_4bytes:
; X64-BMI2:       # %bb.0:
; X64-BMI2-NEXT:    movzbl (%rsi), %eax
; X64-BMI2-NEXT:    shlxl %eax, (%rdi), %eax
; X64-BMI2-NEXT:    movl %eax, (%rdx)
; X64-BMI2-NEXT:    retq
;
; X86-NO-BMI2-LABEL: shl_4bytes:
; X86-NO-BMI2:       # %bb.0:
; X86-NO-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-NEXT:    movl (%edx), %edx
; X86-NO-BMI2-NEXT:    movzbl (%ecx), %ecx
; X86-NO-BMI2-NEXT:    shll %cl, %edx
; X86-NO-BMI2-NEXT:    movl %edx, (%eax)
; X86-NO-BMI2-NEXT:    retl
;
; X86-BMI2-LABEL: shl_4bytes:
; X86-BMI2:       # %bb.0:
; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-BMI2-NEXT:    movzbl (%edx), %edx
; X86-BMI2-NEXT:    shlxl %edx, (%ecx), %ecx
; X86-BMI2-NEXT:    movl %ecx, (%eax)
; X86-BMI2-NEXT:    retl
  %src = load i32, ptr %src.ptr, align 1
  %bitOff = load i32, ptr %bitOff.ptr, align 1
  %res = shl i32 %src, %bitOff
  store i32 %res, ptr %dst, align 1
  ret void
}
define void @ashr_4bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-LABEL: ashr_4bytes:
; X64-NO-BMI2:       # %bb.0:
; X64-NO-BMI2-NEXT:    movl (%rdi), %eax
; X64-NO-BMI2-NEXT:    movzbl (%rsi), %ecx
; X64-NO-BMI2-NEXT:    sarl %cl, %eax
; X64-NO-BMI2-NEXT:    movl %eax, (%rdx)
; X64-NO-BMI2-NEXT:    retq
;
; X64-BMI2-LABEL: ashr_4bytes:
; X64-BMI2:       # %bb.0:
; X64-BMI2-NEXT:    movzbl (%rsi), %eax
; X64-BMI2-NEXT:    sarxl %eax, (%rdi), %eax
; X64-BMI2-NEXT:    movl %eax, (%rdx)
; X64-BMI2-NEXT:    retq
;
; X86-NO-BMI2-LABEL: ashr_4bytes:
; X86-NO-BMI2:       # %bb.0:
; X86-NO-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-NEXT:    movl (%edx), %edx
; X86-NO-BMI2-NEXT:    movzbl (%ecx), %ecx
; X86-NO-BMI2-NEXT:    sarl %cl, %edx
; X86-NO-BMI2-NEXT:    movl %edx, (%eax)
; X86-NO-BMI2-NEXT:    retl
;
; X86-BMI2-LABEL: ashr_4bytes:
; X86-BMI2:       # %bb.0:
; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-BMI2-NEXT:    movzbl (%edx), %edx
; X86-BMI2-NEXT:    sarxl %edx, (%ecx), %ecx
; X86-BMI2-NEXT:    movl %ecx, (%eax)
; X86-BMI2-NEXT:    retl
  %src = load i32, ptr %src.ptr, align 1
  %bitOff = load i32, ptr %bitOff.ptr, align 1
  %res = ashr i32 %src, %bitOff
  store i32 %res, ptr %dst, align 1
  ret void
}

define void @lshr_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-LABEL: lshr_8bytes:
; X64-NO-BMI2:       # %bb.0:
; X64-NO-BMI2-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-NEXT:    movzbl (%rsi), %ecx
; X64-NO-BMI2-NEXT:    shrq %cl, %rax
; X64-NO-BMI2-NEXT:    movq %rax, (%rdx)
; X64-NO-BMI2-NEXT:    retq
;
; X64-BMI2-LABEL: lshr_8bytes:
; X64-BMI2:       # %bb.0:
; X64-BMI2-NEXT:    movzbl (%rsi), %eax
; X64-BMI2-NEXT:    shrxq %rax, (%rdi), %rax
; X64-BMI2-NEXT:    movq %rax, (%rdx)
; X64-BMI2-NEXT:    retq
;
; X86-NO-BMI2-NO-SHLD-LABEL: lshr_8bytes:
; X86-NO-BMI2-NO-SHLD:       # %bb.0:
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%ecx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl (%eax), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    notb %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%esi,%esi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    xorl %ecx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    testb $32, %al
; X86-NO-BMI2-NO-SHLD-NEXT:    cmovnel %esi, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    cmovel %esi, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, (%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    retl
;
; X86-NO-BMI2-HAVE-SHLD-LABEL: lshr_8bytes:
; X86-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esi), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%esi), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %esi, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    xorl %esi, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    testb $32, %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    cmovnel %edi, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    cmovel %edi, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, 4(%eax)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%eax)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-NO-SHLD-LABEL: lshr_8bytes:
; X86-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%edx), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, (%edx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notb %bl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    leal (%esi,%esi), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ebx, %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edx, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %esi, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    xorl %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    testb $32, %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    cmovnel %edx, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    cmovel %edx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, 4(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, (%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_8bytes:
; X86-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esi), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%esi), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %esi, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    xorl %edi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    testb $32, %cl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    cmovnel %esi, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    cmovel %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, 4(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    retl
  %src = load i64, ptr %src.ptr, align 1
  %bitOff = load i64, ptr %bitOff.ptr, align 1
  %res = lshr i64 %src, %bitOff
  store i64 %res, ptr %dst, align 1
  ret void
}
define void @shl_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-LABEL: shl_8bytes:
; X64-NO-BMI2:       # %bb.0:
; X64-NO-BMI2-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-NEXT:    movzbl (%rsi), %ecx
; X64-NO-BMI2-NEXT:    shlq %cl, %rax
; X64-NO-BMI2-NEXT:    movq %rax, (%rdx)
; X64-NO-BMI2-NEXT:    retq
;
; X64-BMI2-LABEL: shl_8bytes:
; X64-BMI2:       # %bb.0:
; X64-BMI2-NEXT:    movzbl (%rsi), %eax
; X64-BMI2-NEXT:    shlxq %rax, (%rdi), %rax
; X64-BMI2-NEXT:    movq %rax, (%rdx)
; X64-BMI2-NEXT:    retq
;
; X86-NO-BMI2-NO-SHLD-LABEL: shl_8bytes:
; X86-NO-BMI2-NO-SHLD:       # %bb.0:
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%ecx), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%ecx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl (%eax), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    notb %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    xorl %ecx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    testb $32, %al
; X86-NO-BMI2-NO-SHLD-NEXT:    cmovnel %esi, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    cmovel %esi, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, (%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, 4(%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    retl
;
; X86-NO-BMI2-HAVE-SHLD-LABEL: shl_8bytes:
; X86-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%edx), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%edx), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %esi, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    xorl %esi, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    testb $32, %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    cmovnel %edi, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    cmovel %edi, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, 4(%eax)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, (%eax)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-NO-SHLD-LABEL: shl_8bytes:
; X86-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%edx), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, 4(%edx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notb %bl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %esi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ebx, %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    xorl %edx, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    testb $32, %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    cmovnel %edi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    cmovel %edi, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, (%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, 4(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-HAVE-SHLD-LABEL: shl_8bytes:
; X86-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%edx), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %esi, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    xorl %edi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    testb $32, %cl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    cmovnel %esi, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    cmovel %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, 4(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, (%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    retl
  %src = load i64, ptr %src.ptr, align 1
  %bitOff = load i64, ptr %bitOff.ptr, align 1
  %res = shl i64 %src, %bitOff
  store i64 %res, ptr %dst, align 1
  ret void
}
define void @ashr_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-LABEL: ashr_8bytes:
; X64-NO-BMI2:       # %bb.0:
; X64-NO-BMI2-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-NEXT:    movzbl (%rsi), %ecx
; X64-NO-BMI2-NEXT:    sarq %cl, %rax
; X64-NO-BMI2-NEXT:    movq %rax, (%rdx)
; X64-NO-BMI2-NEXT:    retq
;
; X64-BMI2-LABEL: ashr_8bytes:
; X64-BMI2:       # %bb.0:
; X64-BMI2-NEXT:    movzbl (%rsi), %eax
; X64-BMI2-NEXT:    sarxq %rax, (%rdi), %rax
; X64-BMI2-NEXT:    movq %rax, (%rdx)
; X64-BMI2-NEXT:    retq
;
; X86-NO-BMI2-NO-SHLD-LABEL: ashr_8bytes:
; X86-NO-BMI2-NO-SHLD:       # %bb.0:
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%ecx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl (%eax), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    notb %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%esi,%esi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    sarl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    sarl $31, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    testb $32, %al
; X86-NO-BMI2-NO-SHLD-NEXT:    cmovnel %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    cmovel %ebx, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, 4(%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, (%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    retl
;
; X86-NO-BMI2-HAVE-SHLD-LABEL: ashr_8bytes:
; X86-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esi), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%esi), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    sarl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %esi, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    sarl $31, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    testb $32, %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    cmovnel %edi, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    cmovel %edi, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, 4(%eax)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%eax)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-NO-SHLD-LABEL: ashr_8bytes:
; X86-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%esi), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%edx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %edx, (%esi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notb %bl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ebx, %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    sarxl %edx, %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    sarl $31, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    testb $32, %dl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    cmovnel %esi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    cmovel %esi, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, (%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_8bytes:
; X86-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esi), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%esi), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %esi, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    sarxl %ecx, %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    sarl $31, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    testb $32, %cl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    cmovnel %edi, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    cmovel %edi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, 4(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    retl
  %src = load i64, ptr %src.ptr, align 1
  %bitOff = load i64, ptr %bitOff.ptr, align 1
  %res = ashr i64 %src, %bitOff
  store i64 %res, ptr %dst, align 1
  ret void
}

define void @lshr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-NO-SHLD-LABEL: lshr_16bytes:
; X64-NO-BMI2-NO-SHLD:       # %bb.0:
; X64-NO-BMI2-NO-SHLD-NEXT:    movq (%rdi), %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movzbl (%rsi), %eax
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    notb %cl
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%rdi,%rdi), %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r8, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    xorl %ecx, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    testb $64, %al
; X64-NO-BMI2-NO-SHLD-NEXT:    cmovneq %rdi, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    cmoveq %rdi, %rcx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rcx, 8(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rsi, (%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    retq
;
; X64-NO-BMI2-HAVE-SHLD-LABEL: lshr_16bytes:
; X64-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%rsi), %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rdi, %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    xorl %edi, %edi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    testb $64, %cl
; X64-NO-BMI2-HAVE-SHLD-NEXT:    cmovneq %rsi, %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    cmoveq %rsi, %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, 8(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rax, (%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-NO-SHLD-LABEL: lshr_16bytes:
; X64-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%rsi), %ecx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, (%rdi), %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %edi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    notb %dil
; X64-HAVE-BMI2-NO-SHLD-NEXT:    leaq (%rax,%rax), %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rdi, %r8, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %rsi, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %rax, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    xorl %esi, %esi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    testb $64, %cl
; X64-HAVE-BMI2-NO-SHLD-NEXT:    cmovneq %rax, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    cmoveq %rax, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rsi, 8(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, (%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_16bytes:
; X64-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%rsi), %ecx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rdi, %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rcx, %rdi, %rsi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    xorl %edi, %edi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    testb $64, %cl
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    cmovneq %rsi, %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    cmoveq %rsi, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, 8(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rax, (%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    retq
;
; X86-NO-BMI2-NO-SHLD-LABEL: lshr_16bytes:
; X86-NO-BMI2-NO-SHLD:       # %bb.0:
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    subl $36, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%ecx), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 8(%ecx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 12(%ecx), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb (%eax), %ah
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %al
; X86-NO-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X86-NO-BMI2-NO-SHLD-NEXT:    shrb $3, %ah
; X86-NO-BMI2-NO-SHLD-NEXT:    andb $15, %ah
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl %ah, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 8(%esp,%ebp), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    notb %dl
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 12(%esp,%ebp), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%esp,%ebp), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %esi, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, (%esp) # 4-byte Folded Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 16(%esp,%ebp), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ebx,%ebx), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl (%esp), %ebp # 4-byte Folded Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, 12(%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, 8(%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, (%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, 4(%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    addl $36, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    retl
;
; X86-NO-BMI2-HAVE-SHLD-LABEL: lshr_16bytes:
; X86-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    subl $32, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%ecx), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 8(%ecx), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 12(%ecx), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movb (%eax), %ah
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movb %ah, %al
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andb $7, %al
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrb $3, %ah
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andb $15, %ah
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movzbl %ah, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%esp,%ebp), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    notb %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 8(%esp,%ebp), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ebx,%ebx), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edx, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp,%ebp), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 12(%esp,%ebp), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %ebp, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, 8(%ecx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %esi, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, 12(%eax)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%eax)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, 4(%eax)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    addl $32, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-NO-SHLD-LABEL: lshr_16bytes:
; X86-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    subl $32, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%ecx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 8(%ecx), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 12(%ecx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%eax), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, (%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrb $3, %bl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andb $15, %bl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movzbl %bl, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%esp,%esi), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 8(%esp,%esi), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notb %dl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    leal (%ebx,%ebx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebp, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, (%esp,%esi), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebp, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ebx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 12(%esp,%esi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %esi, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %esi, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebx, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 12(%esi)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, 8(%esi)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, (%esi)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%esi)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl $32, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_16bytes:
; X86-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    subl $32, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%ecx), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 8(%ecx), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 12(%ecx), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%eax), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andb $7, %cl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrb $3, %al
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andb $15, %al
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl %al, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%esp,%ebx), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %eax, %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    notb %dl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 8(%esp,%ebx), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    leal (%edi,%edi), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %edx, %esi, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %ebp, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp,%ebx), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 12(%esp,%ebx), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %ebx, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, 8(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %ebx, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, 12(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %eax, %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, (%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, 4(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl $32, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    retl
  %src = load i128, ptr %src.ptr, align 1
  %bitOff = load i128, ptr %bitOff.ptr, align 1
  %res = lshr i128 %src, %bitOff
  store i128 %res, ptr %dst, align 1
  ret void
}
define void @shl_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-NO-SHLD-LABEL: shl_16bytes:
; X64-NO-BMI2-NO-SHLD:       # %bb.0:
; X64-NO-BMI2-NO-SHLD-NEXT:    movq (%rdi), %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movzbl (%rsi), %eax
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    notb %cl
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r8, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %rdi, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    xorl %ecx, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    testb $64, %al
; X64-NO-BMI2-NO-SHLD-NEXT:    cmovneq %r8, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    cmoveq %r8, %rcx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rcx, (%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rsi, 8(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    retq
;
; X64-NO-BMI2-HAVE-SHLD-LABEL: shl_16bytes:
; X64-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%rsi), %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rax, %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %rax, %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    xorl %eax, %eax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    testb $64, %cl
; X64-NO-BMI2-HAVE-SHLD-NEXT:    cmovneq %rsi, %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    cmoveq %rsi, %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, 8(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rax, (%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-NO-SHLD-LABEL: shl_16bytes:
; X64-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq (%rdi), %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%rsi), %ecx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rcx, 8(%rdi), %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %edi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    notb %dil
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rcx, %rax, %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrq %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rdi, %rax, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %rsi, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    xorl %esi, %esi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    testb $64, %cl
; X64-HAVE-BMI2-NO-SHLD-NEXT:    cmovneq %r8, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    cmoveq %r8, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rsi, (%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rax, 8(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-HAVE-SHLD-LABEL: shl_16bytes:
; X64-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%rsi), %ecx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %rax, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %rcx, %rax, %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    xorl %esi, %esi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    testb $64, %cl
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    cmovneq %rax, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    cmoveq %rax, %rsi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, 8(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rsi, (%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    retq
;
; X86-NO-BMI2-NO-SHLD-LABEL: shl_16bytes:
; X86-NO-BMI2-NO-SHLD:       # %bb.0:
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    subl $40, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%ecx), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 8(%ecx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 12(%ecx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl (%eax), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X86-NO-BMI2-NO-SHLD-NEXT:    shrb $3, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    andb $15, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    negb %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    movsbl %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 24(%esp,%ebp), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 28(%esp,%ebp), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    notb %dl
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 36(%esp,%ebp), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 32(%esp,%ebp), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, (%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, 8(%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, 12(%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, 4(%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    addl $40, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    retl
;
; X86-NO-BMI2-HAVE-SHLD-LABEL: shl_16bytes:
; X86-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    subl $32, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%ecx), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 8(%ecx), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 12(%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, (%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andb $7, %al
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrb $3, %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andb $15, %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    negb %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movsbl %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 24(%esp,%ebp), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    notb %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 20(%esp,%ebp), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edx, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 16(%esp,%ebp), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 28(%esp,%ebp), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %ebx, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, 12(%ebx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %edx, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%ebx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, 4(%ebx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, 8(%ebx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    addl $32, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-NO-SHLD-LABEL: shl_16bytes:
; X86-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    subl $32, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%ecx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 8(%ecx), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 12(%ecx), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%eax), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andb $7, %bl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, (%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrb $3, %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andb $15, %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    negb %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movsbl %cl, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 16(%esp,%edx), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 20(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ebx, %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ebx, %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notb %al
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ebx, 28(%esp,%edx), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 24(%esp,%edx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ebx, %edx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %edx, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %esi, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ecx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, (%ecx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 8(%ecx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, 12(%ecx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, 4(%ecx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl $32, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-HAVE-SHLD-LABEL: shl_16bytes:
; X86-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    subl $32, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%ecx), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 8(%ecx), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 12(%ecx), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%eax), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andb $7, %cl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, (%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrb $3, %al
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andb $15, %al
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    negb %al
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movsbl %al, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 24(%esp,%ebx), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %edi, %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    notb %dl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 20(%esp,%ebx), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %edx, %esi, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %ebp, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 16(%esp,%ebx), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 28(%esp,%ebx), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %edi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, 12(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %ebp, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, (%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %ebp, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 4(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, 8(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl $32, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    retl
  %src = load i128, ptr %src.ptr, align 1
  %bitOff = load i128, ptr %bitOff.ptr, align 1
  %res = shl i128 %src, %bitOff
  store i128 %res, ptr %dst, align 1
  ret void
}
define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-NO-SHLD-LABEL: ashr_16bytes:
; X64-NO-BMI2-NO-SHLD:       # %bb.0:
; X64-NO-BMI2-NO-SHLD-NEXT:    movq (%rdi), %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movzbl (%rsi), %eax
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    notb %cl
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%rdi,%rdi), %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r8, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    sarq %cl, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    sarq $63, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    testb $64, %al
; X64-NO-BMI2-NO-SHLD-NEXT:    cmovneq %r8, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    cmoveq %r8, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, 8(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rsi, (%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    retq
;
; X64-NO-BMI2-HAVE-SHLD-LABEL: ashr_16bytes:
; X64-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%rsi), %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    sarq %cl, %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rdi, %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    sarq $63, %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    testb $64, %cl
; X64-NO-BMI2-HAVE-SHLD-NEXT:    cmovneq %rsi, %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    cmoveq %rsi, %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, 8(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rax, (%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-NO-SHLD-LABEL: ashr_16bytes:
; X64-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%rsi), %ecx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, (%rdi), %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %edi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    notb %dil
; X64-HAVE-BMI2-NO-SHLD-NEXT:    leaq (%rax,%rax), %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rdi, %r8, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %rsi, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    sarxq %rcx, %rax, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    sarq $63, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    testb $64, %cl
; X64-HAVE-BMI2-NO-SHLD-NEXT:    cmovneq %rsi, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    cmoveq %rsi, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rax, 8(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, (%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_16bytes:
; X64-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%rsi), %ecx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rdi, %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    sarxq %rcx, %rdi, %rsi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    sarq $63, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    testb $64, %cl
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    cmovneq %rsi, %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    cmoveq %rsi, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, 8(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rax, (%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    retq
;
; X86-NO-BMI2-NO-SHLD-LABEL: ashr_16bytes:
; X86-NO-BMI2-NO-SHLD:       # %bb.0:
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    subl $36, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%ecx), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 8(%ecx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 12(%ecx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl (%eax), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    sarl $31, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X86-NO-BMI2-NO-SHLD-NEXT:    shrb $3, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    andb $15, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 8(%esp,%ebp), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    notb %dl
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 12(%esp,%ebp), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%esp,%ebp), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %esi, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, (%esp) # 4-byte Folded Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 16(%esp,%ebp), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ebx,%ebx), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl (%esp), %ebp # 4-byte Folded Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    sarl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, 12(%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, 8(%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, (%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, 4(%edx)
; X86-NO-BMI2-NO-SHLD-NEXT:    addl $36, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    retl
;
; X86-NO-BMI2-HAVE-SHLD-LABEL: ashr_16bytes:
; X86-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    subl $32, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%ecx), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 8(%ecx), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 12(%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    sarl $31, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andb $7, %al
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrb $3, %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andb $15, %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movzbl %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%esp,%ebp), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    notb %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 8(%esp,%ebp), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ebx,%ebx), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edx, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp,%ebp), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 12(%esp,%ebp), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %ebp, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, 8(%ecx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %esi, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    sarl %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, 12(%eax)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%eax)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, 4(%eax)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    addl $32, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-NO-SHLD-LABEL: ashr_16bytes:
; X86-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    subl $32, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%ecx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 8(%ecx), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 12(%ecx), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%eax), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, (%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    sarl $31, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrb $3, %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andb $15, %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movzbl %cl, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%esp,%esi), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 8(%esp,%esi), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notb %dl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    leal (%ebx,%ebx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebp, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, (%esp,%esi), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebp, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ebx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 12(%esp,%esi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    sarxl %eax, %esi, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %esi, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebx, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 12(%esi)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, 8(%esi)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, (%esi)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%esi)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl $32, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_16bytes:
; X86-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    subl $32, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%ecx), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%ecx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 8(%ecx), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 12(%ecx), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%eax), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    sarl $31, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andb $7, %cl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrb $3, %al
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andb $15, %al
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl %al, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%esp,%ebx), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %eax, %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    notb %dl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 8(%esp,%ebx), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    leal (%edi,%edi), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %edx, %esi, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %ebp, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp,%ebx), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 12(%esp,%ebx), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %ebx, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, 8(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    sarxl %ecx, %ebx, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, 12(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %eax, %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, (%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, 4(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl $32, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    retl
  %src = load i128, ptr %src.ptr, align 1
  %bitOff = load i128, ptr %bitOff.ptr, align 1
  %res = ashr i128 %src, %bitOff
  store i128 %res, ptr %dst, align 1
  ret void
}

define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-NO-SHLD-LABEL: lshr_32bytes:
; X64-NO-BMI2-NO-SHLD:       # %bb.0:
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 24(%rdi), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movzbl (%rsi), %esi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %eax
; X64-NO-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X64-NO-BMI2-NO-SHLD-NEXT:    shrb $3, %sil
; X64-NO-BMI2-NO-SHLD-NEXT:    movzbl %sil, %r9d
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -64(%rsp,%r9), %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -56(%rsp,%r9), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %esi
; X64-NO-BMI2-NO-SHLD-NEXT:    notb %sil
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -48(%rsp,%r9), %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%rbx,%rbx), %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r11, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    addq %rdi, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r10, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -40(%rsp,%r9), %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%r9,%r9), %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %rbx, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r9, 24(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r10, 16(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, (%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r8, 8(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    retq
;
; X64-NO-BMI2-HAVE-SHLD-LABEL: lshr_32bytes:
; X64-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 24(%rdi), %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%rsi), %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %eax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    andb $7, %al
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrb $3, %sil
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movzbl %sil, %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -64(%rsp,%rsi), %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -56(%rsp,%rsi), %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r8, %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    notb %cl
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -48(%rsp,%rsi), %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    leaq (%r10,%r10), %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    orq %r9, %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -40(%rsp,%rsi), %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rsi, %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r8, %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r10, 16(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rsi, 24(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, (%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r11, 8(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-NO-SHLD-LABEL: lshr_32bytes:
; X64-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq (%rdi), %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 24(%rdi), %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%rsi), %esi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, %eax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrb $3, %sil
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movzbl %sil, %ecx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -56(%rsp,%rcx), %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -48(%rsp,%rcx), %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rax, %rsi, %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rax, -64(%rsp,%rcx), %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rax, %rdi, %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -40(%rsp,%rcx), %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rax, %rcx, %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    # kill: def $al killed $al killed $rax def $rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    notb %al
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rdi, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %rdi, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r8, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rsi, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %rsi, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r9, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rcx, %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %rcx, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r10, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r11, 24(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rax, 16(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rsi, (%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, 8(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_32bytes:
; X64-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 24(%rdi), %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%rsi), %esi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ecx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    andb $7, %cl
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrb $3, %sil
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl %sil, %eax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -56(%rsp,%rax), %rsi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rcx, %rsi, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -40(%rsp,%rax), %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rcx, %r8, %r9
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %r10d
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    notb %r10b
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -48(%rsp,%rax), %r11
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    leaq (%r11,%r11), %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %r10, %rbx, %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -64(%rsp,%rax), %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    orq %rdi, %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r8, %r11
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    # kill: def $cl killed $cl killed $rcx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rsi, %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r11, 16(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r9, 24(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rax, (%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r10, 8(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    retq
;
; X86-NO-BMI2-NO-SHLD-LABEL: lshr_32bytes:
; X86-NO-BMI2-NO-SHLD:       # %bb.0:
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    subl $88, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%edi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%edi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 8(%edi), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 12(%edi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 16(%edi), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movb (%ecx), %ch
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 20(%edi), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 24(%edi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 28(%edi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ch, %al
; X86-NO-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X86-NO-BMI2-NO-SHLD-NEXT:    shrb $3, %ch
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl %ch, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 24(%esp,%edi), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 28(%esp,%edi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %al, %ah
; X86-NO-BMI2-NO-SHLD-NEXT:    notb %ah
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 32(%esp,%edi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%edi,%edi), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebx, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %esi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 36(%esp,%ebx), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 40(%esp,%ebx), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %esi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 44(%esp,%edx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 48(%esp,%edx), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%esi,%esi), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %al, %ch
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %al, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %edi, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %dl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %al, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 52(%esp,%esi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ebx,%ebx), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %dl, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %eax, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ch, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, 28(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, 24(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, 16(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 20(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, 8(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 12(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, (%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    addl $88, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    retl
;
; X86-NO-BMI2-HAVE-SHLD-LABEL: lshr_32bytes:
; X86-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    subl $92, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%edi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%edi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 8(%edi), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 12(%edi), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 16(%edi), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 20(%edi), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 24(%edi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 28(%edi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andb $7, %al
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrb $3, %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movzbl %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 32(%esp,%ebp), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    notb %dl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 36(%esp,%ebp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 40(%esp,%ebp), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 44(%esp,%ebp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 48(%esp,%ebp), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 52(%esp,%ebp), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ebx,%ebx), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, (%esp) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 28(%esp,%ebp), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 56(%esp,%ebp), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, 24(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %ebx, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, 28(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 16(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 8(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, 20(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 12(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 4(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    addl $92, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-NO-SHLD-LABEL: lshr_32bytes:
; X86-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    subl $84, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%edi), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%edi), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 8(%edi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 12(%edi), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 16(%edi), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%ecx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 20(%edi), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 24(%edi), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 28(%edi), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andb $7, %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrb $3, %dl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movzbl %dl, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 24(%esp,%edi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 28(%esp,%edi), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %esi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notb %dl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    leal (%eax,%eax), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %ebp, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, 20(%esp,%edi), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %esi, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 36(%esp,%edi), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %ebx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 32(%esp,%edi), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ebp, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %ebp, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 44(%esp,%edi), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    leal (%ebp,%ebp), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 40(%esp,%edi), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ebx, %eax, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %esi, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ebx, (%esp), %esi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %eax, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %eax, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl (%esp), %esi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ebx, %ebp, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 48(%esp,%edi), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ebx, %edi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %eax, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, 28(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, 24(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, 16(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 20(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 8(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 12(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, (%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl $84, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_32bytes:
; X86-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    subl $88, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%edi), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%edi), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 8(%edi), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 12(%edi), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 16(%edi), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%eax), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 20(%edi), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 24(%edi), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 28(%edi), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andb $7, %cl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrb $3, %al
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl %al, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 28(%esp,%ebx), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %eax, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    notb %dl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 32(%esp,%ebx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %edx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %eax, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 40(%esp,%ebx), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    leal (%ebp,%ebp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %edx, %eax, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 36(%esp,%ebx), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %eax, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %eax, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 48(%esp,%ebx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    leal (%esi,%esi), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %edx, %edi, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 44(%esp,%ebx), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %edx, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %edi, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %eax, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 24(%esp,%ebx), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 52(%esp,%ebx), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %eax, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, 24(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %eax, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 28(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, 16(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, 8(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %eax, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 20(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 12(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 4(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl $88, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    retl
  %src = load i256, ptr %src.ptr, align 1
  %bitOff = load i256, ptr %bitOff.ptr, align 1
  %res = lshr i256 %src, %bitOff
  store i256 %res, ptr %dst, align 1
  ret void
}
define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-NO-SHLD-LABEL: shl_32bytes:
; X64-NO-BMI2-NO-SHLD:       # %bb.0:
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 24(%rdi), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movzbl (%rsi), %esi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %eax
; X64-NO-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X64-NO-BMI2-NO-SHLD-NEXT:    shrb $3, %sil
; X64-NO-BMI2-NO-SHLD-NEXT:    negb %sil
; X64-NO-BMI2-NO-SHLD-NEXT:    movsbq %sil, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -32(%rsp,%r10), %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -24(%rsp,%r10), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %esi
; X64-NO-BMI2-NO-SHLD-NEXT:    notb %sil
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r8, %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r11, %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -8(%rsp,%r10), %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -16(%rsp,%r10), %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r10, %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r11, %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r10, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r8, (%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, 16(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rbx, 24(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r9, 8(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    retq
;
; X64-NO-BMI2-HAVE-SHLD-LABEL: shl_32bytes:
; X64-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 24(%rdi), %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%rsi), %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %eax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    andb $7, %al
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrb $3, %sil
; X64-NO-BMI2-HAVE-SHLD-NEXT:    negb %sil
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movsbq %sil, %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -16(%rsp,%rsi), %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    notb %cl
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -32(%rsp,%rsi), %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -24(%rsp,%rsi), %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r10, %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    orq %r8, %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -8(%rsp,%rsi), %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %rdi, %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %r9, %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rsi, 24(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r9, (%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r10, 8(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r11, 16(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-NO-SHLD-LABEL: shl_32bytes:
; X64-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq (%rdi), %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 24(%rdi), %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%rsi), %esi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, %eax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrb $3, %sil
; X64-HAVE-BMI2-NO-SHLD-NEXT:    negb %sil
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movsbq %sil, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -32(%rsp,%rsi), %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -24(%rsp,%rsi), %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %rcx, %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, -8(%rsp,%rsi), %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -16(%rsp,%rsi), %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %rsi, %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %rdi, %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    # kill: def $al killed $al killed $rax def $rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    notb %al
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrq %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rax, %rdi, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r8, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrq %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rax, %rsi, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r9, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrq %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rax, %rcx, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r10, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r11, (%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rax, 16(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rsi, 24(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, 8(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-HAVE-SHLD-LABEL: shl_32bytes:
; X64-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 24(%rdi), %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%rsi), %esi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ecx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    andb $7, %cl
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrb $3, %sil
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    negb %sil
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movsbq %sil, %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -16(%rsp,%rax), %rsi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %rcx, %rsi, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -32(%rsp,%rax), %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %rcx, %r8, %r9
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %r10d
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    notb %r10b
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -24(%rsp,%rax), %r11
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r11, %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %r10, %rbx, %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    orq %rdi, %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -8(%rsp,%rax), %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %rsi, %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    # kill: def $cl killed $cl killed $rcx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %r8, %r11
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rax, 24(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r9, (%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r11, 8(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r10, 16(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    retq
;
; X86-NO-BMI2-NO-SHLD-LABEL: shl_32bytes:
; X86-NO-BMI2-NO-SHLD:       # %bb.0:
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    subl $88, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%edi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%edi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 8(%edi), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 12(%edi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 16(%edi), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 20(%edi), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 24(%edi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 28(%edi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X86-NO-BMI2-NO-SHLD-NEXT:    shrb $3, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    negb %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    movsbl %cl, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 56(%esp,%ecx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 60(%esp,%ecx), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %al, %ah
; X86-NO-BMI2-NO-SHLD-NEXT:    notb %ah
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %esi, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 68(%esp,%ebp), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 64(%esp,%ebp), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %esi, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 76(%esp,%ebp), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 72(%esp,%ebp), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %edx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 84(%esp,%edi), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 80(%esp,%edi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edx, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, (%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, 24(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, 28(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 16(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, 20(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 8(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 12(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    addl $88, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    retl
;
; X86-NO-BMI2-HAVE-SHLD-LABEL: shl_32bytes:
; X86-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    subl $92, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%edi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%edi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 8(%edi), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 12(%edi), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 16(%edi), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 20(%edi), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 24(%edi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 28(%edi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andb $7, %al
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrb $3, %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    negb %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movsbl %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 68(%esp,%ebx), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    notb %dl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 64(%esp,%ebx), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 76(%esp,%ebx), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 72(%esp,%ebx), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %esi, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 84(%esp,%ebx), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 80(%esp,%ebx), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, (%esp) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %edx, (%esp) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 60(%esp,%ebx), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 88(%esp,%ebx), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %ebx, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, 28(%ebx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 20(%ebx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 12(%ebx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %edx, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%ebx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, 4(%ebx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, 24(%ebx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, 16(%ebx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 8(%ebx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    addl $92, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-NO-SHLD-LABEL: shl_32bytes:
; X86-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    subl $88, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%edi), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%edi), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 8(%edi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 12(%edi), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 16(%edi), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 20(%edi), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 24(%edi), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 28(%edi), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andb $7, %dl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrb $3, %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    negb %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movsbl %cl, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 56(%esp,%esi), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 60(%esp,%esi), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %eax, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notb %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %ebx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 64(%esp,%esi), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %edi, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 68(%esp,%esi), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebp, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %ebx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %eax, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 72(%esp,%esi), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %ebx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 76(%esp,%esi), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %ebx, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebp, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %eax, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, 84(%esp,%esi), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 80(%esp,%esi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %esi, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebp, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %ebx, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %eax, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, (%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, 24(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, 28(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, 16(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 20(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 8(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 12(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl $88, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-HAVE-SHLD-LABEL: shl_32bytes:
; X86-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    subl $88, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%edi), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%edi), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 8(%edi), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 12(%edi), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 16(%edi), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%eax), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 20(%edi), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 24(%edi), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 28(%edi), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andb $7, %cl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrb $3, %al
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    negb %al
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movsbl %al, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 64(%esp,%esi), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %eax, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    notb %al
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 60(%esp,%esi), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %eax, %ebx, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %edx, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 68(%esp,%esi), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %eax, %edx, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 72(%esp,%esi), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %edx, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %edx, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 76(%esp,%esi), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %eax, %edx, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 80(%esp,%esi), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %edx, %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %ebp, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %eax, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %eax, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 56(%esp,%esi), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 84(%esp,%esi), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %edx, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 28(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, 20(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, 12(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %ebp, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, (%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %ebp, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 4(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 24(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 16(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 8(%esi)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl $88, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    retl
  %src = load i256, ptr %src.ptr, align 1
  %bitOff = load i256, ptr %bitOff.ptr, align 1
  %res = shl i256 %src, %bitOff
  store i256 %res, ptr %dst, align 1
  ret void
}
define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-NO-SHLD-LABEL: ashr_32bytes:
; X64-NO-BMI2-NO-SHLD:       # %bb.0:
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 24(%rdi), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movzbl (%rsi), %esi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    sarq $63, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %eax
; X64-NO-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X64-NO-BMI2-NO-SHLD-NEXT:    shrb $3, %sil
; X64-NO-BMI2-NO-SHLD-NEXT:    movzbl %sil, %r9d
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -64(%rsp,%r9), %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -56(%rsp,%r9), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %esi
; X64-NO-BMI2-NO-SHLD-NEXT:    notb %sil
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -48(%rsp,%r9), %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%rbx,%rbx), %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r11, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    addq %rdi, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r10, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -40(%rsp,%r9), %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%r9,%r9), %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %rbx, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    sarq %cl, %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r9, 24(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r10, 16(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, (%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r8, 8(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    retq
;
; X64-NO-BMI2-HAVE-SHLD-LABEL: ashr_32bytes:
; X64-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 24(%rdi), %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%rsi), %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    sarq $63, %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %eax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    andb $7, %al
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrb $3, %sil
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movzbl %sil, %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -64(%rsp,%rsi), %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -56(%rsp,%rsi), %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r8, %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    notb %cl
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -48(%rsp,%rsi), %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    leaq (%r10,%r10), %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    orq %r9, %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -40(%rsp,%rsi), %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rsi, %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r8, %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    sarq %cl, %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r10, 16(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rsi, 24(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, (%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r11, 8(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-NO-SHLD-LABEL: ashr_32bytes:
; X64-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq (%rdi), %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 24(%rdi), %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%rsi), %esi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    sarq $63, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, %eax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrb $3, %sil
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movzbl %sil, %ecx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -56(%rsp,%rcx), %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -48(%rsp,%rcx), %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rax, %rsi, %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rax, -64(%rsp,%rcx), %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rax, %rdi, %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -40(%rsp,%rcx), %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    sarxq %rax, %rcx, %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    # kill: def $al killed $al killed $rax def $rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    notb %al
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rdi, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %rdi, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r8, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rsi, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %rsi, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r9, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rcx, %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %rcx, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r10, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r11, 24(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rax, 16(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rsi, (%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, 8(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_32bytes:
; X64-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 24(%rdi), %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%rsi), %esi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    sarq $63, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ecx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    andb $7, %cl
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrb $3, %sil
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl %sil, %eax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -56(%rsp,%rax), %rsi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rcx, %rsi, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -40(%rsp,%rax), %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    sarxq %rcx, %r8, %r9
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %r10d
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    notb %r10b
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -48(%rsp,%rax), %r11
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    leaq (%r11,%r11), %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %r10, %rbx, %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -64(%rsp,%rax), %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    orq %rdi, %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r8, %r11
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    # kill: def $cl killed $cl killed $rcx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rsi, %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r11, 16(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r9, 24(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rax, (%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r10, 8(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    retq
;
; X86-NO-BMI2-NO-SHLD-LABEL: ashr_32bytes:
; X86-NO-BMI2-NO-SHLD:       # %bb.0:
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    subl $88, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%edx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%edx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 8(%edx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 12(%edx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 16(%edx), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movb (%ecx), %ch
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 20(%edx), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 24(%edx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 28(%edx), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    sarl $31, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ch, %al
; X86-NO-BMI2-NO-SHLD-NEXT:    andb $7, %al
; X86-NO-BMI2-NO-SHLD-NEXT:    shrb $3, %ch
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl %ch, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 24(%esp,%edi), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 28(%esp,%edi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %al, %ah
; X86-NO-BMI2-NO-SHLD-NEXT:    notb %ah
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 32(%esp,%edi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%edi,%edi), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebx, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %esi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 36(%esp,%ebx), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 40(%esp,%ebx), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %esi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 44(%esp,%edx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 48(%esp,%edx), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%esi,%esi), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %al, %ch
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %al, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %edi, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ah, %dl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %al, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 52(%esp,%esi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ebx,%ebx), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %dl, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %eax, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ch, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    sarl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, 28(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, 24(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, 16(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 20(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, 8(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 12(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, (%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    addl $88, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    retl
;
; X86-NO-BMI2-HAVE-SHLD-LABEL: ashr_32bytes:
; X86-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    subl $92, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%edx), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%edx), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 8(%edx), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 12(%edx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 16(%edx), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 20(%edx), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 24(%edx), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 28(%edx), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    sarl $31, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andb $7, %al
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrb $3, %cl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movzbl %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 32(%esp,%ebp), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    notb %dl
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 36(%esp,%ebp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 40(%esp,%ebp), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 44(%esp,%ebp), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 48(%esp,%ebp), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 52(%esp,%ebp), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ebx,%ebx), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, (%esp) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 28(%esp,%ebp), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 56(%esp,%ebp), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, 24(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %ebx, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    sarl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, 28(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 16(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 8(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, 20(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 12(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 4(%ebp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    addl $92, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-NO-SHLD-LABEL: ashr_32bytes:
; X86-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    subl $84, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 8(%edx), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 12(%edx), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 16(%edx), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movzbl (%ecx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 20(%edx), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 24(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 28(%edx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    sarl $31, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andb $7, %bl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrb $3, %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movzbl %cl, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 24(%esp,%edi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 28(%esp,%edi), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ebx, %esi, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notb %dl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %ebp, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl (%esp), %eax # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ebx, 20(%esp,%edi), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %esi, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 36(%esp,%edi), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    leal (%eax,%eax), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %ebx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 32(%esp,%edi), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %ebp, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %ebp, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 44(%esp,%edi), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    leal (%ebp,%ebp), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 40(%esp,%edi), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ebx, %eax, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %esi, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %eax, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %eax, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ebx, %ebp, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 48(%esp,%edi), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    sarxl %ebx, %edi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edx, %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %eax, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, 28(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, 24(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, 16(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 20(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 8(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 12(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, (%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl $84, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_32bytes:
; X86-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    subl $88, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%edx), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%edx), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 8(%edx), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 12(%edx), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 16(%edx), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl (%eax), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 20(%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 24(%edx), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 28(%edx), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    sarl $31, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andb $7, %cl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrb $3, %al
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movzbl %al, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 28(%esp,%ebx), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %eax, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    notb %dl
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 32(%esp,%ebx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %edx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %eax, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 40(%esp,%ebx), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    leal (%ebp,%ebp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %edx, %eax, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 36(%esp,%ebx), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %eax, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %eax, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 48(%esp,%ebx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    leal (%esi,%esi), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %edx, %edi, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 44(%esp,%ebx), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %edx, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %edi, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %eax, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 24(%esp,%ebx), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 52(%esp,%ebx), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %eax, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, 24(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    sarxl %ecx, %eax, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 28(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, 16(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, 8(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %eax, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, (%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 20(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 12(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 4(%ebx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl $88, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    retl
  %src = load i256, ptr %src.ptr, align 1
  %bitOff = load i256, ptr %bitOff.ptr, align 1
  %res = ashr i256 %src, %bitOff
  store i256 %res, ptr %dst, align 1
  ret void
}

define void @lshr_64bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-NO-SHLD-LABEL: lshr_64bytes:
; X64-NO-BMI2-NO-SHLD:       # %bb.0:
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 16(%rdi), %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 24(%rdi), %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 32(%rdi), %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 40(%rdi), %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 48(%rdi), %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 56(%rdi), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl (%rsi), %r8d
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %r8d, %eax
; X64-NO-BMI2-NO-SHLD-NEXT:    andl $7, %eax
; X64-NO-BMI2-NO-SHLD-NEXT:    shrl $3, %r8d
; X64-NO-BMI2-NO-SHLD-NEXT:    andl $63, %r8d
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -128(%rsp,%r8), %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -120(%rsp,%r8), %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r9, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %edi
; X64-NO-BMI2-NO-SHLD-NEXT:    notl %edi
; X64-NO-BMI2-NO-SHLD-NEXT:    andl $63, %edi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -112(%rsp,%r8), %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%r14,%r14), %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %rsi, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %esi
; X64-NO-BMI2-NO-SHLD-NEXT:    notb %sil
; X64-NO-BMI2-NO-SHLD-NEXT:    addq %r9, %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r11, %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -104(%rsp,%r8), %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r11, %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -96(%rsp,%r8), %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%r15,%r15), %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r12, %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    addq %r11, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r14, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -88(%rsp,%r8), %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r14, %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -80(%rsp,%r8), %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%rbp,%rbp), %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r13, %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    addq %r14, %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r15, %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -72(%rsp,%r8), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%rdi,%rdi), %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %rbp, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, 56(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r8, 48(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r14, 32(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r12, 40(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r11, 16(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rbx, 24(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r9, (%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r10, 8(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    retq
;
; X64-NO-BMI2-HAVE-SHLD-LABEL: lshr_64bytes:
; X64-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %r15
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %r13
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 24(%rdi), %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 32(%rdi), %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 40(%rdi), %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 48(%rdi), %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 56(%rdi), %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl (%rsi), %edi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, (%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, %eax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    andl $7, %eax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrl $3, %edi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    andl $63, %edi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -120(%rsp,%rdi), %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -112(%rsp,%rdi), %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    notl %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    andl $63, %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -104(%rsp,%rdi), %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    leaq (%r11,%r11), %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    orq %rbx, %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -96(%rsp,%rdi), %r15
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r15, %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -88(%rsp,%rdi), %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    leaq (%r14,%r14), %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    orq %r12, %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -80(%rsp,%rdi), %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r12, %r13
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %r13
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -72(%rsp,%rdi), %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    leaq (%rbp,%rbp), %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    orq %r13, %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r15, %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r12, %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -64(%rsp,%rdi), %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rsi, %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rdi, %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rbp, 48(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rsi, 56(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, 32(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r11, 16(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r8, (%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r9, 40(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rbx, 24(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r10, 8(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    addq $8, %rsp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %r13
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %r15
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-NO-SHLD-LABEL: lshr_64bytes:
; X64-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %rbp
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %r15
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %r13
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %r12
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq (%rdi), %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 16(%rdi), %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 24(%rdi), %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 32(%rdi), %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 40(%rdi), %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 48(%rdi), %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 56(%rdi), %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl (%rsi), %eax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    andl $7, %ecx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrl $3, %eax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    andl $63, %eax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -120(%rsp,%rax), %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -112(%rsp,%rax), %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %rdi, %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, -128(%rsp,%rax), %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %esi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -104(%rsp,%rax), %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r8, %r13
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r9, %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -88(%rsp,%rax), %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r11, %r15
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %r12d
; X64-HAVE-BMI2-NO-SHLD-NEXT:    notl %r12d
; X64-HAVE-BMI2-NO-SHLD-NEXT:    andl $63, %r12d
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %r9, %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %r12, %r9, %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %rbx, %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -96(%rsp,%rax), %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %rbx, %rbp
; X64-HAVE-BMI2-NO-SHLD-NEXT:    notb %sil
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rdi, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rsi, %rdi, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r14, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rbx, %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %r12, %rbx, %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r13, %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -80(%rsp,%rax), %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r14, %r13
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -72(%rsp,%rax), %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %rax, %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %r8, %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rsi, %r8, %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r10, %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    leaq (%r14,%r14), %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %r12, %r10, %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r15, %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %r11, %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rsi, %r11, %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %rbp, %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rax, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rsi, %rax, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r13, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rcx, 56(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rax, 48(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r11, 32(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r10, 40(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r8, 16(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rbx, 24(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, (%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r9, 8(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r12
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r13
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r15
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %rbp
; X64-HAVE-BMI2-NO-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_64bytes:
; X64-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %rbp
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %r14
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %r13
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %r12
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rcx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 16(%rdi), %r9
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 24(%rdi), %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 32(%rdi), %r11
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 40(%rdi), %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 48(%rdi), %r14
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 56(%rdi), %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%rsi), %eax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $7, %ecx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl $3, %eax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $63, %eax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -120(%rsp,%rax), %rsi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rcx, %rsi, %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -104(%rsp,%rax), %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rcx, %r8, %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -88(%rsp,%rax), %r11
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rcx, %r11, %r14
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %r12d
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    notl %r12d
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $63, %r12d
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -112(%rsp,%rax), %r9
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    leaq (%r9,%r9), %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %r12, %rdi, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    orq %r10, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -96(%rsp,%rax), %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    leaq (%rbx,%rbx), %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %r12, %r10, %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    orq %r15, %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -80(%rsp,%rax), %r13
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    leaq (%r13,%r13), %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %r12, %r15, %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -72(%rsp,%rax), %r12
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rcx, %r12, %rbp
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -128(%rsp,%rax), %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    orq %r14, %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r8, %r9
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r11, %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r12, %r13
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    # kill: def $cl killed $cl killed $rcx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rsi, %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r13, 48(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rbp, 56(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rbx, 32(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r9, 16(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rax, (%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r15, 40(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r10, 24(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, 8(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %r12
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %r13
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %r14
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %rbp
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    retq
;
; X86-NO-BMI2-NO-SHLD-LABEL: lshr_64bytes:
; X86-NO-BMI2-NO-SHLD:       # %bb.0:
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    subl $208, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%esi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 8(%esi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 12(%esi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 16(%esi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 20(%esi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 24(%esi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 28(%esi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 32(%esi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 36(%esi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 40(%esi), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 44(%esi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 48(%esi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 52(%esi), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 56(%esi), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 60(%esi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esi), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    andl $7, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl $3, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    andl $63, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 84(%esp,%esi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 88(%esp,%esi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%eax,%eax), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    notl %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    andl $31, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 92(%esp,%esi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 96(%esp,%esi), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 100(%esp,%esi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 104(%esp,%esi), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 108(%esp,%esi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 112(%esp,%esi), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 116(%esp,%esi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 120(%esp,%esi), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 124(%esp,%esi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 128(%esp,%esi), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 132(%esp,%esi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 136(%esp,%esi), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 80(%esp,%esi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    notb %dl
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %ebp # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %edi, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 140(%esp,%esi), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%esi,%esi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %eax, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, 60(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, 56(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, 48(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 40(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 32(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 24(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 16(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 8(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, (%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 52(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 44(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 36(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 28(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 20(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 12(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    addl $208, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    retl
;
; X86-NO-BMI2-HAVE-SHLD-LABEL: lshr_64bytes:
; X86-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    subl $204, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%esi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 8(%esi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 12(%esi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 16(%esi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 20(%esi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 24(%esi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 28(%esi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 32(%esi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 36(%esi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 40(%esi), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 44(%esi), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 48(%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 52(%esi), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 56(%esi), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 60(%esi), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esi), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andl $7, %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl $3, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andl $63, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 80(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 84(%esp,%esi), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    notl %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andl $31, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 88(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 92(%esp,%esi), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 96(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 100(%esp,%esi), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 104(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 108(%esp,%esi), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 112(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 116(%esp,%esi), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 120(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 124(%esp,%esi), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 128(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 132(%esp,%esi), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ebx,%ebx), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, (%esp) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 76(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 136(%esp,%esi), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %esi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, 56(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %ebx, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, 60(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 48(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 40(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 32(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 24(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 16(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 8(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, (%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, 52(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 44(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 36(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 28(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 20(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 12(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 4(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    addl $204, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-NO-SHLD-LABEL: lshr_64bytes:
; X86-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    subl $204, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 8(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 12(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 16(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 20(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 24(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 28(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 32(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 36(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 40(%edx), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 44(%edx), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 48(%edx), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 52(%edx), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 56(%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 60(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%edx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andl $7, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl $3, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andl $63, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 80(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 84(%esp,%edx), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andl $31, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 92(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 88(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 100(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 96(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 108(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 104(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 116(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 112(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 124(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 120(%esp,%edx), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ebx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 132(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %ecx, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 128(%esp,%edx), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %esi, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notb %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, 76(%esp,%edx), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, (%esp), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ebx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %ebx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 136(%esp,%edx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %edx, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    leal (%edx,%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %eax, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, 60(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 56(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, 48(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, 40(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, 32(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 24(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 16(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 8(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, (%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 52(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 44(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 36(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 28(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 20(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 12(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl $204, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_64bytes:
; X86-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    subl $200, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 8(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 12(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 16(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 20(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 24(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 28(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 32(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 36(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 40(%eax), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 44(%eax), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 48(%eax), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 52(%eax), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 56(%eax), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 60(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%eax), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $7, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl $3, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $63, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 76(%esp,%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %edx, %ecx, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    notl %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $31, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 80(%esp,%eax), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl %edi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %edi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 88(%esp,%eax), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 84(%esp,%eax), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %edx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 96(%esp,%eax), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 92(%esp,%eax), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %edx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 104(%esp,%eax), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 100(%esp,%eax), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %edx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 112(%esp,%eax), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 108(%esp,%eax), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %edx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 120(%esp,%eax), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    leal (%ebp,%ebp), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 116(%esp,%eax), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %edx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 128(%esp,%eax), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    leal (%ebx,%ebx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %ecx, %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 124(%esp,%eax), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %edx, %esi, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %ecx, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edi, (%esp) # 4-byte Folded Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %esi, %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 72(%esp,%eax), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 132(%esp,%eax), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, 56(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, 48(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 40(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 32(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 24(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 16(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 8(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %edx, %edi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, (%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, 60(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 52(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 44(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 36(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 28(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 20(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 12(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, 4(%eax)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl $200, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    retl
  %src = load i512, ptr %src.ptr, align 1
  %bitOff = load i512, ptr %bitOff.ptr, align 1
  %res = lshr i512 %src, %bitOff
  store i512 %res, ptr %dst, align 1
  ret void
}
define void @shl_64bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-NO-SHLD-LABEL: shl_64bytes:
; X64-NO-BMI2-NO-SHLD:       # %bb.0:
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 24(%rdi), %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 32(%rdi), %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 40(%rdi), %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 48(%rdi), %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 56(%rdi), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl (%rsi), %esi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %eax
; X64-NO-BMI2-NO-SHLD-NEXT:    andl $7, %eax
; X64-NO-BMI2-NO-SHLD-NEXT:    shrl $3, %esi
; X64-NO-BMI2-NO-SHLD-NEXT:    andl $63, %esi
; X64-NO-BMI2-NO-SHLD-NEXT:    negl %esi
; X64-NO-BMI2-NO-SHLD-NEXT:    movslq %esi, %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -64(%rsp,%r14), %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -56(%rsp,%r14), %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r8, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %esi
; X64-NO-BMI2-NO-SHLD-NEXT:    notb %sil
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r9, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %rdi, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -40(%rsp,%r14), %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r11, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -48(%rsp,%r14), %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r15, %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %rdi, %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %edi
; X64-NO-BMI2-NO-SHLD-NEXT:    notl %edi
; X64-NO-BMI2-NO-SHLD-NEXT:    andl $63, %edi
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r15, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -24(%rsp,%r14), %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r15, %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -32(%rsp,%r14), %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rbp, %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r13, %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %rbp, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -8(%rsp,%r14), %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -16(%rsp,%r14), %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r14, %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r13, %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r14, %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r9, (%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r15, 48(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rbp, 56(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r11, 32(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r12, 40(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r8, 16(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rbx, 24(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r10, 8(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    retq
;
; X64-NO-BMI2-HAVE-SHLD-LABEL: shl_64bytes:
; X64-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %r15
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %r13
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 24(%rdi), %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 32(%rdi), %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 40(%rdi), %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 48(%rdi), %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 56(%rdi), %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl (%rsi), %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, (%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %eax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    andl $7, %eax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrl $3, %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    andl $63, %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    negl %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movslq %esi, %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -40(%rsp,%r10), %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    notl %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    andl $63, %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -56(%rsp,%r10), %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -48(%rsp,%r10), %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r8, %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    orq %rbx, %rdi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -24(%rsp,%r10), %r15
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r15, %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -32(%rsp,%r10), %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    orq %r12, %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -8(%rsp,%r10), %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r12, %r13
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %r13
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -16(%rsp,%r10), %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r11, %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    orq %r13, %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %rsi, %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %r15, %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq (%rsp,%r10), %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %r12, %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %r9, %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rsi, 56(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r11, 40(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, 24(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r9, (%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r8, 8(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rbp, 48(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rbx, 32(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rdi, 16(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    addq $8, %rsp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %r13
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %r15
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-NO-SHLD-LABEL: shl_64bytes:
; X64-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %rbp
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %r15
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %r13
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %r12
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq (%rdi), %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 24(%rdi), %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 32(%rdi), %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 40(%rdi), %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 48(%rdi), %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 56(%rdi), %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl (%rsi), %esi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, %eax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    andl $7, %eax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrl $3, %esi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    andl $63, %esi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    negl %esi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movslq %esi, %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -64(%rsp,%rcx), %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -56(%rsp,%rcx), %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %rsi, %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -40(%rsp,%rcx), %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %rdi, %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -48(%rsp,%rcx), %r15
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %r15, %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %r8d
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -24(%rsp,%rcx), %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %r11, %r12
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %r10, %r13
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %ebp
; X64-HAVE-BMI2-NO-SHLD-NEXT:    notb %bpl
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrq %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rbp, %r10, %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r9, %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrq %r15
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rbp, %r15, %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %rbx, %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -32(%rsp,%rcx), %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %rbx, %r15
; X64-HAVE-BMI2-NO-SHLD-NEXT:    notl %r8d
; X64-HAVE-BMI2-NO-SHLD-NEXT:    andl $63, %r8d
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrq %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %r8, %rsi, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r14, %rsi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, -8(%rsp,%rcx), %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -16(%rsp,%rcx), %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rax, %rcx, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrq %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rbp, %rbx, %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r12, %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrq %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %r8, %rdi, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r15, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrq %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rbp, %rcx, %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r14, %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrq %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %r8, %r11, %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %rax, %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r13, (%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r8, 48(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rcx, 56(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, 32(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rbx, 40(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rsi, 16(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r9, 24(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r10, 8(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r12
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r13
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r15
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %rbp
; X64-HAVE-BMI2-NO-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-HAVE-SHLD-LABEL: shl_64bytes:
; X64-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %rbp
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %r14
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %r13
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %r12
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 24(%rdi), %r9
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 32(%rdi), %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 40(%rdi), %r11
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 48(%rdi), %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 56(%rdi), %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%rsi), %esi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ecx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $7, %ecx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl $3, %esi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $63, %esi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    negl %esi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movslq %esi, %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -48(%rsp,%rax), %rsi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %rcx, %rsi, %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -32(%rsp,%rax), %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %rcx, %rdi, %r12
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -16(%rsp,%rax), %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %rcx, %r10, %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %ebp
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    notl %ebp
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $63, %ebp
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -64(%rsp,%rax), %r11
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -56(%rsp,%rax), %r9
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r9, %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrq %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rbp, %r8, %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    orq %rbx, %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -40(%rsp,%rax), %r14
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r14, %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rbp, %rbx, %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    orq %r12, %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -24(%rsp,%rax), %r13
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r13, %r12
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrq %r12
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rbp, %r12, %r12
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %rcx, %r11, %rbp
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    orq %r15, %r12
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %rsi, %r14
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %rdi, %r13
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -8(%rsp,%rax), %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %r10, %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    # kill: def $cl killed $cl killed $rcx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shldq %cl, %r11, %r9
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rax, 56(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r13, 40(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r14, 24(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rbp, (%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r9, 8(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r12, 48(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rbx, 32(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r8, 16(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %r12
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %r13
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %r14
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %rbp
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    retq
;
; X86-NO-BMI2-NO-SHLD-LABEL: shl_64bytes:
; X86-NO-BMI2-NO-SHLD:       # %bb.0:
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    subl $192, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%ebx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%ebx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 8(%ebx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 12(%ebx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 16(%ebx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 20(%ebx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 24(%ebx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 28(%ebx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 32(%ebx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 36(%ebx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 40(%ebx), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 44(%ebx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 48(%ebx), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 52(%ebx), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 56(%ebx), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 60(%ebx), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%ebx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl $3, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    andl $63, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal {{[0-9]+}}(%esp), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    subl %eax, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%ebp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%ebp), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    andl $7, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %bl, %ch
; X86-NO-BMI2-NO-SHLD-NEXT:    notb %ch
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ch, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ch, (%esp) # 1-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %eax, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 12(%ebp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %bl, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 8(%ebp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movb %ch, %cl
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edx, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    notl %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    andl $31, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %eax, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 20(%ebp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 16(%ebp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl (%esp), %ecx # 1-byte Folded Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edx, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %eax, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 28(%ebp), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 24(%ebp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl (%esp), %ecx # 1-byte Folded Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %esi, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edx, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 36(%ebp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 32(%ebp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl (%esp), %ecx # 1-byte Folded Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edx, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %eax, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 44(%ebp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 40(%ebp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl (%esp), %ecx # 1-byte Folded Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %eax, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 52(%ebp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    negl %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 176(%esp,%ecx), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl (%esp), %ecx # 1-byte Folded Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 60(%ebp), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 56(%ebp), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movzbl (%esp), %ecx # 1-byte Folded Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edx, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, (%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, 56(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, 60(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, 48(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, 52(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, 40(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, 44(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, 32(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, 36(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, 24(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, 28(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, 16(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, 20(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, 8(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, 12(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, 4(%ecx)
; X86-NO-BMI2-NO-SHLD-NEXT:    addl $192, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    retl
;
; X86-NO-BMI2-HAVE-SHLD-LABEL: shl_64bytes:
; X86-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    subl $204, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 8(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 12(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 16(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 20(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 24(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 28(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 32(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 36(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 40(%eax), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 44(%eax), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 48(%eax), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 52(%eax), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 56(%eax), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 60(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%eax), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl $3, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andl $63, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal {{[0-9]+}}(%esp), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    subl %esi, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 8(%edi), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andl $7, %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%edi), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    notl %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andl $31, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 16(%edi), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 12(%edi), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 24(%edi), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 20(%edi), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 32(%edi), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 28(%edi), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 40(%edi), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 36(%edi), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    negl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 188(%esp,%esi), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 44(%edi), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, (%esp) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %ebx, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 56(%edi), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 52(%edi), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %edx, (%esp) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%edi), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 60(%edi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %esi, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, 60(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %ebx, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %eax, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, 52(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 44(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 36(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 28(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 20(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 12(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, (%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, 4(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, 56(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 48(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 40(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 32(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 24(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 16(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 8(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    addl $204, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-NO-SHLD-LABEL: shl_64bytes:
; X86-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    subl $216, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 8(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 12(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 16(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 20(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 24(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 28(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 32(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 36(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 40(%edx), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 44(%edx), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 48(%edx), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 52(%edx), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 56(%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 60(%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%edx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andl $7, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl $3, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andl $63, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    leal {{[0-9]+}}(%esp), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    subl %edx, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 8(%edi), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notl %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andl $31, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%edi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %esi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, %ebp, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %esi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 12(%edi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %esi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 16(%edi), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, %edx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %esi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 20(%edi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %esi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 24(%edi), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, %edx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %esi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 28(%edi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %esi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 32(%edi), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, %edx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %esi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 36(%edi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %esi, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 40(%edi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %esi, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 44(%edi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %esi, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 48(%edi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %esi, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 52(%edi), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %ecx, %ebx, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 56(%edi), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, (%esp), %ebx # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, %esi, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    negl %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, 212(%esp,%ecx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%edi), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %eax, %ecx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notb %dl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %edx, %ecx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ebx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %edx, %ebp, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl (%esp), %eax # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %edx, %eax, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %edx, %eax, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %edx, %eax, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %edx, %eax, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %edx, %eax, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %edx, %ebp, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edx, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, (%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, 60(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 52(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 44(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, 36(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, 28(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 20(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 12(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 4(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 56(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 48(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 40(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 32(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 24(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 16(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, 8(%edx)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl $216, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-HAVE-SHLD-LABEL: shl_64bytes:
; X86-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    subl $204, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%edi), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%edi), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 8(%edi), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 12(%edi), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 16(%edi), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 20(%edi), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 24(%edi), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 28(%edi), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 32(%edi), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 36(%edi), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 40(%edi), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 44(%edi), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 48(%edi), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 52(%edi), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 56(%edi), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 60(%edi), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%edi), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $7, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl $3, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $63, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    leal {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    subl %edi, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl $0, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 8(%edx), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    notl %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $31, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %esi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %ebp, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 12(%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %esi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 16(%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 20(%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %esi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 24(%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 28(%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %esi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 32(%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 36(%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %esi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 40(%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 52(%edx), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %esi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 56(%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 44(%edx), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %esi, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    negl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 188(%esp,%esi), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %ecx, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %esi, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%edx), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 60(%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %edx, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, 60(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %edi, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %edi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shldl %cl, %eax, %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, 52(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, 44(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 36(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 28(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 20(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 12(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, (%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, 4(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 56(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 48(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 40(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 32(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 24(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 16(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 8(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl $204, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    retl
  %src = load i512, ptr %src.ptr, align 1
  %bitOff = load i512, ptr %bitOff.ptr, align 1
  %res = shl i512 %src, %bitOff
  store i512 %res, ptr %dst, align 1
  ret void
}
define void @ashr_64bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-NO-SHLD-LABEL: ashr_64bytes:
; X64-NO-BMI2-NO-SHLD:       # %bb.0:
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    pushq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 16(%rdi), %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 24(%rdi), %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 32(%rdi), %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 40(%rdi), %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 48(%rdi), %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    movq 56(%rdi), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl (%rsi), %r8d
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    sarq $63, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %r8d, %eax
; X64-NO-BMI2-NO-SHLD-NEXT:    andl $7, %eax
; X64-NO-BMI2-NO-SHLD-NEXT:    shrl $3, %r8d
; X64-NO-BMI2-NO-SHLD-NEXT:    andl $63, %r8d
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -128(%rsp,%r8), %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -120(%rsp,%r8), %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r9, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %rsi
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %edi
; X64-NO-BMI2-NO-SHLD-NEXT:    notl %edi
; X64-NO-BMI2-NO-SHLD-NEXT:    andl $63, %edi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -112(%rsp,%r8), %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%r14,%r14), %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %rsi, %r10
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %esi
; X64-NO-BMI2-NO-SHLD-NEXT:    notb %sil
; X64-NO-BMI2-NO-SHLD-NEXT:    addq %r9, %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r11, %r9
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -104(%rsp,%r8), %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r11, %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -96(%rsp,%r8), %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%r15,%r15), %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r12, %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    addq %r11, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r14, %r11
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -88(%rsp,%r8), %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r14, %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -80(%rsp,%r8), %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%rbp,%rbp), %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r13, %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    addq %r14, %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %r15, %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shrq %cl, %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    movq -72(%rsp,%r8), %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    leaq (%rdi,%rdi), %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    shlq %cl, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    orq %rbp, %r8
; X64-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-NO-SHLD-NEXT:    sarq %cl, %rdi
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rdi, 56(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r8, 48(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r14, 32(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r12, 40(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r11, 16(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %rbx, 24(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r9, (%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    movq %r10, 8(%rdx)
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %rbx
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %r12
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %r13
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %r14
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %r15
; X64-NO-BMI2-NO-SHLD-NEXT:    popq %rbp
; X64-NO-BMI2-NO-SHLD-NEXT:    retq
;
; X64-NO-BMI2-HAVE-SHLD-LABEL: ashr_64bytes:
; X64-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %r15
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %r13
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    pushq %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %rcx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 16(%rdi), %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 24(%rdi), %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 32(%rdi), %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 40(%rdi), %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 48(%rdi), %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq 56(%rdi), %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl (%rsi), %edi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    sarq $63, %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, (%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, %eax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    andl $7, %eax
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrl $3, %edi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    andl $63, %edi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -120(%rsp,%rdi), %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -112(%rsp,%rdi), %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    notl %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    andl $63, %esi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -104(%rsp,%rdi), %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    leaq (%r11,%r11), %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    orq %rbx, %r10
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -96(%rsp,%rdi), %r15
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r15, %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -88(%rsp,%rdi), %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    leaq (%r14,%r14), %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    orq %r12, %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -80(%rsp,%rdi), %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r12, %r13
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrq %cl, %r13
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -72(%rsp,%rdi), %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    leaq (%rbp,%rbp), %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shlq %cl, %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    orq %r13, %r9
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r15, %r11
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r12, %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq -64(%rsp,%rdi), %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rsi, %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; X64-NO-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rdi, %r8
; X64-NO-BMI2-HAVE-SHLD-NEXT:    sarq %cl, %rsi
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rbp, 48(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rsi, 56(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r14, 32(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r11, 16(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r8, (%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r9, 40(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %rbx, 24(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    movq %r10, 8(%rdx)
; X64-NO-BMI2-HAVE-SHLD-NEXT:    addq $8, %rsp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %rbx
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %r12
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %r13
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %r14
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %r15
; X64-NO-BMI2-HAVE-SHLD-NEXT:    popq %rbp
; X64-NO-BMI2-HAVE-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-NO-SHLD-LABEL: ashr_64bytes:
; X64-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %rbp
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %r15
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %r13
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %r12
; X64-HAVE-BMI2-NO-SHLD-NEXT:    pushq %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq (%rdi), %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 8(%rdi), %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 16(%rdi), %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 24(%rdi), %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 32(%rdi), %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 40(%rdi), %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 48(%rdi), %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq 56(%rdi), %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl (%rsi), %eax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    sarq $63, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    andl $7, %ecx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrl $3, %eax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    andl $63, %eax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -120(%rsp,%rax), %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -112(%rsp,%rax), %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %rdi, %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, -128(%rsp,%rax), %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %esi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -104(%rsp,%rax), %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r8, %r13
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r9, %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -88(%rsp,%rax), %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r11, %r15
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, %r12d
; X64-HAVE-BMI2-NO-SHLD-NEXT:    notl %r12d
; X64-HAVE-BMI2-NO-SHLD-NEXT:    andl $63, %r12d
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %r9, %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %r12, %r9, %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %rbx, %r9
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -96(%rsp,%rax), %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %rbx, %rbp
; X64-HAVE-BMI2-NO-SHLD-NEXT:    notb %sil
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rdi, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rsi, %rdi, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r14, %rdi
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rbx, %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %r12, %rbx, %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r13, %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -80(%rsp,%rax), %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shrxq %rcx, %r14, %r13
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq -72(%rsp,%rax), %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    sarxq %rcx, %rax, %rcx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %r8, %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rsi, %r8, %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r10, %r8
; X64-HAVE-BMI2-NO-SHLD-NEXT:    leaq (%r14,%r14), %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %r12, %r10, %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r15, %r10
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %r11, %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rsi, %r11, %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %rbp, %r11
; X64-HAVE-BMI2-NO-SHLD-NEXT:    addq %rax, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    shlxq %rsi, %rax, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    orq %r13, %rax
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rcx, 56(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rax, 48(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r11, 32(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r10, 40(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r8, 16(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rbx, 24(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %rdi, (%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    movq %r9, 8(%rdx)
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %rbx
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r12
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r13
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r14
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %r15
; X64-HAVE-BMI2-NO-SHLD-NEXT:    popq %rbp
; X64-HAVE-BMI2-NO-SHLD-NEXT:    retq
;
; X64-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_64bytes:
; X64-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %rbp
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %r14
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %r13
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %r12
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    pushq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq (%rdi), %rcx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 8(%rdi), %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 16(%rdi), %r9
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 24(%rdi), %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 32(%rdi), %r11
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 40(%rdi), %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 48(%rdi), %r14
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq 56(%rdi), %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%rsi), %eax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    sarq $63, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $7, %ecx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl $3, %eax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $63, %eax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -120(%rsp,%rax), %rsi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rcx, %rsi, %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -104(%rsp,%rax), %r8
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rcx, %r8, %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -88(%rsp,%rax), %r11
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxq %rcx, %r11, %r14
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %r12d
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    notl %r12d
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $63, %r12d
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -112(%rsp,%rax), %r9
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    leaq (%r9,%r9), %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %r12, %rdi, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    orq %r10, %rdi
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -96(%rsp,%rax), %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    leaq (%rbx,%rbx), %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %r12, %r10, %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    orq %r15, %r10
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -80(%rsp,%rax), %r13
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    leaq (%r13,%r13), %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxq %r12, %r15, %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -72(%rsp,%rax), %r12
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    sarxq %rcx, %r12, %rbp
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq -128(%rsp,%rax), %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    orq %r14, %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r8, %r9
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r11, %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %r12, %r13
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    # kill: def $cl killed $cl killed $rcx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdq %cl, %rsi, %rax
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r13, 48(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rbp, 56(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rbx, 32(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r9, 16(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rax, (%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r15, 40(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %r10, 24(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    movq %rdi, 8(%rdx)
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %rbx
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %r12
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %r13
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %r14
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %r15
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    popq %rbp
; X64-HAVE-BMI2-HAVE-SHLD-NEXT:    retq
;
; X86-NO-BMI2-NO-SHLD-LABEL: ashr_64bytes:
; X86-NO-BMI2-NO-SHLD:       # %bb.0:
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    subl $208, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%eax), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 4(%eax), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 8(%eax), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 12(%eax), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 16(%eax), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 20(%eax), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 24(%eax), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 28(%eax), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 32(%eax), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 36(%eax), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 40(%eax), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 44(%eax), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 48(%eax), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 52(%eax), %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 56(%eax), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 60(%eax), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esi), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    sarl $31, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    andl $7, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl $3, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    andl $63, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 84(%esp,%esi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 88(%esp,%esi), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%eax,%eax), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    notl %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    andl $31, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 92(%esp,%esi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 96(%esp,%esi), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 100(%esp,%esi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 104(%esp,%esi), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 108(%esp,%esi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 112(%esp,%esi), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 116(%esp,%esi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 120(%esp,%esi), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 124(%esp,%esi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 128(%esp,%esi), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 132(%esp,%esi), %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 136(%esp,%esi), %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%ecx,%ecx), %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 80(%esp,%esi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %edx
; X86-NO-BMI2-NO-SHLD-NEXT:    notb %dl
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %ebp # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, (%esp) # 4-byte Spill
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    addl %edi, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %ebx, %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %eax, %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    shrl %cl, %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl 140(%esp,%esi), %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    leal (%esi,%esi), %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    orl %eax, %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebp, %ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-NO-BMI2-NO-SHLD-NEXT:    sarl %cl, %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %esi, 60(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ebx, 56(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %edi, 48(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 40(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 32(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 24(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 16(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 8(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, (%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 52(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 44(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 36(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 28(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 20(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 12(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%eax)
; X86-NO-BMI2-NO-SHLD-NEXT:    addl $208, %esp
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-NO-SHLD-NEXT:    retl
;
; X86-NO-BMI2-HAVE-SHLD-LABEL: ashr_64bytes:
; X86-NO-BMI2-HAVE-SHLD:       # %bb.0:
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    subl $204, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 4(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 8(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 12(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 16(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 20(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 24(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 28(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 32(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 36(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 40(%eax), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 44(%eax), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 48(%eax), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 52(%eax), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 56(%eax), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 60(%eax), %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esi), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    sarl $31, %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andl $7, %eax
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl $3, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andl $63, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 80(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 84(%esp,%esi), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    notl %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    andl $31, %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 88(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 92(%esp,%esi), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 96(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 100(%esp,%esi), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 104(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 108(%esp,%esi), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 112(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 116(%esp,%esi), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 120(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 124(%esp,%esi), %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ecx,%ecx), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 128(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrl %cl, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 132(%esp,%esi), %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    leal (%ebx,%ebx), %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shll %cl, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    orl %edi, %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, %ecx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edx, (%esp) # 4-byte Folded Spill
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 76(%esp,%esi), %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl 136(%esp,%esi), %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %esi, %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebx, 56(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %ebx, %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    sarl %cl, %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %esi, 60(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 48(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 40(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 32(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 24(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 16(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 8(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %edi, (%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %ebp, 52(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 44(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 36(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 28(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 20(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 12(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-BMI2-HAVE-SHLD-NEXT:    movl %eax, 4(%edx)
; X86-NO-BMI2-HAVE-SHLD-NEXT:    addl $204, %esp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-NO-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-NO-BMI2-HAVE-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-NO-SHLD-LABEL: ashr_64bytes:
; X86-HAVE-BMI2-NO-SHLD:       # %bb.0:
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    subl $204, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%eax), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 4(%eax), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 8(%eax), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 12(%eax), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 16(%eax), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 20(%eax), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 24(%eax), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 28(%eax), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 32(%eax), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 36(%eax), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 40(%eax), %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 44(%eax), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 48(%eax), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 52(%eax), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 56(%eax), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 60(%eax), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%edx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    sarl $31, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edx, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andl $7, %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrl $3, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andl $63, %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 80(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 84(%esp,%edx), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    andl $31, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 92(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 88(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 100(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 96(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 108(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 104(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 116(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 112(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 124(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 120(%esp,%edx), %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %ebx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 132(%esp,%edx), %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ecx, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %edi, %ecx, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 128(%esp,%edx), %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, %esi, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %ecx, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %eax, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    notb %cl
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %edi, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, 76(%esp,%edx), %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, (%esp), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %ebp, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %ebx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %ebx, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %esi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl %edi, %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shrxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl 136(%esp,%edx), %edx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    sarxl %eax, %edx, %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    leal (%edx,%edx), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    shlxl %ecx, %eax, %ecx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %edi, 60(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 56(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %esi, 48(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebx, 40(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ebp, 32(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 24(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 16(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 8(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, (%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 52(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 44(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 36(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 28(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 20(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 12(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-NO-SHLD-NEXT:    movl %ecx, 4(%eax)
; X86-HAVE-BMI2-NO-SHLD-NEXT:    addl $204, %esp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-NO-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-NO-SHLD-NEXT:    retl
;
; X86-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_64bytes:
; X86-HAVE-BMI2-HAVE-SHLD:       # %bb.0:
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    pushl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    subl $200, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 4(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 8(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 12(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 16(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 20(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 24(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 28(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 32(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 36(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 40(%eax), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 44(%eax), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 48(%eax), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 52(%eax), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 56(%eax), %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 60(%eax), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%edx), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    sarl $31, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[0-9]+}}(%esp)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edx, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $7, %ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrl $3, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $63, %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 76(%esp,%edx), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %eax, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ecx, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    notl %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    andl $31, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 80(%esp,%edx), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl %edi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %edi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 88(%esp,%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 84(%esp,%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 96(%esp,%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 92(%esp,%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 104(%esp,%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 100(%esp,%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 112(%esp,%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, (%esp) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 108(%esp,%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 120(%esp,%edx), %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    leal (%ebx,%ebx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 116(%esp,%edx), %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %esi, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %esi, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 128(%esp,%edx), %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    leal (%edi,%edi), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shlxl %eax, %ebp, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 124(%esp,%edx), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrxl %ecx, %eax, %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    orl %ebp, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %ebp, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %ebp, (%esp) # 4-byte Folded Spill
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %eax, %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 72(%esp,%edx), %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl 132(%esp,%edx), %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %ebp, %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[0-9]+}}(%esp), %edx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, 56(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %ebx, 48(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl (%esp), %edi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %edi, 40(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, 32(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, 24(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, 16(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, 8(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    sarxl %ecx, %ebp, %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    shrdl %cl, %edi, %eax
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, (%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %esi, 60(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 52(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 44(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 36(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 28(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 20(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 12(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    movl %eax, 4(%edx)
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    addl $200, %esp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %esi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %edi
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebx
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    popl %ebp
; X86-HAVE-BMI2-HAVE-SHLD-NEXT:    retl
  %src = load i512, ptr %src.ptr, align 1
  %bitOff = load i512, ptr %bitOff.ptr, align 1
  %res = ashr i512 %src, %bitOff
  store i512 %res, ptr %dst, align 1
  ret void
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; ALL: {{.*}}
; X86: {{.*}}
; X86-NO-SHLD: {{.*}}
; X86-SHLD: {{.*}}
; X64: {{.*}}
; X64-NO-SHLD: {{.*}}
; X64-SHLD: {{.*}}