lib/Target/AArch64/AArch64SelectionDAGInfo.cpp

   1 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the AArch64SelectionDAGInfo class.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "AArch64TargetMachine.h"
  14 using namespace llvm;
  15
  16 #define DEBUG_TYPE "aarch64-selectiondag-info"
  17
  18 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
  19     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
  20     SDValue Size, unsigned Align, bool isVolatile,
  21     MachinePointerInfo DstPtrInfo) const {
  22   // Check to see if there is a specialized entry-point for memory zeroing.
  23   ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);
  24   ConstantSDNode *SizeValue = dyn_cast<ConstantSDNode>(Size);
  25   const AArch64Subtarget &STI =
  26       DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
  27   const char *bzeroName = (V && V->isNullValue())
  28       ? DAG.getTargetLoweringInfo().getLibcallName(RTLIB::BZERO) : nullptr;
  29   // For small size (< 256), it is not beneficial to use bzero
  30   // instead of memset.
  31   if (bzeroName && (!SizeValue || SizeValue->getZExtValue() > 256)) {
  32     const AArch64TargetLowering &TLI = *STI.getTargetLowering();
  33
  34     EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
  35     Type *IntPtrTy = Type::getInt8PtrTy(*DAG.getContext());
  36     TargetLowering::ArgListTy Args;
  37     TargetLowering::ArgListEntry Entry;
  38     Entry.Node = Dst;
  39     Entry.Ty = IntPtrTy;
  40     Args.push_back(Entry);
  41     Entry.Node = Size;
  42     Args.push_back(Entry);
  43     TargetLowering::CallLoweringInfo CLI(DAG);
  44     CLI.setDebugLoc(dl)
  45         .setChain(Chain)
  46         .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
  47                       DAG.getExternalSymbol(bzeroName, IntPtr),
  48                       std::move(Args))
  49         .setDiscardResult();
  50     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
  51     return CallResult.second;
  52   }
  53   return SDValue();
  54 }
  55 bool AArch64SelectionDAGInfo::generateFMAsInMachineCombiner(
  56     CodeGenOpt::Level OptLevel) const {
  57   return OptLevel >= CodeGenOpt::Aggressive;
  58 }
  59
  60 static const int kSetTagLoopThreshold = 176;
  61
  62 static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl,
  63                                   SDValue Chain, SDValue Ptr, uint64_t ObjSize,
  64                                   const MachineMemOperand *BaseMemOperand,
  65                                   bool ZeroData) {
  66   MachineFunction &MF = DAG.getMachineFunction();
  67   unsigned ObjSizeScaled = ObjSize / 16;
  68
  69   SDValue TagSrc = Ptr;
  70   if (Ptr.getOpcode() == ISD::FrameIndex) {
  71     int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();
  72     Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);
  73     // A frame index operand may end up as [SP + offset] => it is fine to use SP
  74     // register as the tag source.
  75     TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);
  76   }
  77
  78   const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
  79   const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
  80
  81   SmallVector<SDValue, 8> OutChains;
  82   unsigned OffsetScaled = 0;
  83   while (OffsetScaled < ObjSizeScaled) {
  84     if (ObjSizeScaled - OffsetScaled >= 2) {
  85       SDValue AddrNode = DAG.getMemBasePlusOffset(Ptr, OffsetScaled * 16, dl);
  86       SDValue St = DAG.getMemIntrinsicNode(
  87           OpCode2, dl, DAG.getVTList(MVT::Other),
  88           {Chain, TagSrc, AddrNode},
  89           MVT::v4i64,
  90           MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));
  91       OffsetScaled += 2;
  92       OutChains.push_back(St);
  93       continue;
  94     }
  95
  96     if (ObjSizeScaled - OffsetScaled > 0) {
  97       SDValue AddrNode = DAG.getMemBasePlusOffset(Ptr, OffsetScaled * 16, dl);
  98       SDValue St = DAG.getMemIntrinsicNode(
  99           OpCode1, dl, DAG.getVTList(MVT::Other),
 100           {Chain, TagSrc, AddrNode},
 101           MVT::v2i64,
 102           MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));
 103       OffsetScaled += 1;
 104       OutChains.push_back(St);
 105     }
 106   }
 107
 108   SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
 109   return Res;
 110 }
 111
 112 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(
 113     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
 114     SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
 115   uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue();
 116   assert(ObjSize % 16 == 0);
 117
 118   MachineFunction &MF = DAG.getMachineFunction();
 119   MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
 120       DstPtrInfo, MachineMemOperand::MOStore, ObjSize, 16);
 121
 122   bool UseSetTagRangeLoop =
 123       kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
 124   if (!UseSetTagRangeLoop)
 125     return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,
 126                               ZeroData);
 127
 128   if (ObjSize % 32 != 0) {
 129     SDNode *St1 = DAG.getMachineNode(
 130         ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex, dl,
 131         {MVT::i64, MVT::Other},
 132         {Addr, Addr, DAG.getTargetConstant(1, dl, MVT::i64), Chain});
 133     DAG.setNodeMemRefs(cast<MachineSDNode>(St1), {BaseMemOperand});
 134     ObjSize -= 16;
 135     Addr = SDValue(St1, 0);
 136     Chain = SDValue(St1, 1);
 137   }
 138
 139   const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
 140   SDValue Ops[] = {DAG.getConstant(ObjSize, dl, MVT::i64), Addr, Chain};
 141   SDNode *St = DAG.getMachineNode(
 142       ZeroData ? AArch64::STZGloop : AArch64::STGloop, dl, ResTys, Ops);
 143
 144   DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});
 145   return SDValue(St, 2);
 146 }