llvm/lib/Target/ARM/ARMCallingConv.cpp

   1 //=== ARMCallingConv.cpp - ARM Custom CC Routines ---------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains the custom routines for the ARM Calling Convention that
  10 // aren't done by tablegen, and includes the table generated implementations.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "ARMCallingConv.h"
  15 #include "ARM.h"
  16 #include "ARMSubtarget.h"
  17 using namespace llvm;
  18
  19 // APCS f64 is in register pairs, possibly split to stack
  20 static bool f64AssignAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
  21                           CCValAssign::LocInfo LocInfo,
  22                           CCState &State, bool CanFail) {
  23   static const MCPhysReg RegList[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
  24
  25   // Try to get the first register.
  26   if (MCRegister Reg = State.AllocateReg(RegList))
  27     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
  28   else {
  29     // For the 2nd half of a v2f64, do not fail.
  30     if (CanFail)
  31       return false;
  32
  33     // Put the whole thing on the stack.
  34     State.addLoc(CCValAssign::getCustomMem(
  35         ValNo, ValVT, State.AllocateStack(8, Align(4)), LocVT, LocInfo));
  36     return true;
  37   }
  38
  39   // Try to get the second register.
  40   if (MCRegister Reg = State.AllocateReg(RegList))
  41     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
  42   else
  43     State.addLoc(CCValAssign::getCustomMem(
  44         ValNo, ValVT, State.AllocateStack(4, Align(4)), LocVT, LocInfo));
  45   return true;
  46 }
  47
  48 static bool CC_ARM_APCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
  49                                    CCValAssign::LocInfo LocInfo,
  50                                    ISD::ArgFlagsTy ArgFlags,
  51                                    CCState &State) {
  52   if (!f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))
  53     return false;
  54   if (LocVT == MVT::v2f64 &&
  55       !f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))
  56     return false;
  57   return true;  // we handled it
  58 }
  59
  60 // AAPCS f64 is in aligned register pairs
  61 static bool f64AssignAAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
  62                            CCValAssign::LocInfo LocInfo,
  63                            CCState &State, bool CanFail) {
  64   static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };
  65   static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };
  66   static const MCPhysReg ShadowRegList[] = { ARM::R0, ARM::R1 };
  67   static const MCPhysReg GPRArgRegs[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
  68
  69   MCRegister Reg = State.AllocateReg(HiRegList, ShadowRegList);
  70   if (!Reg) {
  71
  72     // If we had R3 unallocated only, now we still must to waste it.
  73     Reg = State.AllocateReg(GPRArgRegs);
  74     assert((!Reg || Reg == ARM::R3) && "Wrong GPRs usage for f64");
  75
  76     // For the 2nd half of a v2f64, do not just fail.
  77     if (CanFail)
  78       return false;
  79
  80     // Put the whole thing on the stack.
  81     State.addLoc(CCValAssign::getCustomMem(
  82         ValNo, ValVT, State.AllocateStack(8, Align(8)), LocVT, LocInfo));
  83     return true;
  84   }
  85
  86   unsigned i;
  87   for (i = 0; i < 2; ++i)
  88     if (HiRegList[i] == Reg)
  89       break;
  90
  91   MCRegister T = State.AllocateReg(LoRegList[i]);
  92   (void)T;
  93   assert(T == LoRegList[i] && "Could not allocate register");
  94
  95   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
  96   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],
  97                                          LocVT, LocInfo));
  98   return true;
  99 }
 100
 101 static bool CC_ARM_AAPCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
 102                                     CCValAssign::LocInfo LocInfo,
 103                                     ISD::ArgFlagsTy ArgFlags,
 104                                     CCState &State) {
 105   if (!f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))
 106     return false;
 107   if (LocVT == MVT::v2f64 &&
 108       !f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))
 109     return false;
 110   return true;  // we handled it
 111 }
 112
 113 static bool f64RetAssign(unsigned ValNo, MVT ValVT, MVT LocVT,
 114                          CCValAssign::LocInfo LocInfo, CCState &State) {
 115   static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };
 116   static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };
 117
 118   MCRegister Reg = State.AllocateReg(HiRegList, LoRegList);
 119   if (!Reg)
 120     return false; // we didn't handle it
 121
 122   unsigned i;
 123   for (i = 0; i < 2; ++i)
 124     if (HiRegList[i] == Reg)
 125       break;
 126
 127   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
 128   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],
 129                                          LocVT, LocInfo));
 130   return true;
 131 }
 132
 133 static bool RetCC_ARM_APCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
 134                                       CCValAssign::LocInfo LocInfo,
 135                                       ISD::ArgFlagsTy ArgFlags,
 136                                       CCState &State) {
 137   if (!f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
 138     return false;
 139   if (LocVT == MVT::v2f64 && !f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
 140     return false;
 141   return true;  // we handled it
 142 }
 143
 144 static bool RetCC_ARM_AAPCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
 145                                        CCValAssign::LocInfo LocInfo,
 146                                        ISD::ArgFlagsTy ArgFlags,
 147                                        CCState &State) {
 148   return RetCC_ARM_APCS_Custom_f64(ValNo, ValVT, LocVT, LocInfo, ArgFlags,
 149                                    State);
 150 }
 151
 152 static const MCPhysReg RRegList[] = { ARM::R0,  ARM::R1,  ARM::R2,  ARM::R3 };
 153
 154 static const MCPhysReg SRegList[] = { ARM::S0,  ARM::S1,  ARM::S2,  ARM::S3,
 155                                       ARM::S4,  ARM::S5,  ARM::S6,  ARM::S7,
 156                                       ARM::S8,  ARM::S9,  ARM::S10, ARM::S11,
 157                                       ARM::S12, ARM::S13, ARM::S14,  ARM::S15 };
 158 static const MCPhysReg DRegList[] = { ARM::D0, ARM::D1, ARM::D2, ARM::D3,
 159                                       ARM::D4, ARM::D5, ARM::D6, ARM::D7 };
 160 static const MCPhysReg QRegList[] = { ARM::Q0, ARM::Q1, ARM::Q2, ARM::Q3 };
 161
 162
 163 // Allocate part of an AAPCS HFA or HVA. We assume that each member of the HA
 164 // has InConsecutiveRegs set, and that the last member also has
 165 // InConsecutiveRegsLast set. We must process all members of the HA before
 166 // we can allocate it, as we need to know the total number of registers that
 167 // will be needed in order to (attempt to) allocate a contiguous block.
 168 static bool CC_ARM_AAPCS_Custom_Aggregate(unsigned ValNo, MVT ValVT,
 169                                           MVT LocVT,
 170                                           CCValAssign::LocInfo LocInfo,
 171                                           ISD::ArgFlagsTy ArgFlags,
 172                                           CCState &State) {
 173   SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();
 174
 175   // AAPCS HFAs must have 1-4 elements, all of the same type
 176   if (PendingMembers.size() > 0)
 177     assert(PendingMembers[0].getLocVT() == LocVT);
 178
 179   // Add the argument to the list to be allocated once we know the size of the
 180   // aggregate. Store the type's required alignment as extra info for later: in
 181   // the [N x i64] case all trace has been removed by the time we actually get
 182   // to do allocation.
 183   PendingMembers.push_back(CCValAssign::getPending(
 184       ValNo, ValVT, LocVT, LocInfo, ArgFlags.getNonZeroOrigAlign().value()));
 185
 186   if (!ArgFlags.isInConsecutiveRegsLast())
 187     return true;
 188
 189   // Try to allocate a contiguous block of registers, each of the correct
 190   // size to hold one member.
 191   auto &DL = State.getMachineFunction().getDataLayout();
 192   const MaybeAlign StackAlign = DL.getStackAlignment();
 193   assert(StackAlign && "data layout string is missing stack alignment");
 194   const Align FirstMemberAlign(PendingMembers[0].getExtraInfo());
 195   Align Alignment = std::min(FirstMemberAlign, *StackAlign);
 196
 197   ArrayRef<MCPhysReg> RegList;
 198   switch (LocVT.SimpleTy) {
 199   case MVT::i32: {
 200     RegList = RRegList;
 201     unsigned RegIdx = State.getFirstUnallocated(RegList);
 202
 203     // First consume all registers that would give an unaligned object. Whether
 204     // we go on stack or in regs, no-one will be using them in future.
 205     unsigned RegAlign = alignTo(Alignment.value(), 4) / 4;
 206     while (RegIdx % RegAlign != 0 && RegIdx < RegList.size())
 207       State.AllocateReg(RegList[RegIdx++]);
 208
 209     break;
 210   }
 211   case MVT::f16:
 212   case MVT::bf16:
 213   case MVT::f32:
 214     RegList = SRegList;
 215     break;
 216   case MVT::v4f16:
 217   case MVT::v4bf16:
 218   case MVT::f64:
 219     RegList = DRegList;
 220     break;
 221   case MVT::v8f16:
 222   case MVT::v8bf16:
 223   case MVT::v2f64:
 224     RegList = QRegList;
 225     break;
 226   default:
 227     llvm_unreachable("Unexpected member type for block aggregate");
 228     break;
 229   }
 230
 231   unsigned RegResult = State.AllocateRegBlock(RegList, PendingMembers.size());
 232   if (RegResult) {
 233     for (CCValAssign &PendingMember : PendingMembers) {
 234       PendingMember.convertToReg(RegResult);
 235       State.addLoc(PendingMember);
 236       ++RegResult;
 237     }
 238     PendingMembers.clear();
 239     return true;
 240   }
 241
 242   // Register allocation failed, we'll be needing the stack
 243   unsigned Size = LocVT.getSizeInBits() / 8;
 244   if (LocVT == MVT::i32 && State.getStackSize() == 0) {
 245     // If nothing else has used the stack until this point, a non-HFA aggregate
 246     // can be split between regs and stack.
 247     unsigned RegIdx = State.getFirstUnallocated(RegList);
 248     for (auto &It : PendingMembers) {
 249       if (RegIdx >= RegList.size())
 250         It.convertToMem(State.AllocateStack(Size, Align(Size)));
 251       else
 252         It.convertToReg(State.AllocateReg(RegList[RegIdx++]));
 253
 254       State.addLoc(It);
 255     }
 256     PendingMembers.clear();
 257     return true;
 258   }
 259
 260   if (LocVT != MVT::i32)
 261     RegList = SRegList;
 262
 263   // Mark all regs as unavailable (AAPCS rule C.2.vfp for VFP, C.6 for core)
 264   for (auto Reg : RegList)
 265     State.AllocateReg(Reg);
 266
 267   // Clamp the alignment between 4 and 8.
 268   if (State.getMachineFunction().getSubtarget<ARMSubtarget>().isTargetAEABI())
 269     Alignment = ArgFlags.getNonZeroMemAlign() <= 4 ? Align(4) : Align(8);
 270
 271   // After the first item has been allocated, the rest are packed as tightly as
 272   // possible. (E.g. an incoming i64 would have starting Align of 8, but we'll
 273   // be allocating a bunch of i32 slots).
 274   for (auto &It : PendingMembers) {
 275     It.convertToMem(State.AllocateStack(Size, Alignment));
 276     State.addLoc(It);
 277     Alignment = Align(1);
 278   }
 279
 280   // All pending members have now been allocated
 281   PendingMembers.clear();
 282
 283   // This will be allocated by the last member of the aggregate
 284   return true;
 285 }
 286
 287 static bool CustomAssignInRegList(unsigned ValNo, MVT ValVT, MVT LocVT,
 288                                   CCValAssign::LocInfo LocInfo, CCState &State,
 289                                   ArrayRef<MCPhysReg> RegList) {
 290   MCRegister Reg = State.AllocateReg(RegList);
 291   if (Reg) {
 292     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
 293     return true;
 294   }
 295   return false;
 296 }
 297
 298 static bool CC_ARM_AAPCS_Custom_f16(unsigned ValNo, MVT ValVT, MVT LocVT,
 299                                     CCValAssign::LocInfo LocInfo,
 300                                     ISD::ArgFlagsTy ArgFlags, CCState &State) {
 301   // f16 arguments are extended to i32 and assigned to a register in [r0, r3]
 302   return CustomAssignInRegList(ValNo, ValVT, MVT::i32, LocInfo, State,
 303                                RRegList);
 304 }
 305
 306 static bool CC_ARM_AAPCS_VFP_Custom_f16(unsigned ValNo, MVT ValVT, MVT LocVT,
 307                                         CCValAssign::LocInfo LocInfo,
 308                                         ISD::ArgFlagsTy ArgFlags,
 309                                         CCState &State) {
 310   // f16 arguments are extended to f32 and assigned to a register in [s0, s15]
 311   return CustomAssignInRegList(ValNo, ValVT, MVT::f32, LocInfo, State,
 312                                SRegList);
 313 }
 314
 315 // Include the table generated calling convention implementations.
 316 #include "ARMGenCallingConv.inc"