llvm/lib/Target/ARM/ARMCallingConv.cpp

   1 //=== ARMCallingConv.cpp - ARM Custom CC Routines ---------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains the custom routines for the ARM Calling Convention that
  10 // aren't done by tablegen, and includes the table generated implementations.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "ARM.h"
  15 #include "ARMCallingConv.h"
  16 #include "ARMSubtarget.h"
  17 #include "ARMRegisterInfo.h"
  18 using namespace llvm;
  19
  20 // APCS f64 is in register pairs, possibly split to stack
  21 static bool f64AssignAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
  22                           CCValAssign::LocInfo LocInfo,
  23                           CCState &State, bool CanFail) {
  24   static const MCPhysReg RegList[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
  25
  26   // Try to get the first register.
  27   if (unsigned Reg = State.AllocateReg(RegList))
  28     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
  29   else {
  30     // For the 2nd half of a v2f64, do not fail.
  31     if (CanFail)
  32       return false;
  33
  34     // Put the whole thing on the stack.
  35     State.addLoc(CCValAssign::getCustomMem(
  36         ValNo, ValVT, State.AllocateStack(8, Align(4)), LocVT, LocInfo));
  37     return true;
  38   }
  39
  40   // Try to get the second register.
  41   if (unsigned Reg = State.AllocateReg(RegList))
  42     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
  43   else
  44     State.addLoc(CCValAssign::getCustomMem(
  45         ValNo, ValVT, State.AllocateStack(4, Align(4)), LocVT, LocInfo));
  46   return true;
  47 }
  48
  49 static bool CC_ARM_APCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
  50                                    CCValAssign::LocInfo LocInfo,
  51                                    ISD::ArgFlagsTy ArgFlags,
  52                                    CCState &State) {
  53   if (!f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))
  54     return false;
  55   if (LocVT == MVT::v2f64 &&
  56       !f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))
  57     return false;
  58   return true;  // we handled it
  59 }
  60
  61 // AAPCS f64 is in aligned register pairs
  62 static bool f64AssignAAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
  63                            CCValAssign::LocInfo LocInfo,
  64                            CCState &State, bool CanFail) {
  65   static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };
  66   static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };
  67   static const MCPhysReg ShadowRegList[] = { ARM::R0, ARM::R1 };
  68   static const MCPhysReg GPRArgRegs[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
  69
  70   unsigned Reg = State.AllocateReg(HiRegList, ShadowRegList);
  71   if (Reg == 0) {
  72
  73     // If we had R3 unallocated only, now we still must to waste it.
  74     Reg = State.AllocateReg(GPRArgRegs);
  75     assert((!Reg || Reg == ARM::R3) && "Wrong GPRs usage for f64");
  76
  77     // For the 2nd half of a v2f64, do not just fail.
  78     if (CanFail)
  79       return false;
  80
  81     // Put the whole thing on the stack.
  82     State.addLoc(CCValAssign::getCustomMem(
  83         ValNo, ValVT, State.AllocateStack(8, Align(8)), LocVT, LocInfo));
  84     return true;
  85   }
  86
  87   unsigned i;
  88   for (i = 0; i < 2; ++i)
  89     if (HiRegList[i] == Reg)
  90       break;
  91
  92   unsigned T = State.AllocateReg(LoRegList[i]);
  93   (void)T;
  94   assert(T == LoRegList[i] && "Could not allocate register");
  95
  96   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
  97   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],
  98                                          LocVT, LocInfo));
  99   return true;
 100 }
 101
 102 static bool CC_ARM_AAPCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
 103                                     CCValAssign::LocInfo LocInfo,
 104                                     ISD::ArgFlagsTy ArgFlags,
 105                                     CCState &State) {
 106   if (!f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))
 107     return false;
 108   if (LocVT == MVT::v2f64 &&
 109       !f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))
 110     return false;
 111   return true;  // we handled it
 112 }
 113
 114 static bool f64RetAssign(unsigned ValNo, MVT ValVT, MVT LocVT,
 115                          CCValAssign::LocInfo LocInfo, CCState &State) {
 116   static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };
 117   static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };
 118
 119   unsigned Reg = State.AllocateReg(HiRegList, LoRegList);
 120   if (Reg == 0)
 121     return false; // we didn't handle it
 122
 123   unsigned i;
 124   for (i = 0; i < 2; ++i)
 125     if (HiRegList[i] == Reg)
 126       break;
 127
 128   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
 129   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],
 130                                          LocVT, LocInfo));
 131   return true;
 132 }
 133
 134 static bool RetCC_ARM_APCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
 135                                       CCValAssign::LocInfo LocInfo,
 136                                       ISD::ArgFlagsTy ArgFlags,
 137                                       CCState &State) {
 138   if (!f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
 139     return false;
 140   if (LocVT == MVT::v2f64 && !f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
 141     return false;
 142   return true;  // we handled it
 143 }
 144
 145 static bool RetCC_ARM_AAPCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
 146                                        CCValAssign::LocInfo LocInfo,
 147                                        ISD::ArgFlagsTy ArgFlags,
 148                                        CCState &State) {
 149   return RetCC_ARM_APCS_Custom_f64(ValNo, ValVT, LocVT, LocInfo, ArgFlags,
 150                                    State);
 151 }
 152
 153 static const MCPhysReg RRegList[] = { ARM::R0,  ARM::R1,  ARM::R2,  ARM::R3 };
 154
 155 static const MCPhysReg SRegList[] = { ARM::S0,  ARM::S1,  ARM::S2,  ARM::S3,
 156                                       ARM::S4,  ARM::S5,  ARM::S6,  ARM::S7,
 157                                       ARM::S8,  ARM::S9,  ARM::S10, ARM::S11,
 158                                       ARM::S12, ARM::S13, ARM::S14,  ARM::S15 };
 159 static const MCPhysReg DRegList[] = { ARM::D0, ARM::D1, ARM::D2, ARM::D3,
 160                                       ARM::D4, ARM::D5, ARM::D6, ARM::D7 };
 161 static const MCPhysReg QRegList[] = { ARM::Q0, ARM::Q1, ARM::Q2, ARM::Q3 };
 162
 163
 164 // Allocate part of an AAPCS HFA or HVA. We assume that each member of the HA
 165 // has InConsecutiveRegs set, and that the last member also has
 166 // InConsecutiveRegsLast set. We must process all members of the HA before
 167 // we can allocate it, as we need to know the total number of registers that
 168 // will be needed in order to (attempt to) allocate a contiguous block.
 169 static bool CC_ARM_AAPCS_Custom_Aggregate(unsigned ValNo, MVT ValVT,
 170                                           MVT LocVT,
 171                                           CCValAssign::LocInfo LocInfo,
 172                                           ISD::ArgFlagsTy ArgFlags,
 173                                           CCState &State) {
 174   SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();
 175
 176   // AAPCS HFAs must have 1-4 elements, all of the same type
 177   if (PendingMembers.size() > 0)
 178     assert(PendingMembers[0].getLocVT() == LocVT);
 179
 180   // Add the argument to the list to be allocated once we know the size of the
 181   // aggregate. Store the type's required alignment as extra info for later: in
 182   // the [N x i64] case all trace has been removed by the time we actually get
 183   // to do allocation.
 184   PendingMembers.push_back(CCValAssign::getPending(
 185       ValNo, ValVT, LocVT, LocInfo, ArgFlags.getNonZeroOrigAlign().value()));
 186
 187   if (!ArgFlags.isInConsecutiveRegsLast())
 188     return true;
 189
 190   // Try to allocate a contiguous block of registers, each of the correct
 191   // size to hold one member.
 192   auto &DL = State.getMachineFunction().getDataLayout();
 193   const Align StackAlign = DL.getStackAlignment();
 194   const Align FirstMemberAlign(PendingMembers[0].getExtraInfo());
 195   Align Alignment = std::min(FirstMemberAlign, StackAlign);
 196
 197   ArrayRef<MCPhysReg> RegList;
 198   switch (LocVT.SimpleTy) {
 199   case MVT::i32: {
 200     RegList = RRegList;
 201     unsigned RegIdx = State.getFirstUnallocated(RegList);
 202
 203     // First consume all registers that would give an unaligned object. Whether
 204     // we go on stack or in regs, no-one will be using them in future.
 205     unsigned RegAlign = alignTo(Alignment.value(), 4) / 4;
 206     while (RegIdx % RegAlign != 0 && RegIdx < RegList.size())
 207       State.AllocateReg(RegList[RegIdx++]);
 208
 209     break;
 210   }
 211   case MVT::f16:
 212   case MVT::bf16:
 213   case MVT::f32:
 214     RegList = SRegList;
 215     break;
 216   case MVT::v4f16:
 217   case MVT::v4bf16:
 218   case MVT::f64:
 219     RegList = DRegList;
 220     break;
 221   case MVT::v8f16:
 222   case MVT::v8bf16:
 223   case MVT::v2f64:
 224     RegList = QRegList;
 225     break;
 226   default:
 227     llvm_unreachable("Unexpected member type for block aggregate");
 228     break;
 229   }
 230
 231   unsigned RegResult = State.AllocateRegBlock(RegList, PendingMembers.size());
 232   if (RegResult) {
 233     for (SmallVectorImpl<CCValAssign>::iterator It = PendingMembers.begin();
 234          It != PendingMembers.end(); ++It) {
 235       It->convertToReg(RegResult);
 236       State.addLoc(*It);
 237       ++RegResult;
 238     }
 239     PendingMembers.clear();
 240     return true;
 241   }
 242
 243   // Register allocation failed, we'll be needing the stack
 244   unsigned Size = LocVT.getSizeInBits() / 8;
 245   if (LocVT == MVT::i32 && State.getNextStackOffset() == 0) {
 246     // If nothing else has used the stack until this point, a non-HFA aggregate
 247     // can be split between regs and stack.
 248     unsigned RegIdx = State.getFirstUnallocated(RegList);
 249     for (auto &It : PendingMembers) {
 250       if (RegIdx >= RegList.size())
 251         It.convertToMem(State.AllocateStack(Size, Align(Size)));
 252       else
 253         It.convertToReg(State.AllocateReg(RegList[RegIdx++]));
 254
 255       State.addLoc(It);
 256     }
 257     PendingMembers.clear();
 258     return true;
 259   }
 260
 261   if (LocVT != MVT::i32)
 262     RegList = SRegList;
 263
 264   // Mark all regs as unavailable (AAPCS rule C.2.vfp for VFP, C.6 for core)
 265   for (auto Reg : RegList)
 266     State.AllocateReg(Reg);
 267
 268   // Clamp the alignment between 4 and 8.
 269   if (State.getMachineFunction().getSubtarget<ARMSubtarget>().isTargetAEABI())
 270     Alignment = ArgFlags.getNonZeroMemAlign() <= 4 ? Align(4) : Align(8);
 271
 272   // After the first item has been allocated, the rest are packed as tightly as
 273   // possible. (E.g. an incoming i64 would have starting Align of 8, but we'll
 274   // be allocating a bunch of i32 slots).
 275   for (auto &It : PendingMembers) {
 276     It.convertToMem(State.AllocateStack(Size, Alignment));
 277     State.addLoc(It);
 278     Alignment = Align(1);
 279   }
 280
 281   // All pending members have now been allocated
 282   PendingMembers.clear();
 283
 284   // This will be allocated by the last member of the aggregate
 285   return true;
 286 }
 287
 288 static bool CustomAssignInRegList(unsigned ValNo, MVT ValVT, MVT LocVT,
 289                                   CCValAssign::LocInfo LocInfo, CCState &State,
 290                                   ArrayRef<MCPhysReg> RegList) {
 291   unsigned Reg = State.AllocateReg(RegList);
 292   if (Reg) {
 293     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
 294     return true;
 295   }
 296   return false;
 297 }
 298
 299 static bool CC_ARM_AAPCS_Custom_f16(unsigned ValNo, MVT ValVT, MVT LocVT,
 300                                     CCValAssign::LocInfo LocInfo,
 301                                     ISD::ArgFlagsTy ArgFlags, CCState &State) {
 302   // f16 arguments are extended to i32 and assigned to a register in [r0, r3]
 303   return CustomAssignInRegList(ValNo, ValVT, MVT::i32, LocInfo, State,
 304                                RRegList);
 305 }
 306
 307 static bool CC_ARM_AAPCS_VFP_Custom_f16(unsigned ValNo, MVT ValVT, MVT LocVT,
 308                                         CCValAssign::LocInfo LocInfo,
 309                                         ISD::ArgFlagsTy ArgFlags,
 310                                         CCState &State) {
 311   // f16 arguments are extended to f32 and assigned to a register in [s0, s15]
 312   return CustomAssignInRegList(ValNo, ValVT, MVT::f32, LocInfo, State,
 313                                SRegList);
 314 }
 315
 316 // Include the table generated calling convention implementations.
 317 #include "ARMGenCallingConv.inc"