1 //===- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ----===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This implements the TargetLoweringBase class.
11 //===----------------------------------------------------------------------===//
13 #include "llvm/ADT/BitVector.h"
14 #include "llvm/ADT/STLExtras.h"
15 #include "llvm/ADT/SmallVector.h"
16 #include "llvm/ADT/StringExtras.h"
17 #include "llvm/ADT/StringRef.h"
18 #include "llvm/ADT/Twine.h"
19 #include "llvm/Analysis/Loads.h"
20 #include "llvm/Analysis/TargetTransformInfo.h"
21 #include "llvm/CodeGen/Analysis.h"
22 #include "llvm/CodeGen/ISDOpcodes.h"
23 #include "llvm/CodeGen/MachineBasicBlock.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineFunction.h"
26 #include "llvm/CodeGen/MachineInstr.h"
27 #include "llvm/CodeGen/MachineInstrBuilder.h"
28 #include "llvm/CodeGen/MachineMemOperand.h"
29 #include "llvm/CodeGen/MachineOperand.h"
30 #include "llvm/CodeGen/MachineRegisterInfo.h"
31 #include "llvm/CodeGen/MachineValueType.h"
32 #include "llvm/CodeGen/RuntimeLibcalls.h"
33 #include "llvm/CodeGen/StackMaps.h"
34 #include "llvm/CodeGen/TargetLowering.h"
35 #include "llvm/CodeGen/TargetOpcodes.h"
36 #include "llvm/CodeGen/TargetRegisterInfo.h"
37 #include "llvm/CodeGen/ValueTypes.h"
38 #include "llvm/IR/Attributes.h"
39 #include "llvm/IR/CallingConv.h"
40 #include "llvm/IR/DataLayout.h"
41 #include "llvm/IR/DerivedTypes.h"
42 #include "llvm/IR/Function.h"
43 #include "llvm/IR/GlobalValue.h"
44 #include "llvm/IR/GlobalVariable.h"
45 #include "llvm/IR/IRBuilder.h"
46 #include "llvm/IR/Module.h"
47 #include "llvm/IR/Type.h"
48 #include "llvm/Support/Casting.h"
49 #include "llvm/Support/CommandLine.h"
50 #include "llvm/Support/Compiler.h"
51 #include "llvm/Support/ErrorHandling.h"
52 #include "llvm/Support/MathExtras.h"
53 #include "llvm/Target/TargetMachine.h"
54 #include "llvm/Target/TargetOptions.h"
55 #include "llvm/TargetParser/Triple.h"
56 #include "llvm/Transforms/Utils/SizeOpts.h"
68 static cl::opt
<bool> JumpIsExpensiveOverride(
69 "jump-is-expensive", cl::init(false),
70 cl::desc("Do not create extra branches to split comparison logic."),
73 static cl::opt
<unsigned> MinimumJumpTableEntries
74 ("min-jump-table-entries", cl::init(4), cl::Hidden
,
75 cl::desc("Set minimum number of entries to use a jump table."));
77 static cl::opt
<unsigned> MaximumJumpTableSize
78 ("max-jump-table-size", cl::init(UINT_MAX
), cl::Hidden
,
79 cl::desc("Set maximum size of jump tables."));
81 /// Minimum jump table density for normal functions.
82 static cl::opt
<unsigned>
83 JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden
,
84 cl::desc("Minimum density for building a jump table in "
85 "a normal function"));
87 /// Minimum jump table density for -Os or -Oz functions.
88 static cl::opt
<unsigned> OptsizeJumpTableDensity(
89 "optsize-jump-table-density", cl::init(40), cl::Hidden
,
90 cl::desc("Minimum density for building a jump table in "
91 "an optsize function"));
93 // FIXME: This option is only to test if the strict fp operation processed
94 // correctly by preventing mutating strict fp operation to normal fp operation
95 // during development. When the backend supports strict float operation, this
96 // option will be meaningless.
97 static cl::opt
<bool> DisableStrictNodeMutation("disable-strictnode-mutation",
98 cl::desc("Don't mutate strict-float node to a legalize node"),
99 cl::init(false), cl::Hidden
);
101 static bool darwinHasSinCos(const Triple
&TT
) {
102 assert(TT
.isOSDarwin() && "should be called with darwin triple");
103 // Don't bother with 32 bit x86.
104 if (TT
.getArch() == Triple::x86
)
106 // Macos < 10.9 has no sincos_stret.
108 return !TT
.isMacOSXVersionLT(10, 9) && TT
.isArch64Bit();
109 // iOS < 7.0 has no sincos_stret.
111 return !TT
.isOSVersionLT(7, 0);
112 // Any other darwin such as WatchOS/TvOS is new enough.
116 void TargetLoweringBase::InitLibcalls(const Triple
&TT
) {
117 #define HANDLE_LIBCALL(code, name) \
118 setLibcallName(RTLIB::code, name);
119 #include "llvm/IR/RuntimeLibcalls.def"
120 #undef HANDLE_LIBCALL
121 // Initialize calling conventions to their default.
122 for (int LC
= 0; LC
< RTLIB::UNKNOWN_LIBCALL
; ++LC
)
123 setLibcallCallingConv((RTLIB::Libcall
)LC
, CallingConv::C
);
125 // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
127 setLibcallName(RTLIB::ADD_F128
, "__addkf3");
128 setLibcallName(RTLIB::SUB_F128
, "__subkf3");
129 setLibcallName(RTLIB::MUL_F128
, "__mulkf3");
130 setLibcallName(RTLIB::DIV_F128
, "__divkf3");
131 setLibcallName(RTLIB::POWI_F128
, "__powikf2");
132 setLibcallName(RTLIB::FPEXT_F32_F128
, "__extendsfkf2");
133 setLibcallName(RTLIB::FPEXT_F64_F128
, "__extenddfkf2");
134 setLibcallName(RTLIB::FPROUND_F128_F32
, "__trunckfsf2");
135 setLibcallName(RTLIB::FPROUND_F128_F64
, "__trunckfdf2");
136 setLibcallName(RTLIB::FPTOSINT_F128_I32
, "__fixkfsi");
137 setLibcallName(RTLIB::FPTOSINT_F128_I64
, "__fixkfdi");
138 setLibcallName(RTLIB::FPTOSINT_F128_I128
, "__fixkfti");
139 setLibcallName(RTLIB::FPTOUINT_F128_I32
, "__fixunskfsi");
140 setLibcallName(RTLIB::FPTOUINT_F128_I64
, "__fixunskfdi");
141 setLibcallName(RTLIB::FPTOUINT_F128_I128
, "__fixunskfti");
142 setLibcallName(RTLIB::SINTTOFP_I32_F128
, "__floatsikf");
143 setLibcallName(RTLIB::SINTTOFP_I64_F128
, "__floatdikf");
144 setLibcallName(RTLIB::SINTTOFP_I128_F128
, "__floattikf");
145 setLibcallName(RTLIB::UINTTOFP_I32_F128
, "__floatunsikf");
146 setLibcallName(RTLIB::UINTTOFP_I64_F128
, "__floatundikf");
147 setLibcallName(RTLIB::UINTTOFP_I128_F128
, "__floatuntikf");
148 setLibcallName(RTLIB::OEQ_F128
, "__eqkf2");
149 setLibcallName(RTLIB::UNE_F128
, "__nekf2");
150 setLibcallName(RTLIB::OGE_F128
, "__gekf2");
151 setLibcallName(RTLIB::OLT_F128
, "__ltkf2");
152 setLibcallName(RTLIB::OLE_F128
, "__lekf2");
153 setLibcallName(RTLIB::OGT_F128
, "__gtkf2");
154 setLibcallName(RTLIB::UO_F128
, "__unordkf2");
157 // A few names are different on particular architectures or environments.
158 if (TT
.isOSDarwin()) {
159 // For f16/f32 conversions, Darwin uses the standard naming scheme, instead
160 // of the gnueabi-style __gnu_*_ieee.
161 // FIXME: What about other targets?
162 setLibcallName(RTLIB::FPEXT_F16_F32
, "__extendhfsf2");
163 setLibcallName(RTLIB::FPROUND_F32_F16
, "__truncsfhf2");
165 // Some darwins have an optimized __bzero/bzero function.
166 switch (TT
.getArch()) {
169 if (TT
.isMacOSX() && !TT
.isMacOSXVersionLT(10, 6))
170 setLibcallName(RTLIB::BZERO
, "__bzero");
172 case Triple::aarch64
:
173 case Triple::aarch64_32
:
174 setLibcallName(RTLIB::BZERO
, "bzero");
180 if (darwinHasSinCos(TT
)) {
181 setLibcallName(RTLIB::SINCOS_STRET_F32
, "__sincosf_stret");
182 setLibcallName(RTLIB::SINCOS_STRET_F64
, "__sincos_stret");
183 if (TT
.isWatchABI()) {
184 setLibcallCallingConv(RTLIB::SINCOS_STRET_F32
,
185 CallingConv::ARM_AAPCS_VFP
);
186 setLibcallCallingConv(RTLIB::SINCOS_STRET_F64
,
187 CallingConv::ARM_AAPCS_VFP
);
191 setLibcallName(RTLIB::FPEXT_F16_F32
, "__gnu_h2f_ieee");
192 setLibcallName(RTLIB::FPROUND_F32_F16
, "__gnu_f2h_ieee");
195 if (TT
.isGNUEnvironment() || TT
.isOSFuchsia() ||
196 (TT
.isAndroid() && !TT
.isAndroidVersionLT(9))) {
197 setLibcallName(RTLIB::SINCOS_F32
, "sincosf");
198 setLibcallName(RTLIB::SINCOS_F64
, "sincos");
199 setLibcallName(RTLIB::SINCOS_F80
, "sincosl");
200 setLibcallName(RTLIB::SINCOS_F128
, "sincosl");
201 setLibcallName(RTLIB::SINCOS_PPCF128
, "sincosl");
205 setLibcallName(RTLIB::SINCOS_F32
, "sincosf");
206 setLibcallName(RTLIB::SINCOS_F64
, "sincos");
209 if (TT
.isOSOpenBSD()) {
210 setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL
, nullptr);
213 if (TT
.isOSWindows() && !TT
.isOSCygMing()) {
214 setLibcallName(RTLIB::LDEXP_F32
, nullptr);
215 setLibcallName(RTLIB::LDEXP_F80
, nullptr);
216 setLibcallName(RTLIB::LDEXP_F128
, nullptr);
217 setLibcallName(RTLIB::LDEXP_PPCF128
, nullptr);
219 setLibcallName(RTLIB::FREXP_F32
, nullptr);
220 setLibcallName(RTLIB::FREXP_F80
, nullptr);
221 setLibcallName(RTLIB::FREXP_F128
, nullptr);
222 setLibcallName(RTLIB::FREXP_PPCF128
, nullptr);
226 /// GetFPLibCall - Helper to return the right libcall for the given floating
227 /// point type, or UNKNOWN_LIBCALL if there is none.
228 RTLIB::Libcall
RTLIB::getFPLibCall(EVT VT
,
229 RTLIB::Libcall Call_F32
,
230 RTLIB::Libcall Call_F64
,
231 RTLIB::Libcall Call_F80
,
232 RTLIB::Libcall Call_F128
,
233 RTLIB::Libcall Call_PPCF128
) {
235 VT
== MVT::f32
? Call_F32
:
236 VT
== MVT::f64
? Call_F64
:
237 VT
== MVT::f80
? Call_F80
:
238 VT
== MVT::f128
? Call_F128
:
239 VT
== MVT::ppcf128
? Call_PPCF128
:
240 RTLIB::UNKNOWN_LIBCALL
;
243 /// getFPEXT - Return the FPEXT_*_* value for the given types, or
244 /// UNKNOWN_LIBCALL if there is none.
245 RTLIB::Libcall
RTLIB::getFPEXT(EVT OpVT
, EVT RetVT
) {
246 if (OpVT
== MVT::f16
) {
247 if (RetVT
== MVT::f32
)
248 return FPEXT_F16_F32
;
249 if (RetVT
== MVT::f64
)
250 return FPEXT_F16_F64
;
251 if (RetVT
== MVT::f80
)
252 return FPEXT_F16_F80
;
253 if (RetVT
== MVT::f128
)
254 return FPEXT_F16_F128
;
255 } else if (OpVT
== MVT::f32
) {
256 if (RetVT
== MVT::f64
)
257 return FPEXT_F32_F64
;
258 if (RetVT
== MVT::f128
)
259 return FPEXT_F32_F128
;
260 if (RetVT
== MVT::ppcf128
)
261 return FPEXT_F32_PPCF128
;
262 } else if (OpVT
== MVT::f64
) {
263 if (RetVT
== MVT::f128
)
264 return FPEXT_F64_F128
;
265 else if (RetVT
== MVT::ppcf128
)
266 return FPEXT_F64_PPCF128
;
267 } else if (OpVT
== MVT::f80
) {
268 if (RetVT
== MVT::f128
)
269 return FPEXT_F80_F128
;
272 return UNKNOWN_LIBCALL
;
275 /// getFPROUND - Return the FPROUND_*_* value for the given types, or
276 /// UNKNOWN_LIBCALL if there is none.
277 RTLIB::Libcall
RTLIB::getFPROUND(EVT OpVT
, EVT RetVT
) {
278 if (RetVT
== MVT::f16
) {
279 if (OpVT
== MVT::f32
)
280 return FPROUND_F32_F16
;
281 if (OpVT
== MVT::f64
)
282 return FPROUND_F64_F16
;
283 if (OpVT
== MVT::f80
)
284 return FPROUND_F80_F16
;
285 if (OpVT
== MVT::f128
)
286 return FPROUND_F128_F16
;
287 if (OpVT
== MVT::ppcf128
)
288 return FPROUND_PPCF128_F16
;
289 } else if (RetVT
== MVT::bf16
) {
290 if (OpVT
== MVT::f32
)
291 return FPROUND_F32_BF16
;
292 if (OpVT
== MVT::f64
)
293 return FPROUND_F64_BF16
;
294 } else if (RetVT
== MVT::f32
) {
295 if (OpVT
== MVT::f64
)
296 return FPROUND_F64_F32
;
297 if (OpVT
== MVT::f80
)
298 return FPROUND_F80_F32
;
299 if (OpVT
== MVT::f128
)
300 return FPROUND_F128_F32
;
301 if (OpVT
== MVT::ppcf128
)
302 return FPROUND_PPCF128_F32
;
303 } else if (RetVT
== MVT::f64
) {
304 if (OpVT
== MVT::f80
)
305 return FPROUND_F80_F64
;
306 if (OpVT
== MVT::f128
)
307 return FPROUND_F128_F64
;
308 if (OpVT
== MVT::ppcf128
)
309 return FPROUND_PPCF128_F64
;
310 } else if (RetVT
== MVT::f80
) {
311 if (OpVT
== MVT::f128
)
312 return FPROUND_F128_F80
;
315 return UNKNOWN_LIBCALL
;
318 /// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or
319 /// UNKNOWN_LIBCALL if there is none.
320 RTLIB::Libcall
RTLIB::getFPTOSINT(EVT OpVT
, EVT RetVT
) {
321 if (OpVT
== MVT::f16
) {
322 if (RetVT
== MVT::i32
)
323 return FPTOSINT_F16_I32
;
324 if (RetVT
== MVT::i64
)
325 return FPTOSINT_F16_I64
;
326 if (RetVT
== MVT::i128
)
327 return FPTOSINT_F16_I128
;
328 } else if (OpVT
== MVT::f32
) {
329 if (RetVT
== MVT::i32
)
330 return FPTOSINT_F32_I32
;
331 if (RetVT
== MVT::i64
)
332 return FPTOSINT_F32_I64
;
333 if (RetVT
== MVT::i128
)
334 return FPTOSINT_F32_I128
;
335 } else if (OpVT
== MVT::f64
) {
336 if (RetVT
== MVT::i32
)
337 return FPTOSINT_F64_I32
;
338 if (RetVT
== MVT::i64
)
339 return FPTOSINT_F64_I64
;
340 if (RetVT
== MVT::i128
)
341 return FPTOSINT_F64_I128
;
342 } else if (OpVT
== MVT::f80
) {
343 if (RetVT
== MVT::i32
)
344 return FPTOSINT_F80_I32
;
345 if (RetVT
== MVT::i64
)
346 return FPTOSINT_F80_I64
;
347 if (RetVT
== MVT::i128
)
348 return FPTOSINT_F80_I128
;
349 } else if (OpVT
== MVT::f128
) {
350 if (RetVT
== MVT::i32
)
351 return FPTOSINT_F128_I32
;
352 if (RetVT
== MVT::i64
)
353 return FPTOSINT_F128_I64
;
354 if (RetVT
== MVT::i128
)
355 return FPTOSINT_F128_I128
;
356 } else if (OpVT
== MVT::ppcf128
) {
357 if (RetVT
== MVT::i32
)
358 return FPTOSINT_PPCF128_I32
;
359 if (RetVT
== MVT::i64
)
360 return FPTOSINT_PPCF128_I64
;
361 if (RetVT
== MVT::i128
)
362 return FPTOSINT_PPCF128_I128
;
364 return UNKNOWN_LIBCALL
;
367 /// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or
368 /// UNKNOWN_LIBCALL if there is none.
369 RTLIB::Libcall
RTLIB::getFPTOUINT(EVT OpVT
, EVT RetVT
) {
370 if (OpVT
== MVT::f16
) {
371 if (RetVT
== MVT::i32
)
372 return FPTOUINT_F16_I32
;
373 if (RetVT
== MVT::i64
)
374 return FPTOUINT_F16_I64
;
375 if (RetVT
== MVT::i128
)
376 return FPTOUINT_F16_I128
;
377 } else if (OpVT
== MVT::f32
) {
378 if (RetVT
== MVT::i32
)
379 return FPTOUINT_F32_I32
;
380 if (RetVT
== MVT::i64
)
381 return FPTOUINT_F32_I64
;
382 if (RetVT
== MVT::i128
)
383 return FPTOUINT_F32_I128
;
384 } else if (OpVT
== MVT::f64
) {
385 if (RetVT
== MVT::i32
)
386 return FPTOUINT_F64_I32
;
387 if (RetVT
== MVT::i64
)
388 return FPTOUINT_F64_I64
;
389 if (RetVT
== MVT::i128
)
390 return FPTOUINT_F64_I128
;
391 } else if (OpVT
== MVT::f80
) {
392 if (RetVT
== MVT::i32
)
393 return FPTOUINT_F80_I32
;
394 if (RetVT
== MVT::i64
)
395 return FPTOUINT_F80_I64
;
396 if (RetVT
== MVT::i128
)
397 return FPTOUINT_F80_I128
;
398 } else if (OpVT
== MVT::f128
) {
399 if (RetVT
== MVT::i32
)
400 return FPTOUINT_F128_I32
;
401 if (RetVT
== MVT::i64
)
402 return FPTOUINT_F128_I64
;
403 if (RetVT
== MVT::i128
)
404 return FPTOUINT_F128_I128
;
405 } else if (OpVT
== MVT::ppcf128
) {
406 if (RetVT
== MVT::i32
)
407 return FPTOUINT_PPCF128_I32
;
408 if (RetVT
== MVT::i64
)
409 return FPTOUINT_PPCF128_I64
;
410 if (RetVT
== MVT::i128
)
411 return FPTOUINT_PPCF128_I128
;
413 return UNKNOWN_LIBCALL
;
416 /// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or
417 /// UNKNOWN_LIBCALL if there is none.
418 RTLIB::Libcall
RTLIB::getSINTTOFP(EVT OpVT
, EVT RetVT
) {
419 if (OpVT
== MVT::i32
) {
420 if (RetVT
== MVT::f16
)
421 return SINTTOFP_I32_F16
;
422 if (RetVT
== MVT::f32
)
423 return SINTTOFP_I32_F32
;
424 if (RetVT
== MVT::f64
)
425 return SINTTOFP_I32_F64
;
426 if (RetVT
== MVT::f80
)
427 return SINTTOFP_I32_F80
;
428 if (RetVT
== MVT::f128
)
429 return SINTTOFP_I32_F128
;
430 if (RetVT
== MVT::ppcf128
)
431 return SINTTOFP_I32_PPCF128
;
432 } else if (OpVT
== MVT::i64
) {
433 if (RetVT
== MVT::f16
)
434 return SINTTOFP_I64_F16
;
435 if (RetVT
== MVT::f32
)
436 return SINTTOFP_I64_F32
;
437 if (RetVT
== MVT::f64
)
438 return SINTTOFP_I64_F64
;
439 if (RetVT
== MVT::f80
)
440 return SINTTOFP_I64_F80
;
441 if (RetVT
== MVT::f128
)
442 return SINTTOFP_I64_F128
;
443 if (RetVT
== MVT::ppcf128
)
444 return SINTTOFP_I64_PPCF128
;
445 } else if (OpVT
== MVT::i128
) {
446 if (RetVT
== MVT::f16
)
447 return SINTTOFP_I128_F16
;
448 if (RetVT
== MVT::f32
)
449 return SINTTOFP_I128_F32
;
450 if (RetVT
== MVT::f64
)
451 return SINTTOFP_I128_F64
;
452 if (RetVT
== MVT::f80
)
453 return SINTTOFP_I128_F80
;
454 if (RetVT
== MVT::f128
)
455 return SINTTOFP_I128_F128
;
456 if (RetVT
== MVT::ppcf128
)
457 return SINTTOFP_I128_PPCF128
;
459 return UNKNOWN_LIBCALL
;
462 /// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or
463 /// UNKNOWN_LIBCALL if there is none.
464 RTLIB::Libcall
RTLIB::getUINTTOFP(EVT OpVT
, EVT RetVT
) {
465 if (OpVT
== MVT::i32
) {
466 if (RetVT
== MVT::f16
)
467 return UINTTOFP_I32_F16
;
468 if (RetVT
== MVT::f32
)
469 return UINTTOFP_I32_F32
;
470 if (RetVT
== MVT::f64
)
471 return UINTTOFP_I32_F64
;
472 if (RetVT
== MVT::f80
)
473 return UINTTOFP_I32_F80
;
474 if (RetVT
== MVT::f128
)
475 return UINTTOFP_I32_F128
;
476 if (RetVT
== MVT::ppcf128
)
477 return UINTTOFP_I32_PPCF128
;
478 } else if (OpVT
== MVT::i64
) {
479 if (RetVT
== MVT::f16
)
480 return UINTTOFP_I64_F16
;
481 if (RetVT
== MVT::f32
)
482 return UINTTOFP_I64_F32
;
483 if (RetVT
== MVT::f64
)
484 return UINTTOFP_I64_F64
;
485 if (RetVT
== MVT::f80
)
486 return UINTTOFP_I64_F80
;
487 if (RetVT
== MVT::f128
)
488 return UINTTOFP_I64_F128
;
489 if (RetVT
== MVT::ppcf128
)
490 return UINTTOFP_I64_PPCF128
;
491 } else if (OpVT
== MVT::i128
) {
492 if (RetVT
== MVT::f16
)
493 return UINTTOFP_I128_F16
;
494 if (RetVT
== MVT::f32
)
495 return UINTTOFP_I128_F32
;
496 if (RetVT
== MVT::f64
)
497 return UINTTOFP_I128_F64
;
498 if (RetVT
== MVT::f80
)
499 return UINTTOFP_I128_F80
;
500 if (RetVT
== MVT::f128
)
501 return UINTTOFP_I128_F128
;
502 if (RetVT
== MVT::ppcf128
)
503 return UINTTOFP_I128_PPCF128
;
505 return UNKNOWN_LIBCALL
;
508 RTLIB::Libcall
RTLIB::getPOWI(EVT RetVT
) {
509 return getFPLibCall(RetVT
, POWI_F32
, POWI_F64
, POWI_F80
, POWI_F128
,
513 RTLIB::Libcall
RTLIB::getLDEXP(EVT RetVT
) {
514 return getFPLibCall(RetVT
, LDEXP_F32
, LDEXP_F64
, LDEXP_F80
, LDEXP_F128
,
518 RTLIB::Libcall
RTLIB::getFREXP(EVT RetVT
) {
519 return getFPLibCall(RetVT
, FREXP_F32
, FREXP_F64
, FREXP_F80
, FREXP_F128
,
523 RTLIB::Libcall
RTLIB::getOUTLINE_ATOMIC(unsigned Opc
, AtomicOrdering Order
,
525 unsigned ModeN
, ModelN
;
526 switch (VT
.SimpleTy
) {
543 return UNKNOWN_LIBCALL
;
547 case AtomicOrdering::Monotonic
:
550 case AtomicOrdering::Acquire
:
553 case AtomicOrdering::Release
:
556 case AtomicOrdering::AcquireRelease
:
557 case AtomicOrdering::SequentiallyConsistent
:
561 return UNKNOWN_LIBCALL
;
564 #define LCALLS(A, B) \
565 { A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
567 LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
569 case ISD::ATOMIC_CMP_SWAP
: {
570 const Libcall LC
[5][4] = {LCALL5(OUTLINE_ATOMIC_CAS
)};
571 return LC
[ModeN
][ModelN
];
573 case ISD::ATOMIC_SWAP
: {
574 const Libcall LC
[5][4] = {LCALL5(OUTLINE_ATOMIC_SWP
)};
575 return LC
[ModeN
][ModelN
];
577 case ISD::ATOMIC_LOAD_ADD
: {
578 const Libcall LC
[5][4] = {LCALL5(OUTLINE_ATOMIC_LDADD
)};
579 return LC
[ModeN
][ModelN
];
581 case ISD::ATOMIC_LOAD_OR
: {
582 const Libcall LC
[5][4] = {LCALL5(OUTLINE_ATOMIC_LDSET
)};
583 return LC
[ModeN
][ModelN
];
585 case ISD::ATOMIC_LOAD_CLR
: {
586 const Libcall LC
[5][4] = {LCALL5(OUTLINE_ATOMIC_LDCLR
)};
587 return LC
[ModeN
][ModelN
];
589 case ISD::ATOMIC_LOAD_XOR
: {
590 const Libcall LC
[5][4] = {LCALL5(OUTLINE_ATOMIC_LDEOR
)};
591 return LC
[ModeN
][ModelN
];
594 return UNKNOWN_LIBCALL
;
600 RTLIB::Libcall
RTLIB::getSYNC(unsigned Opc
, MVT VT
) {
601 #define OP_TO_LIBCALL(Name, Enum) \
603 switch (VT.SimpleTy) { \
605 return UNKNOWN_LIBCALL; \
619 OP_TO_LIBCALL(ISD::ATOMIC_SWAP
, SYNC_LOCK_TEST_AND_SET
)
620 OP_TO_LIBCALL(ISD::ATOMIC_CMP_SWAP
, SYNC_VAL_COMPARE_AND_SWAP
)
621 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_ADD
, SYNC_FETCH_AND_ADD
)
622 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_SUB
, SYNC_FETCH_AND_SUB
)
623 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_AND
, SYNC_FETCH_AND_AND
)
624 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_OR
, SYNC_FETCH_AND_OR
)
625 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_XOR
, SYNC_FETCH_AND_XOR
)
626 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_NAND
, SYNC_FETCH_AND_NAND
)
627 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MAX
, SYNC_FETCH_AND_MAX
)
628 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMAX
, SYNC_FETCH_AND_UMAX
)
629 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MIN
, SYNC_FETCH_AND_MIN
)
630 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMIN
, SYNC_FETCH_AND_UMIN
)
635 return UNKNOWN_LIBCALL
;
638 RTLIB::Libcall
RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize
) {
639 switch (ElementSize
) {
641 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_1
;
643 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_2
;
645 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_4
;
647 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_8
;
649 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_16
;
651 return UNKNOWN_LIBCALL
;
655 RTLIB::Libcall
RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize
) {
656 switch (ElementSize
) {
658 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1
;
660 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2
;
662 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4
;
664 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8
;
666 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16
;
668 return UNKNOWN_LIBCALL
;
672 RTLIB::Libcall
RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize
) {
673 switch (ElementSize
) {
675 return MEMSET_ELEMENT_UNORDERED_ATOMIC_1
;
677 return MEMSET_ELEMENT_UNORDERED_ATOMIC_2
;
679 return MEMSET_ELEMENT_UNORDERED_ATOMIC_4
;
681 return MEMSET_ELEMENT_UNORDERED_ATOMIC_8
;
683 return MEMSET_ELEMENT_UNORDERED_ATOMIC_16
;
685 return UNKNOWN_LIBCALL
;
689 /// InitCmpLibcallCCs - Set default comparison libcall CC.
690 static void InitCmpLibcallCCs(ISD::CondCode
*CCs
) {
691 std::fill(CCs
, CCs
+ RTLIB::UNKNOWN_LIBCALL
, ISD::SETCC_INVALID
);
692 CCs
[RTLIB::OEQ_F32
] = ISD::SETEQ
;
693 CCs
[RTLIB::OEQ_F64
] = ISD::SETEQ
;
694 CCs
[RTLIB::OEQ_F128
] = ISD::SETEQ
;
695 CCs
[RTLIB::OEQ_PPCF128
] = ISD::SETEQ
;
696 CCs
[RTLIB::UNE_F32
] = ISD::SETNE
;
697 CCs
[RTLIB::UNE_F64
] = ISD::SETNE
;
698 CCs
[RTLIB::UNE_F128
] = ISD::SETNE
;
699 CCs
[RTLIB::UNE_PPCF128
] = ISD::SETNE
;
700 CCs
[RTLIB::OGE_F32
] = ISD::SETGE
;
701 CCs
[RTLIB::OGE_F64
] = ISD::SETGE
;
702 CCs
[RTLIB::OGE_F128
] = ISD::SETGE
;
703 CCs
[RTLIB::OGE_PPCF128
] = ISD::SETGE
;
704 CCs
[RTLIB::OLT_F32
] = ISD::SETLT
;
705 CCs
[RTLIB::OLT_F64
] = ISD::SETLT
;
706 CCs
[RTLIB::OLT_F128
] = ISD::SETLT
;
707 CCs
[RTLIB::OLT_PPCF128
] = ISD::SETLT
;
708 CCs
[RTLIB::OLE_F32
] = ISD::SETLE
;
709 CCs
[RTLIB::OLE_F64
] = ISD::SETLE
;
710 CCs
[RTLIB::OLE_F128
] = ISD::SETLE
;
711 CCs
[RTLIB::OLE_PPCF128
] = ISD::SETLE
;
712 CCs
[RTLIB::OGT_F32
] = ISD::SETGT
;
713 CCs
[RTLIB::OGT_F64
] = ISD::SETGT
;
714 CCs
[RTLIB::OGT_F128
] = ISD::SETGT
;
715 CCs
[RTLIB::OGT_PPCF128
] = ISD::SETGT
;
716 CCs
[RTLIB::UO_F32
] = ISD::SETNE
;
717 CCs
[RTLIB::UO_F64
] = ISD::SETNE
;
718 CCs
[RTLIB::UO_F128
] = ISD::SETNE
;
719 CCs
[RTLIB::UO_PPCF128
] = ISD::SETNE
;
722 /// NOTE: The TargetMachine owns TLOF.
723 TargetLoweringBase::TargetLoweringBase(const TargetMachine
&tm
) : TM(tm
) {
726 // Perform these initializations only once.
727 MaxStoresPerMemset
= MaxStoresPerMemcpy
= MaxStoresPerMemmove
=
728 MaxLoadsPerMemcmp
= 8;
729 MaxGluedStoresPerMemcpy
= 0;
730 MaxStoresPerMemsetOptSize
= MaxStoresPerMemcpyOptSize
=
731 MaxStoresPerMemmoveOptSize
= MaxLoadsPerMemcmpOptSize
= 4;
732 HasMultipleConditionRegisters
= false;
733 HasExtractBitsInsn
= false;
734 JumpIsExpensive
= JumpIsExpensiveOverride
;
735 PredictableSelectIsExpensive
= false;
736 EnableExtLdPromotion
= false;
737 StackPointerRegisterToSaveRestore
= 0;
738 BooleanContents
= UndefinedBooleanContent
;
739 BooleanFloatContents
= UndefinedBooleanContent
;
740 BooleanVectorContents
= UndefinedBooleanContent
;
741 SchedPreferenceInfo
= Sched::ILP
;
742 GatherAllAliasesMaxDepth
= 18;
743 IsStrictFPEnabled
= DisableStrictNodeMutation
;
744 MaxBytesForAlignment
= 0;
745 // TODO: the default will be switched to 0 in the next commit, along
746 // with the Target-specific changes necessary.
747 MaxAtomicSizeInBitsSupported
= 1024;
749 // Assume that even with libcalls, no target supports wider than 128 bit
751 MaxDivRemBitWidthSupported
= 128;
753 MaxLargeFPConvertBitWidthSupported
= llvm::IntegerType::MAX_INT_BITS
;
755 MinCmpXchgSizeInBits
= 0;
756 SupportsUnalignedAtomics
= false;
758 std::fill(std::begin(LibcallRoutineNames
), std::end(LibcallRoutineNames
), nullptr);
760 InitLibcalls(TM
.getTargetTriple());
761 InitCmpLibcallCCs(CmpLibcallCCs
);
764 void TargetLoweringBase::initActions() {
765 // All operations default to being supported.
766 memset(OpActions
, 0, sizeof(OpActions
));
767 memset(LoadExtActions
, 0, sizeof(LoadExtActions
));
768 memset(TruncStoreActions
, 0, sizeof(TruncStoreActions
));
769 memset(IndexedModeActions
, 0, sizeof(IndexedModeActions
));
770 memset(CondCodeActions
, 0, sizeof(CondCodeActions
));
771 std::fill(std::begin(RegClassForVT
), std::end(RegClassForVT
), nullptr);
772 std::fill(std::begin(TargetDAGCombineArray
),
773 std::end(TargetDAGCombineArray
), 0);
775 // We're somewhat special casing MVT::i2 and MVT::i4. Ideally we want to
776 // remove this and targets should individually set these types if not legal.
777 for (ISD::NodeType NT
: enum_seq(ISD::DELETED_NODE
, ISD::BUILTIN_OP_END
,
778 force_iteration_on_noniterable_enum
)) {
779 for (MVT VT
: {MVT::i2
, MVT::i4
})
780 OpActions
[(unsigned)VT
.SimpleTy
][NT
] = Expand
;
782 for (MVT AVT
: MVT::all_valuetypes()) {
783 for (MVT VT
: {MVT::i2
, MVT::i4
, MVT::v128i2
, MVT::v64i4
}) {
784 setTruncStoreAction(AVT
, VT
, Expand
);
785 setLoadExtAction(ISD::EXTLOAD
, AVT
, VT
, Expand
);
786 setLoadExtAction(ISD::ZEXTLOAD
, AVT
, VT
, Expand
);
789 for (unsigned IM
= (unsigned)ISD::PRE_INC
;
790 IM
!= (unsigned)ISD::LAST_INDEXED_MODE
; ++IM
) {
791 for (MVT VT
: {MVT::i2
, MVT::i4
}) {
792 setIndexedLoadAction(IM
, VT
, Expand
);
793 setIndexedStoreAction(IM
, VT
, Expand
);
794 setIndexedMaskedLoadAction(IM
, VT
, Expand
);
795 setIndexedMaskedStoreAction(IM
, VT
, Expand
);
799 for (MVT VT
: MVT::fp_valuetypes()) {
800 MVT IntVT
= MVT::getIntegerVT(VT
.getFixedSizeInBits());
801 if (IntVT
.isValid()) {
802 setOperationAction(ISD::ATOMIC_SWAP
, VT
, Promote
);
803 AddPromotedToType(ISD::ATOMIC_SWAP
, VT
, IntVT
);
807 // Set default actions for various operations.
808 for (MVT VT
: MVT::all_valuetypes()) {
809 // Default all indexed load / store to expand.
810 for (unsigned IM
= (unsigned)ISD::PRE_INC
;
811 IM
!= (unsigned)ISD::LAST_INDEXED_MODE
; ++IM
) {
812 setIndexedLoadAction(IM
, VT
, Expand
);
813 setIndexedStoreAction(IM
, VT
, Expand
);
814 setIndexedMaskedLoadAction(IM
, VT
, Expand
);
815 setIndexedMaskedStoreAction(IM
, VT
, Expand
);
818 // Most backends expect to see the node which just returns the value loaded.
819 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS
, VT
, Expand
);
821 // These operations default to expand.
822 setOperationAction({ISD::FGETSIGN
, ISD::CONCAT_VECTORS
,
823 ISD::FMINNUM
, ISD::FMAXNUM
,
824 ISD::FMINNUM_IEEE
, ISD::FMAXNUM_IEEE
,
825 ISD::FMINIMUM
, ISD::FMAXIMUM
,
826 ISD::FMAD
, ISD::SMIN
,
827 ISD::SMAX
, ISD::UMIN
,
829 ISD::FSHL
, ISD::FSHR
,
830 ISD::SADDSAT
, ISD::UADDSAT
,
831 ISD::SSUBSAT
, ISD::USUBSAT
,
832 ISD::SSHLSAT
, ISD::USHLSAT
,
833 ISD::SMULFIX
, ISD::SMULFIXSAT
,
834 ISD::UMULFIX
, ISD::UMULFIXSAT
,
835 ISD::SDIVFIX
, ISD::SDIVFIXSAT
,
836 ISD::UDIVFIX
, ISD::UDIVFIXSAT
,
837 ISD::FP_TO_SINT_SAT
, ISD::FP_TO_UINT_SAT
,
841 // Overflow operations default to expand
842 setOperationAction({ISD::SADDO
, ISD::SSUBO
, ISD::UADDO
, ISD::USUBO
,
843 ISD::SMULO
, ISD::UMULO
},
846 // Carry-using overflow operations default to expand.
847 setOperationAction({ISD::UADDO_CARRY
, ISD::USUBO_CARRY
, ISD::SETCCCARRY
,
848 ISD::SADDO_CARRY
, ISD::SSUBO_CARRY
},
851 // ADDC/ADDE/SUBC/SUBE default to expand.
852 setOperationAction({ISD::ADDC
, ISD::ADDE
, ISD::SUBC
, ISD::SUBE
}, VT
,
857 {ISD::AVGFLOORS
, ISD::AVGFLOORU
, ISD::AVGCEILS
, ISD::AVGCEILU
}, VT
,
860 // Absolute difference
861 setOperationAction({ISD::ABDS
, ISD::ABDU
}, VT
, Expand
);
863 // These default to Expand so they will be expanded to CTLZ/CTTZ by default.
864 setOperationAction({ISD::CTLZ_ZERO_UNDEF
, ISD::CTTZ_ZERO_UNDEF
}, VT
,
867 setOperationAction({ISD::BITREVERSE
, ISD::PARITY
}, VT
, Expand
);
869 // These library functions default to expand.
871 {ISD::FROUND
, ISD::FROUNDEVEN
, ISD::FPOWI
, ISD::FLDEXP
, ISD::FFREXP
},
874 // These operations default to expand for vector types.
877 {ISD::FCOPYSIGN
, ISD::SIGN_EXTEND_INREG
, ISD::ANY_EXTEND_VECTOR_INREG
,
878 ISD::SIGN_EXTEND_VECTOR_INREG
, ISD::ZERO_EXTEND_VECTOR_INREG
,
879 ISD::SPLAT_VECTOR
, ISD::LRINT
, ISD::LLRINT
},
882 // Constrained floating-point operations default to expand.
883 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
884 setOperationAction(ISD::STRICT_##DAGN, VT, Expand);
885 #include "llvm/IR/ConstrainedOps.def"
887 // For most targets @llvm.get.dynamic.area.offset just returns 0.
888 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET
, VT
, Expand
);
890 // Vector reduction default to expand.
892 {ISD::VECREDUCE_FADD
, ISD::VECREDUCE_FMUL
, ISD::VECREDUCE_ADD
,
893 ISD::VECREDUCE_MUL
, ISD::VECREDUCE_AND
, ISD::VECREDUCE_OR
,
894 ISD::VECREDUCE_XOR
, ISD::VECREDUCE_SMAX
, ISD::VECREDUCE_SMIN
,
895 ISD::VECREDUCE_UMAX
, ISD::VECREDUCE_UMIN
, ISD::VECREDUCE_FMAX
,
896 ISD::VECREDUCE_FMIN
, ISD::VECREDUCE_FMAXIMUM
, ISD::VECREDUCE_FMINIMUM
,
897 ISD::VECREDUCE_SEQ_FADD
, ISD::VECREDUCE_SEQ_FMUL
},
900 // Named vector shuffles default to expand.
901 setOperationAction(ISD::VECTOR_SPLICE
, VT
, Expand
);
903 // VP operations default to expand.
904 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \
905 setOperationAction(ISD::SDOPC, VT, Expand);
906 #include "llvm/IR/VPIntrinsics.def"
908 // FP environment operations default to expand.
909 setOperationAction(ISD::GET_FPENV
, VT
, Expand
);
910 setOperationAction(ISD::SET_FPENV
, VT
, Expand
);
911 setOperationAction(ISD::RESET_FPENV
, VT
, Expand
);
914 // Most targets ignore the @llvm.prefetch intrinsic.
915 setOperationAction(ISD::PREFETCH
, MVT::Other
, Expand
);
917 // Most targets also ignore the @llvm.readcyclecounter intrinsic.
918 setOperationAction(ISD::READCYCLECOUNTER
, MVT::i64
, Expand
);
920 // ConstantFP nodes default to expand. Targets can either change this to
921 // Legal, in which case all fp constants are legal, or use isFPImmLegal()
922 // to optimize expansions for certain constants.
923 setOperationAction(ISD::ConstantFP
,
924 {MVT::bf16
, MVT::f16
, MVT::f32
, MVT::f64
, MVT::f80
, MVT::f128
},
927 // These library functions default to expand.
928 setOperationAction({ISD::FCBRT
, ISD::FLOG
, ISD::FLOG2
, ISD::FLOG10
, ISD::FEXP
,
929 ISD::FEXP2
, ISD::FEXP10
, ISD::FFLOOR
, ISD::FNEARBYINT
,
930 ISD::FCEIL
, ISD::FRINT
, ISD::FTRUNC
, ISD::LROUND
,
931 ISD::LLROUND
, ISD::LRINT
, ISD::LLRINT
},
932 {MVT::f32
, MVT::f64
, MVT::f128
}, Expand
);
934 // Default ISD::TRAP to expand (which turns it into abort).
935 setOperationAction(ISD::TRAP
, MVT::Other
, Expand
);
937 // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand"
938 // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP.
939 setOperationAction(ISD::DEBUGTRAP
, MVT::Other
, Expand
);
941 setOperationAction(ISD::UBSANTRAP
, MVT::Other
, Expand
);
943 setOperationAction(ISD::GET_FPENV_MEM
, MVT::Other
, Expand
);
944 setOperationAction(ISD::SET_FPENV_MEM
, MVT::Other
, Expand
);
946 for (MVT VT
: {MVT::i8
, MVT::i16
, MVT::i32
, MVT::i64
}) {
947 setOperationAction(ISD::GET_FPMODE
, VT
, Expand
);
948 setOperationAction(ISD::SET_FPMODE
, VT
, Expand
);
950 setOperationAction(ISD::RESET_FPMODE
, MVT::Other
, Expand
);
953 MVT
TargetLoweringBase::getScalarShiftAmountTy(const DataLayout
&DL
,
955 return MVT::getIntegerVT(DL
.getPointerSizeInBits(0));
958 EVT
TargetLoweringBase::getShiftAmountTy(EVT LHSTy
, const DataLayout
&DL
,
959 bool LegalTypes
) const {
960 assert(LHSTy
.isInteger() && "Shift amount is not an integer type!");
961 if (LHSTy
.isVector())
964 LegalTypes
? getScalarShiftAmountTy(DL
, LHSTy
) : getPointerTy(DL
);
965 // If any possible shift value won't fit in the prefered type, just use
966 // something safe. Assume it will be legalized when the shift is expanded.
967 if (ShiftVT
.getSizeInBits() < Log2_32_Ceil(LHSTy
.getSizeInBits()))
969 assert(ShiftVT
.getSizeInBits() >= Log2_32_Ceil(LHSTy
.getSizeInBits()) &&
970 "ShiftVT is still too small!");
974 bool TargetLoweringBase::canOpTrap(unsigned Op
, EVT VT
) const {
975 assert(isTypeLegal(VT
));
987 bool TargetLoweringBase::isFreeAddrSpaceCast(unsigned SrcAS
,
988 unsigned DestAS
) const {
989 return TM
.isNoopAddrSpaceCast(SrcAS
, DestAS
);
992 void TargetLoweringBase::setJumpIsExpensive(bool isExpensive
) {
993 // If the command-line option was specified, ignore this request.
994 if (!JumpIsExpensiveOverride
.getNumOccurrences())
995 JumpIsExpensive
= isExpensive
;
998 TargetLoweringBase::LegalizeKind
999 TargetLoweringBase::getTypeConversion(LLVMContext
&Context
, EVT VT
) const {
1000 // If this is a simple type, use the ComputeRegisterProp mechanism.
1001 if (VT
.isSimple()) {
1002 MVT SVT
= VT
.getSimpleVT();
1003 assert((unsigned)SVT
.SimpleTy
< std::size(TransformToType
));
1004 MVT NVT
= TransformToType
[SVT
.SimpleTy
];
1005 LegalizeTypeAction LA
= ValueTypeActions
.getTypeAction(SVT
);
1007 assert((LA
== TypeLegal
|| LA
== TypeSoftenFloat
||
1008 LA
== TypeSoftPromoteHalf
||
1010 ValueTypeActions
.getTypeAction(NVT
) != TypePromoteInteger
)) &&
1011 "Promote may not follow Expand or Promote");
1013 if (LA
== TypeSplitVector
)
1014 return LegalizeKind(LA
, EVT(SVT
).getHalfNumVectorElementsVT(Context
));
1015 if (LA
== TypeScalarizeVector
)
1016 return LegalizeKind(LA
, SVT
.getVectorElementType());
1017 return LegalizeKind(LA
, NVT
);
1020 // Handle Extended Scalar Types.
1021 if (!VT
.isVector()) {
1022 assert(VT
.isInteger() && "Float types must be simple");
1023 unsigned BitSize
= VT
.getSizeInBits();
1024 // First promote to a power-of-two size, then expand if necessary.
1025 if (BitSize
< 8 || !isPowerOf2_32(BitSize
)) {
1026 EVT NVT
= VT
.getRoundIntegerType(Context
);
1027 assert(NVT
!= VT
&& "Unable to round integer VT");
1028 LegalizeKind NextStep
= getTypeConversion(Context
, NVT
);
1029 // Avoid multi-step promotion.
1030 if (NextStep
.first
== TypePromoteInteger
)
1032 // Return rounded integer type.
1033 return LegalizeKind(TypePromoteInteger
, NVT
);
1036 return LegalizeKind(TypeExpandInteger
,
1037 EVT::getIntegerVT(Context
, VT
.getSizeInBits() / 2));
1040 // Handle vector types.
1041 ElementCount NumElts
= VT
.getVectorElementCount();
1042 EVT EltVT
= VT
.getVectorElementType();
1044 // Vectors with only one element are always scalarized.
1045 if (NumElts
.isScalar())
1046 return LegalizeKind(TypeScalarizeVector
, EltVT
);
1048 // Try to widen vector elements until the element type is a power of two and
1049 // promote it to a legal type later on, for example:
1050 // <3 x i8> -> <4 x i8> -> <4 x i32>
1051 if (EltVT
.isInteger()) {
1052 // Vectors with a number of elements that is not a power of two are always
1053 // widened, for example <3 x i8> -> <4 x i8>.
1054 if (!VT
.isPow2VectorType()) {
1055 NumElts
= NumElts
.coefficientNextPowerOf2();
1056 EVT NVT
= EVT::getVectorVT(Context
, EltVT
, NumElts
);
1057 return LegalizeKind(TypeWidenVector
, NVT
);
1060 // Examine the element type.
1061 LegalizeKind LK
= getTypeConversion(Context
, EltVT
);
1063 // If type is to be expanded, split the vector.
1064 // <4 x i140> -> <2 x i140>
1065 if (LK
.first
== TypeExpandInteger
) {
1066 if (VT
.getVectorElementCount().isScalable())
1067 return LegalizeKind(TypeScalarizeScalableVector
, EltVT
);
1068 return LegalizeKind(TypeSplitVector
,
1069 VT
.getHalfNumVectorElementsVT(Context
));
1072 // Promote the integer element types until a legal vector type is found
1073 // or until the element integer type is too big. If a legal type was not
1074 // found, fallback to the usual mechanism of widening/splitting the
1076 EVT OldEltVT
= EltVT
;
1078 // Increase the bitwidth of the element to the next pow-of-two
1079 // (which is greater than 8 bits).
1080 EltVT
= EVT::getIntegerVT(Context
, 1 + EltVT
.getSizeInBits())
1081 .getRoundIntegerType(Context
);
1083 // Stop trying when getting a non-simple element type.
1084 // Note that vector elements may be greater than legal vector element
1085 // types. Example: X86 XMM registers hold 64bit element on 32bit
1087 if (!EltVT
.isSimple())
1090 // Build a new vector type and check if it is legal.
1091 MVT NVT
= MVT::getVectorVT(EltVT
.getSimpleVT(), NumElts
);
1092 // Found a legal promoted vector type.
1093 if (NVT
!= MVT() && ValueTypeActions
.getTypeAction(NVT
) == TypeLegal
)
1094 return LegalizeKind(TypePromoteInteger
,
1095 EVT::getVectorVT(Context
, EltVT
, NumElts
));
1098 // Reset the type to the unexpanded type if we did not find a legal vector
1099 // type with a promoted vector element type.
1103 // Try to widen the vector until a legal type is found.
1104 // If there is no wider legal type, split the vector.
1106 // Round up to the next power of 2.
1107 NumElts
= NumElts
.coefficientNextPowerOf2();
1109 // If there is no simple vector type with this many elements then there
1110 // cannot be a larger legal vector type. Note that this assumes that
1111 // there are no skipped intermediate vector types in the simple types.
1112 if (!EltVT
.isSimple())
1114 MVT LargerVector
= MVT::getVectorVT(EltVT
.getSimpleVT(), NumElts
);
1115 if (LargerVector
== MVT())
1118 // If this type is legal then widen the vector.
1119 if (ValueTypeActions
.getTypeAction(LargerVector
) == TypeLegal
)
1120 return LegalizeKind(TypeWidenVector
, LargerVector
);
1123 // Widen odd vectors to next power of two.
1124 if (!VT
.isPow2VectorType()) {
1125 EVT NVT
= VT
.getPow2VectorType(Context
);
1126 return LegalizeKind(TypeWidenVector
, NVT
);
1129 if (VT
.getVectorElementCount() == ElementCount::getScalable(1))
1130 return LegalizeKind(TypeScalarizeScalableVector
, EltVT
);
1132 // Vectors with illegal element types are expanded.
1133 EVT NVT
= EVT::getVectorVT(Context
, EltVT
,
1134 VT
.getVectorElementCount().divideCoefficientBy(2));
1135 return LegalizeKind(TypeSplitVector
, NVT
);
1138 static unsigned getVectorTypeBreakdownMVT(MVT VT
, MVT
&IntermediateVT
,
1139 unsigned &NumIntermediates
,
1141 TargetLoweringBase
*TLI
) {
1142 // Figure out the right, legal destination reg to copy into.
1143 ElementCount EC
= VT
.getVectorElementCount();
1144 MVT EltTy
= VT
.getVectorElementType();
1146 unsigned NumVectorRegs
= 1;
1148 // Scalable vectors cannot be scalarized, so splitting or widening is
1150 if (VT
.isScalableVector() && !isPowerOf2_32(EC
.getKnownMinValue()))
1152 "Splitting or widening of non-power-of-2 MVTs is not implemented.");
1154 // FIXME: We don't support non-power-of-2-sized vectors for now.
1155 // Ideally we could break down into LHS/RHS like LegalizeDAG does.
1156 if (!isPowerOf2_32(EC
.getKnownMinValue())) {
1157 // Split EC to unit size (scalable property is preserved).
1158 NumVectorRegs
= EC
.getKnownMinValue();
1159 EC
= ElementCount::getFixed(1);
1162 // Divide the input until we get to a supported size. This will
1163 // always end up with an EC that represent a scalar or a scalable
1165 while (EC
.getKnownMinValue() > 1 &&
1166 !TLI
->isTypeLegal(MVT::getVectorVT(EltTy
, EC
))) {
1167 EC
= EC
.divideCoefficientBy(2);
1168 NumVectorRegs
<<= 1;
1171 NumIntermediates
= NumVectorRegs
;
1173 MVT NewVT
= MVT::getVectorVT(EltTy
, EC
);
1174 if (!TLI
->isTypeLegal(NewVT
))
1176 IntermediateVT
= NewVT
;
1178 unsigned LaneSizeInBits
= NewVT
.getScalarSizeInBits();
1180 // Convert sizes such as i33 to i64.
1181 LaneSizeInBits
= llvm::bit_ceil(LaneSizeInBits
);
1183 MVT DestVT
= TLI
->getRegisterType(NewVT
);
1184 RegisterVT
= DestVT
;
1185 if (EVT(DestVT
).bitsLT(NewVT
)) // Value is expanded, e.g. i64 -> i16.
1186 return NumVectorRegs
* (LaneSizeInBits
/ DestVT
.getScalarSizeInBits());
1188 // Otherwise, promotion or legal types use the same number of registers as
1189 // the vector decimated to the appropriate level.
1190 return NumVectorRegs
;
1193 /// isLegalRC - Return true if the value types that can be represented by the
1194 /// specified register class are all legal.
1195 bool TargetLoweringBase::isLegalRC(const TargetRegisterInfo
&TRI
,
1196 const TargetRegisterClass
&RC
) const {
1197 for (const auto *I
= TRI
.legalclasstypes_begin(RC
); *I
!= MVT::Other
; ++I
)
1198 if (isTypeLegal(*I
))
1203 /// Replace/modify any TargetFrameIndex operands with a targte-dependent
1204 /// sequence of memory operands that is recognized by PrologEpilogInserter.
1206 TargetLoweringBase::emitPatchPoint(MachineInstr
&InitialMI
,
1207 MachineBasicBlock
*MBB
) const {
1208 MachineInstr
*MI
= &InitialMI
;
1209 MachineFunction
&MF
= *MI
->getMF();
1210 MachineFrameInfo
&MFI
= MF
.getFrameInfo();
1212 // We're handling multiple types of operands here:
1213 // PATCHPOINT MetaArgs - live-in, read only, direct
1214 // STATEPOINT Deopt Spill - live-through, read only, indirect
1215 // STATEPOINT Deopt Alloca - live-through, read only, direct
1216 // (We're currently conservative and mark the deopt slots read/write in
1218 // STATEPOINT GC Spill - live-through, read/write, indirect
1219 // STATEPOINT GC Alloca - live-through, read/write, direct
1220 // The live-in vs live-through is handled already (the live through ones are
1221 // all stack slots), but we need to handle the different type of stackmap
1222 // operands and memory effects here.
1224 if (llvm::none_of(MI
->operands(),
1225 [](MachineOperand
&Operand
) { return Operand
.isFI(); }))
1228 MachineInstrBuilder MIB
= BuildMI(MF
, MI
->getDebugLoc(), MI
->getDesc());
1230 // Inherit previous memory operands.
1231 MIB
.cloneMemRefs(*MI
);
1233 for (unsigned i
= 0; i
< MI
->getNumOperands(); ++i
) {
1234 MachineOperand
&MO
= MI
->getOperand(i
);
1236 // Index of Def operand this Use it tied to.
1237 // Since Defs are coming before Uses, if Use is tied, then
1238 // index of Def must be smaller that index of that Use.
1239 // Also, Defs preserve their position in new MI.
1240 unsigned TiedTo
= i
;
1241 if (MO
.isReg() && MO
.isTied())
1242 TiedTo
= MI
->findTiedOperandIdx(i
);
1245 MIB
->tieOperands(TiedTo
, MIB
->getNumOperands() - 1);
1249 // foldMemoryOperand builds a new MI after replacing a single FI operand
1250 // with the canonical set of five x86 addressing-mode operands.
1251 int FI
= MO
.getIndex();
1253 // Add frame index operands recognized by stackmaps.cpp
1254 if (MFI
.isStatepointSpillSlotObjectIndex(FI
)) {
1255 // indirect-mem-ref tag, size, #FI, offset.
1256 // Used for spills inserted by StatepointLowering. This codepath is not
1257 // used for patchpoints/stackmaps at all, for these spilling is done via
1258 // foldMemoryOperand callback only.
1259 assert(MI
->getOpcode() == TargetOpcode::STATEPOINT
&& "sanity");
1260 MIB
.addImm(StackMaps::IndirectMemRefOp
);
1261 MIB
.addImm(MFI
.getObjectSize(FI
));
1265 // direct-mem-ref tag, #FI, offset.
1266 // Used by patchpoint, and direct alloca arguments to statepoints
1267 MIB
.addImm(StackMaps::DirectMemRefOp
);
1272 assert(MIB
->mayLoad() && "Folded a stackmap use to a non-load!");
1274 // Add a new memory operand for this FI.
1275 assert(MFI
.getObjectOffset(FI
) != -1);
1277 // Note: STATEPOINT MMOs are added during SelectionDAG. STACKMAP, and
1278 // PATCHPOINT should be updated to do the same. (TODO)
1279 if (MI
->getOpcode() != TargetOpcode::STATEPOINT
) {
1280 auto Flags
= MachineMemOperand::MOLoad
;
1281 MachineMemOperand
*MMO
= MF
.getMachineMemOperand(
1282 MachinePointerInfo::getFixedStack(MF
, FI
), Flags
,
1283 MF
.getDataLayout().getPointerSize(), MFI
.getObjectAlign(FI
));
1284 MIB
->addMemOperand(MF
, MMO
);
1287 MBB
->insert(MachineBasicBlock::iterator(MI
), MIB
);
1288 MI
->eraseFromParent();
1292 /// findRepresentativeClass - Return the largest legal super-reg register class
1293 /// of the register class for the specified type and its associated "cost".
1294 // This function is in TargetLowering because it uses RegClassForVT which would
1295 // need to be moved to TargetRegisterInfo and would necessitate moving
1296 // isTypeLegal over as well - a massive change that would just require
1297 // TargetLowering having a TargetRegisterInfo class member that it would use.
1298 std::pair
<const TargetRegisterClass
*, uint8_t>
1299 TargetLoweringBase::findRepresentativeClass(const TargetRegisterInfo
*TRI
,
1301 const TargetRegisterClass
*RC
= RegClassForVT
[VT
.SimpleTy
];
1303 return std::make_pair(RC
, 0);
1305 // Compute the set of all super-register classes.
1306 BitVector
SuperRegRC(TRI
->getNumRegClasses());
1307 for (SuperRegClassIterator
RCI(RC
, TRI
); RCI
.isValid(); ++RCI
)
1308 SuperRegRC
.setBitsInMask(RCI
.getMask());
1310 // Find the first legal register class with the largest spill size.
1311 const TargetRegisterClass
*BestRC
= RC
;
1312 for (unsigned i
: SuperRegRC
.set_bits()) {
1313 const TargetRegisterClass
*SuperRC
= TRI
->getRegClass(i
);
1314 // We want the largest possible spill size.
1315 if (TRI
->getSpillSize(*SuperRC
) <= TRI
->getSpillSize(*BestRC
))
1317 if (!isLegalRC(*TRI
, *SuperRC
))
1321 return std::make_pair(BestRC
, 1);
1324 /// computeRegisterProperties - Once all of the register classes are added,
1325 /// this allows us to compute derived properties we expose.
1326 void TargetLoweringBase::computeRegisterProperties(
1327 const TargetRegisterInfo
*TRI
) {
1328 static_assert(MVT::VALUETYPE_SIZE
<= MVT::MAX_ALLOWED_VALUETYPE
,
1329 "Too many value types for ValueTypeActions to hold!");
1331 // Everything defaults to needing one register.
1332 for (unsigned i
= 0; i
!= MVT::VALUETYPE_SIZE
; ++i
) {
1333 NumRegistersForVT
[i
] = 1;
1334 RegisterTypeForVT
[i
] = TransformToType
[i
] = (MVT::SimpleValueType
)i
;
1336 // ...except isVoid, which doesn't need any registers.
1337 NumRegistersForVT
[MVT::isVoid
] = 0;
1339 // Find the largest integer register class.
1340 unsigned LargestIntReg
= MVT::LAST_INTEGER_VALUETYPE
;
1341 for (; RegClassForVT
[LargestIntReg
] == nullptr; --LargestIntReg
)
1342 assert(LargestIntReg
!= MVT::i1
&& "No integer registers defined!");
1344 // Every integer value type larger than this largest register takes twice as
1345 // many registers to represent as the previous ValueType.
1346 for (unsigned ExpandedReg
= LargestIntReg
+ 1;
1347 ExpandedReg
<= MVT::LAST_INTEGER_VALUETYPE
; ++ExpandedReg
) {
1348 NumRegistersForVT
[ExpandedReg
] = 2*NumRegistersForVT
[ExpandedReg
-1];
1349 RegisterTypeForVT
[ExpandedReg
] = (MVT::SimpleValueType
)LargestIntReg
;
1350 TransformToType
[ExpandedReg
] = (MVT::SimpleValueType
)(ExpandedReg
- 1);
1351 ValueTypeActions
.setTypeAction((MVT::SimpleValueType
)ExpandedReg
,
1355 // Inspect all of the ValueType's smaller than the largest integer
1356 // register to see which ones need promotion.
1357 unsigned LegalIntReg
= LargestIntReg
;
1358 for (unsigned IntReg
= LargestIntReg
- 1;
1359 IntReg
>= (unsigned)MVT::i1
; --IntReg
) {
1360 MVT IVT
= (MVT::SimpleValueType
)IntReg
;
1361 if (isTypeLegal(IVT
)) {
1362 LegalIntReg
= IntReg
;
1364 RegisterTypeForVT
[IntReg
] = TransformToType
[IntReg
] =
1365 (MVT::SimpleValueType
)LegalIntReg
;
1366 ValueTypeActions
.setTypeAction(IVT
, TypePromoteInteger
);
1370 // ppcf128 type is really two f64's.
1371 if (!isTypeLegal(MVT::ppcf128
)) {
1372 if (isTypeLegal(MVT::f64
)) {
1373 NumRegistersForVT
[MVT::ppcf128
] = 2*NumRegistersForVT
[MVT::f64
];
1374 RegisterTypeForVT
[MVT::ppcf128
] = MVT::f64
;
1375 TransformToType
[MVT::ppcf128
] = MVT::f64
;
1376 ValueTypeActions
.setTypeAction(MVT::ppcf128
, TypeExpandFloat
);
1378 NumRegistersForVT
[MVT::ppcf128
] = NumRegistersForVT
[MVT::i128
];
1379 RegisterTypeForVT
[MVT::ppcf128
] = RegisterTypeForVT
[MVT::i128
];
1380 TransformToType
[MVT::ppcf128
] = MVT::i128
;
1381 ValueTypeActions
.setTypeAction(MVT::ppcf128
, TypeSoftenFloat
);
1385 // Decide how to handle f128. If the target does not have native f128 support,
1386 // expand it to i128 and we will be generating soft float library calls.
1387 if (!isTypeLegal(MVT::f128
)) {
1388 NumRegistersForVT
[MVT::f128
] = NumRegistersForVT
[MVT::i128
];
1389 RegisterTypeForVT
[MVT::f128
] = RegisterTypeForVT
[MVT::i128
];
1390 TransformToType
[MVT::f128
] = MVT::i128
;
1391 ValueTypeActions
.setTypeAction(MVT::f128
, TypeSoftenFloat
);
1394 // Decide how to handle f80. If the target does not have native f80 support,
1395 // expand it to i96 and we will be generating soft float library calls.
1396 if (!isTypeLegal(MVT::f80
)) {
1397 NumRegistersForVT
[MVT::f80
] = 3*NumRegistersForVT
[MVT::i32
];
1398 RegisterTypeForVT
[MVT::f80
] = RegisterTypeForVT
[MVT::i32
];
1399 TransformToType
[MVT::f80
] = MVT::i32
;
1400 ValueTypeActions
.setTypeAction(MVT::f80
, TypeSoftenFloat
);
1403 // Decide how to handle f64. If the target does not have native f64 support,
1404 // expand it to i64 and we will be generating soft float library calls.
1405 if (!isTypeLegal(MVT::f64
)) {
1406 NumRegistersForVT
[MVT::f64
] = NumRegistersForVT
[MVT::i64
];
1407 RegisterTypeForVT
[MVT::f64
] = RegisterTypeForVT
[MVT::i64
];
1408 TransformToType
[MVT::f64
] = MVT::i64
;
1409 ValueTypeActions
.setTypeAction(MVT::f64
, TypeSoftenFloat
);
1412 // Decide how to handle f32. If the target does not have native f32 support,
1413 // expand it to i32 and we will be generating soft float library calls.
1414 if (!isTypeLegal(MVT::f32
)) {
1415 NumRegistersForVT
[MVT::f32
] = NumRegistersForVT
[MVT::i32
];
1416 RegisterTypeForVT
[MVT::f32
] = RegisterTypeForVT
[MVT::i32
];
1417 TransformToType
[MVT::f32
] = MVT::i32
;
1418 ValueTypeActions
.setTypeAction(MVT::f32
, TypeSoftenFloat
);
1421 // Decide how to handle f16. If the target does not have native f16 support,
1422 // promote it to f32, because there are no f16 library calls (except for
1424 if (!isTypeLegal(MVT::f16
)) {
1425 // Allow targets to control how we legalize half.
1426 if (softPromoteHalfType()) {
1427 NumRegistersForVT
[MVT::f16
] = NumRegistersForVT
[MVT::i16
];
1428 RegisterTypeForVT
[MVT::f16
] = RegisterTypeForVT
[MVT::i16
];
1429 TransformToType
[MVT::f16
] = MVT::f32
;
1430 ValueTypeActions
.setTypeAction(MVT::f16
, TypeSoftPromoteHalf
);
1432 NumRegistersForVT
[MVT::f16
] = NumRegistersForVT
[MVT::f32
];
1433 RegisterTypeForVT
[MVT::f16
] = RegisterTypeForVT
[MVT::f32
];
1434 TransformToType
[MVT::f16
] = MVT::f32
;
1435 ValueTypeActions
.setTypeAction(MVT::f16
, TypePromoteFloat
);
1439 // Decide how to handle bf16. If the target does not have native bf16 support,
1440 // promote it to f32, because there are no bf16 library calls (except for
1441 // converting from f32 to bf16).
1442 if (!isTypeLegal(MVT::bf16
)) {
1443 NumRegistersForVT
[MVT::bf16
] = NumRegistersForVT
[MVT::f32
];
1444 RegisterTypeForVT
[MVT::bf16
] = RegisterTypeForVT
[MVT::f32
];
1445 TransformToType
[MVT::bf16
] = MVT::f32
;
1446 ValueTypeActions
.setTypeAction(MVT::bf16
, TypeSoftPromoteHalf
);
1449 // Loop over all of the vector value types to see which need transformations.
1450 for (unsigned i
= MVT::FIRST_VECTOR_VALUETYPE
;
1451 i
<= (unsigned)MVT::LAST_VECTOR_VALUETYPE
; ++i
) {
1452 MVT VT
= (MVT::SimpleValueType
) i
;
1453 if (isTypeLegal(VT
))
1456 MVT EltVT
= VT
.getVectorElementType();
1457 ElementCount EC
= VT
.getVectorElementCount();
1458 bool IsLegalWiderType
= false;
1459 bool IsScalable
= VT
.isScalableVector();
1460 LegalizeTypeAction PreferredAction
= getPreferredVectorAction(VT
);
1461 switch (PreferredAction
) {
1462 case TypePromoteInteger
: {
1463 MVT::SimpleValueType EndVT
= IsScalable
?
1464 MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE
:
1465 MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE
;
1466 // Try to promote the elements of integer vectors. If no legal
1467 // promotion was found, fall through to the widen-vector method.
1468 for (unsigned nVT
= i
+ 1;
1469 (MVT::SimpleValueType
)nVT
<= EndVT
; ++nVT
) {
1470 MVT SVT
= (MVT::SimpleValueType
) nVT
;
1471 // Promote vectors of integers to vectors with the same number
1472 // of elements, with a wider element type.
1473 if (SVT
.getScalarSizeInBits() > EltVT
.getFixedSizeInBits() &&
1474 SVT
.getVectorElementCount() == EC
&& isTypeLegal(SVT
)) {
1475 TransformToType
[i
] = SVT
;
1476 RegisterTypeForVT
[i
] = SVT
;
1477 NumRegistersForVT
[i
] = 1;
1478 ValueTypeActions
.setTypeAction(VT
, TypePromoteInteger
);
1479 IsLegalWiderType
= true;
1483 if (IsLegalWiderType
)
1488 case TypeWidenVector
:
1489 if (isPowerOf2_32(EC
.getKnownMinValue())) {
1490 // Try to widen the vector.
1491 for (unsigned nVT
= i
+ 1; nVT
<= MVT::LAST_VECTOR_VALUETYPE
; ++nVT
) {
1492 MVT SVT
= (MVT::SimpleValueType
) nVT
;
1493 if (SVT
.getVectorElementType() == EltVT
&&
1494 SVT
.isScalableVector() == IsScalable
&&
1495 SVT
.getVectorElementCount().getKnownMinValue() >
1496 EC
.getKnownMinValue() &&
1498 TransformToType
[i
] = SVT
;
1499 RegisterTypeForVT
[i
] = SVT
;
1500 NumRegistersForVT
[i
] = 1;
1501 ValueTypeActions
.setTypeAction(VT
, TypeWidenVector
);
1502 IsLegalWiderType
= true;
1506 if (IsLegalWiderType
)
1509 // Only widen to the next power of 2 to keep consistency with EVT.
1510 MVT NVT
= VT
.getPow2VectorType();
1511 if (isTypeLegal(NVT
)) {
1512 TransformToType
[i
] = NVT
;
1513 ValueTypeActions
.setTypeAction(VT
, TypeWidenVector
);
1514 RegisterTypeForVT
[i
] = NVT
;
1515 NumRegistersForVT
[i
] = 1;
1521 case TypeSplitVector
:
1522 case TypeScalarizeVector
: {
1525 unsigned NumIntermediates
;
1526 unsigned NumRegisters
= getVectorTypeBreakdownMVT(VT
, IntermediateVT
,
1527 NumIntermediates
, RegisterVT
, this);
1528 NumRegistersForVT
[i
] = NumRegisters
;
1529 assert(NumRegistersForVT
[i
] == NumRegisters
&&
1530 "NumRegistersForVT size cannot represent NumRegisters!");
1531 RegisterTypeForVT
[i
] = RegisterVT
;
1533 MVT NVT
= VT
.getPow2VectorType();
1535 // Type is already a power of 2. The default action is to split.
1536 TransformToType
[i
] = MVT::Other
;
1537 if (PreferredAction
== TypeScalarizeVector
)
1538 ValueTypeActions
.setTypeAction(VT
, TypeScalarizeVector
);
1539 else if (PreferredAction
== TypeSplitVector
)
1540 ValueTypeActions
.setTypeAction(VT
, TypeSplitVector
);
1541 else if (EC
.getKnownMinValue() > 1)
1542 ValueTypeActions
.setTypeAction(VT
, TypeSplitVector
);
1544 ValueTypeActions
.setTypeAction(VT
, EC
.isScalable()
1545 ? TypeScalarizeScalableVector
1546 : TypeScalarizeVector
);
1548 TransformToType
[i
] = NVT
;
1549 ValueTypeActions
.setTypeAction(VT
, TypeWidenVector
);
1554 llvm_unreachable("Unknown vector legalization action!");
1558 // Determine the 'representative' register class for each value type.
1559 // An representative register class is the largest (meaning one which is
1560 // not a sub-register class / subreg register class) legal register class for
1561 // a group of value types. For example, on i386, i8, i16, and i32
1562 // representative would be GR32; while on x86_64 it's GR64.
1563 for (unsigned i
= 0; i
!= MVT::VALUETYPE_SIZE
; ++i
) {
1564 const TargetRegisterClass
* RRC
;
1566 std::tie(RRC
, Cost
) = findRepresentativeClass(TRI
, (MVT::SimpleValueType
)i
);
1567 RepRegClassForVT
[i
] = RRC
;
1568 RepRegClassCostForVT
[i
] = Cost
;
1572 EVT
TargetLoweringBase::getSetCCResultType(const DataLayout
&DL
, LLVMContext
&,
1574 assert(!VT
.isVector() && "No default SetCC type for vectors!");
1575 return getPointerTy(DL
).SimpleTy
;
1578 MVT::SimpleValueType
TargetLoweringBase::getCmpLibcallReturnType() const {
1579 return MVT::i32
; // return the default value
1582 /// getVectorTypeBreakdown - Vector types are broken down into some number of
1583 /// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32
1584 /// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
1585 /// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86.
1587 /// This method returns the number of registers needed, and the VT for each
1588 /// register. It also returns the VT and quantity of the intermediate values
1589 /// before they are promoted/expanded.
1590 unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext
&Context
,
1591 EVT VT
, EVT
&IntermediateVT
,
1592 unsigned &NumIntermediates
,
1593 MVT
&RegisterVT
) const {
1594 ElementCount EltCnt
= VT
.getVectorElementCount();
1596 // If there is a wider vector type with the same element type as this one,
1597 // or a promoted vector type that has the same number of elements which
1598 // are wider, then we should convert to that legal vector type.
1599 // This handles things like <2 x float> -> <4 x float> and
1600 // <4 x i1> -> <4 x i32>.
1601 LegalizeTypeAction TA
= getTypeAction(Context
, VT
);
1602 if (!EltCnt
.isScalar() &&
1603 (TA
== TypeWidenVector
|| TA
== TypePromoteInteger
)) {
1604 EVT RegisterEVT
= getTypeToTransformTo(Context
, VT
);
1605 if (isTypeLegal(RegisterEVT
)) {
1606 IntermediateVT
= RegisterEVT
;
1607 RegisterVT
= RegisterEVT
.getSimpleVT();
1608 NumIntermediates
= 1;
1613 // Figure out the right, legal destination reg to copy into.
1614 EVT EltTy
= VT
.getVectorElementType();
1616 unsigned NumVectorRegs
= 1;
1618 // Scalable vectors cannot be scalarized, so handle the legalisation of the
1619 // types like done elsewhere in SelectionDAG.
1620 if (EltCnt
.isScalable()) {
1624 // Iterate until we've found a legal (part) type to hold VT.
1625 LK
= getTypeConversion(Context
, PartVT
);
1627 } while (LK
.first
!= TypeLegal
);
1629 if (!PartVT
.isVector()) {
1631 "Don't know how to legalize this scalable vector type");
1635 divideCeil(VT
.getVectorElementCount().getKnownMinValue(),
1636 PartVT
.getVectorElementCount().getKnownMinValue());
1637 IntermediateVT
= PartVT
;
1638 RegisterVT
= getRegisterType(Context
, IntermediateVT
);
1639 return NumIntermediates
;
1642 // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally
1643 // we could break down into LHS/RHS like LegalizeDAG does.
1644 if (!isPowerOf2_32(EltCnt
.getKnownMinValue())) {
1645 NumVectorRegs
= EltCnt
.getKnownMinValue();
1646 EltCnt
= ElementCount::getFixed(1);
1649 // Divide the input until we get to a supported size. This will always
1650 // end with a scalar if the target doesn't support vectors.
1651 while (EltCnt
.getKnownMinValue() > 1 &&
1652 !isTypeLegal(EVT::getVectorVT(Context
, EltTy
, EltCnt
))) {
1653 EltCnt
= EltCnt
.divideCoefficientBy(2);
1654 NumVectorRegs
<<= 1;
1657 NumIntermediates
= NumVectorRegs
;
1659 EVT NewVT
= EVT::getVectorVT(Context
, EltTy
, EltCnt
);
1660 if (!isTypeLegal(NewVT
))
1662 IntermediateVT
= NewVT
;
1664 MVT DestVT
= getRegisterType(Context
, NewVT
);
1665 RegisterVT
= DestVT
;
1667 if (EVT(DestVT
).bitsLT(NewVT
)) { // Value is expanded, e.g. i64 -> i16.
1668 TypeSize NewVTSize
= NewVT
.getSizeInBits();
1669 // Convert sizes such as i33 to i64.
1670 if (!llvm::has_single_bit
<uint32_t>(NewVTSize
.getKnownMinValue()))
1671 NewVTSize
= NewVTSize
.coefficientNextPowerOf2();
1672 return NumVectorRegs
*(NewVTSize
/DestVT
.getSizeInBits());
1675 // Otherwise, promotion or legal types use the same number of registers as
1676 // the vector decimated to the appropriate level.
1677 return NumVectorRegs
;
1680 bool TargetLoweringBase::isSuitableForJumpTable(const SwitchInst
*SI
,
1683 ProfileSummaryInfo
*PSI
,
1684 BlockFrequencyInfo
*BFI
) const {
1685 // FIXME: This function check the maximum table size and density, but the
1686 // minimum size is not checked. It would be nice if the minimum size is
1687 // also combined within this function. Currently, the minimum size check is
1688 // performed in findJumpTable() in SelectionDAGBuiler and
1689 // getEstimatedNumberOfCaseClusters() in BasicTTIImpl.
1690 const bool OptForSize
=
1691 SI
->getParent()->getParent()->hasOptSize() ||
1692 llvm::shouldOptimizeForSize(SI
->getParent(), PSI
, BFI
);
1693 const unsigned MinDensity
= getMinimumJumpTableDensity(OptForSize
);
1694 const unsigned MaxJumpTableSize
= getMaximumJumpTableSize();
1696 // Check whether the number of cases is small enough and
1697 // the range is dense enough for a jump table.
1698 return (OptForSize
|| Range
<= MaxJumpTableSize
) &&
1699 (NumCases
* 100 >= Range
* MinDensity
);
1702 MVT
TargetLoweringBase::getPreferredSwitchConditionType(LLVMContext
&Context
,
1703 EVT ConditionVT
) const {
1704 return getRegisterType(Context
, ConditionVT
);
1707 /// Get the EVTs and ArgFlags collections that represent the legalized return
1708 /// type of the given function. This does not require a DAG or a return value,
1709 /// and is suitable for use before any DAGs for the function are constructed.
1710 /// TODO: Move this out of TargetLowering.cpp.
1711 void llvm::GetReturnInfo(CallingConv::ID CC
, Type
*ReturnType
,
1713 SmallVectorImpl
<ISD::OutputArg
> &Outs
,
1714 const TargetLowering
&TLI
, const DataLayout
&DL
) {
1715 SmallVector
<EVT
, 4> ValueVTs
;
1716 ComputeValueVTs(TLI
, DL
, ReturnType
, ValueVTs
);
1717 unsigned NumValues
= ValueVTs
.size();
1718 if (NumValues
== 0) return;
1720 for (unsigned j
= 0, f
= NumValues
; j
!= f
; ++j
) {
1721 EVT VT
= ValueVTs
[j
];
1722 ISD::NodeType ExtendKind
= ISD::ANY_EXTEND
;
1724 if (attr
.hasRetAttr(Attribute::SExt
))
1725 ExtendKind
= ISD::SIGN_EXTEND
;
1726 else if (attr
.hasRetAttr(Attribute::ZExt
))
1727 ExtendKind
= ISD::ZERO_EXTEND
;
1729 // FIXME: C calling convention requires the return type to be promoted to
1730 // at least 32-bit. But this is not necessary for non-C calling
1731 // conventions. The frontend should mark functions whose return values
1732 // require promoting with signext or zeroext attributes.
1733 if (ExtendKind
!= ISD::ANY_EXTEND
&& VT
.isInteger()) {
1734 MVT MinVT
= TLI
.getRegisterType(MVT::i32
);
1735 if (VT
.bitsLT(MinVT
))
1740 TLI
.getNumRegistersForCallingConv(ReturnType
->getContext(), CC
, VT
);
1742 TLI
.getRegisterTypeForCallingConv(ReturnType
->getContext(), CC
, VT
);
1744 // 'inreg' on function refers to return value
1745 ISD::ArgFlagsTy Flags
= ISD::ArgFlagsTy();
1746 if (attr
.hasRetAttr(Attribute::InReg
))
1749 // Propagate extension type if any
1750 if (attr
.hasRetAttr(Attribute::SExt
))
1752 else if (attr
.hasRetAttr(Attribute::ZExt
))
1755 for (unsigned i
= 0; i
< NumParts
; ++i
)
1756 Outs
.push_back(ISD::OutputArg(Flags
, PartVT
, VT
, /*isfixed=*/true, 0, 0));
1760 /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
1761 /// function arguments in the caller parameter area. This is the actual
1762 /// alignment, not its logarithm.
1763 uint64_t TargetLoweringBase::getByValTypeAlignment(Type
*Ty
,
1764 const DataLayout
&DL
) const {
1765 return DL
.getABITypeAlign(Ty
).value();
1768 bool TargetLoweringBase::allowsMemoryAccessForAlignment(
1769 LLVMContext
&Context
, const DataLayout
&DL
, EVT VT
, unsigned AddrSpace
,
1770 Align Alignment
, MachineMemOperand::Flags Flags
, unsigned *Fast
) const {
1771 // Check if the specified alignment is sufficient based on the data layout.
1772 // TODO: While using the data layout works in practice, a better solution
1773 // would be to implement this check directly (make this a virtual function).
1774 // For example, the ABI alignment may change based on software platform while
1775 // this function should only be affected by hardware implementation.
1776 Type
*Ty
= VT
.getTypeForEVT(Context
);
1777 if (VT
.isZeroSized() || Alignment
>= DL
.getABITypeAlign(Ty
)) {
1778 // Assume that an access that meets the ABI-specified alignment is fast.
1779 if (Fast
!= nullptr)
1784 // This is a misaligned access.
1785 return allowsMisalignedMemoryAccesses(VT
, AddrSpace
, Alignment
, Flags
, Fast
);
1788 bool TargetLoweringBase::allowsMemoryAccessForAlignment(
1789 LLVMContext
&Context
, const DataLayout
&DL
, EVT VT
,
1790 const MachineMemOperand
&MMO
, unsigned *Fast
) const {
1791 return allowsMemoryAccessForAlignment(Context
, DL
, VT
, MMO
.getAddrSpace(),
1792 MMO
.getAlign(), MMO
.getFlags(), Fast
);
1795 bool TargetLoweringBase::allowsMemoryAccess(LLVMContext
&Context
,
1796 const DataLayout
&DL
, EVT VT
,
1797 unsigned AddrSpace
, Align Alignment
,
1798 MachineMemOperand::Flags Flags
,
1799 unsigned *Fast
) const {
1800 return allowsMemoryAccessForAlignment(Context
, DL
, VT
, AddrSpace
, Alignment
,
1804 bool TargetLoweringBase::allowsMemoryAccess(LLVMContext
&Context
,
1805 const DataLayout
&DL
, EVT VT
,
1806 const MachineMemOperand
&MMO
,
1807 unsigned *Fast
) const {
1808 return allowsMemoryAccess(Context
, DL
, VT
, MMO
.getAddrSpace(), MMO
.getAlign(),
1809 MMO
.getFlags(), Fast
);
1812 bool TargetLoweringBase::allowsMemoryAccess(LLVMContext
&Context
,
1813 const DataLayout
&DL
, LLT Ty
,
1814 const MachineMemOperand
&MMO
,
1815 unsigned *Fast
) const {
1816 EVT VT
= getApproximateEVTForLLT(Ty
, DL
, Context
);
1817 return allowsMemoryAccess(Context
, DL
, VT
, MMO
.getAddrSpace(), MMO
.getAlign(),
1818 MMO
.getFlags(), Fast
);
1821 //===----------------------------------------------------------------------===//
1822 // TargetTransformInfo Helpers
1823 //===----------------------------------------------------------------------===//
1825 int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode
) const {
1826 enum InstructionOpcodes
{
1827 #define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM,
1828 #define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM
1829 #include "llvm/IR/Instruction.def"
1831 switch (static_cast<InstructionOpcodes
>(Opcode
)) {
1834 case Switch
: return 0;
1835 case IndirectBr
: return 0;
1836 case Invoke
: return 0;
1837 case CallBr
: return 0;
1838 case Resume
: return 0;
1839 case Unreachable
: return 0;
1840 case CleanupRet
: return 0;
1841 case CatchRet
: return 0;
1842 case CatchPad
: return 0;
1843 case CatchSwitch
: return 0;
1844 case CleanupPad
: return 0;
1845 case FNeg
: return ISD::FNEG
;
1846 case Add
: return ISD::ADD
;
1847 case FAdd
: return ISD::FADD
;
1848 case Sub
: return ISD::SUB
;
1849 case FSub
: return ISD::FSUB
;
1850 case Mul
: return ISD::MUL
;
1851 case FMul
: return ISD::FMUL
;
1852 case UDiv
: return ISD::UDIV
;
1853 case SDiv
: return ISD::SDIV
;
1854 case FDiv
: return ISD::FDIV
;
1855 case URem
: return ISD::UREM
;
1856 case SRem
: return ISD::SREM
;
1857 case FRem
: return ISD::FREM
;
1858 case Shl
: return ISD::SHL
;
1859 case LShr
: return ISD::SRL
;
1860 case AShr
: return ISD::SRA
;
1861 case And
: return ISD::AND
;
1862 case Or
: return ISD::OR
;
1863 case Xor
: return ISD::XOR
;
1864 case Alloca
: return 0;
1865 case Load
: return ISD::LOAD
;
1866 case Store
: return ISD::STORE
;
1867 case GetElementPtr
: return 0;
1868 case Fence
: return 0;
1869 case AtomicCmpXchg
: return 0;
1870 case AtomicRMW
: return 0;
1871 case Trunc
: return ISD::TRUNCATE
;
1872 case ZExt
: return ISD::ZERO_EXTEND
;
1873 case SExt
: return ISD::SIGN_EXTEND
;
1874 case FPToUI
: return ISD::FP_TO_UINT
;
1875 case FPToSI
: return ISD::FP_TO_SINT
;
1876 case UIToFP
: return ISD::UINT_TO_FP
;
1877 case SIToFP
: return ISD::SINT_TO_FP
;
1878 case FPTrunc
: return ISD::FP_ROUND
;
1879 case FPExt
: return ISD::FP_EXTEND
;
1880 case PtrToInt
: return ISD::BITCAST
;
1881 case IntToPtr
: return ISD::BITCAST
;
1882 case BitCast
: return ISD::BITCAST
;
1883 case AddrSpaceCast
: return ISD::ADDRSPACECAST
;
1884 case ICmp
: return ISD::SETCC
;
1885 case FCmp
: return ISD::SETCC
;
1887 case Call
: return 0;
1888 case Select
: return ISD::SELECT
;
1889 case UserOp1
: return 0;
1890 case UserOp2
: return 0;
1891 case VAArg
: return 0;
1892 case ExtractElement
: return ISD::EXTRACT_VECTOR_ELT
;
1893 case InsertElement
: return ISD::INSERT_VECTOR_ELT
;
1894 case ShuffleVector
: return ISD::VECTOR_SHUFFLE
;
1895 case ExtractValue
: return ISD::MERGE_VALUES
;
1896 case InsertValue
: return ISD::MERGE_VALUES
;
1897 case LandingPad
: return 0;
1898 case Freeze
: return ISD::FREEZE
;
1901 llvm_unreachable("Unknown instruction type encountered!");
1905 TargetLoweringBase::getDefaultSafeStackPointerLocation(IRBuilderBase
&IRB
,
1906 bool UseTLS
) const {
1907 // compiler-rt provides a variable with a magic name. Targets that do not
1908 // link with compiler-rt may also provide such a variable.
1909 Module
*M
= IRB
.GetInsertBlock()->getParent()->getParent();
1910 const char *UnsafeStackPtrVar
= "__safestack_unsafe_stack_ptr";
1911 auto UnsafeStackPtr
=
1912 dyn_cast_or_null
<GlobalVariable
>(M
->getNamedValue(UnsafeStackPtrVar
));
1914 Type
*StackPtrTy
= Type::getInt8PtrTy(M
->getContext());
1916 if (!UnsafeStackPtr
) {
1917 auto TLSModel
= UseTLS
?
1918 GlobalValue::InitialExecTLSModel
:
1919 GlobalValue::NotThreadLocal
;
1920 // The global variable is not defined yet, define it ourselves.
1921 // We use the initial-exec TLS model because we do not support the
1922 // variable living anywhere other than in the main executable.
1923 UnsafeStackPtr
= new GlobalVariable(
1924 *M
, StackPtrTy
, false, GlobalValue::ExternalLinkage
, nullptr,
1925 UnsafeStackPtrVar
, nullptr, TLSModel
);
1927 // The variable exists, check its type and attributes.
1928 if (UnsafeStackPtr
->getValueType() != StackPtrTy
)
1929 report_fatal_error(Twine(UnsafeStackPtrVar
) + " must have void* type");
1930 if (UseTLS
!= UnsafeStackPtr
->isThreadLocal())
1931 report_fatal_error(Twine(UnsafeStackPtrVar
) + " must " +
1932 (UseTLS
? "" : "not ") + "be thread-local");
1934 return UnsafeStackPtr
;
1938 TargetLoweringBase::getSafeStackPointerLocation(IRBuilderBase
&IRB
) const {
1939 if (!TM
.getTargetTriple().isAndroid())
1940 return getDefaultSafeStackPointerLocation(IRB
, true);
1942 // Android provides a libc function to retrieve the address of the current
1943 // thread's unsafe stack pointer.
1944 Module
*M
= IRB
.GetInsertBlock()->getParent()->getParent();
1945 Type
*StackPtrTy
= Type::getInt8PtrTy(M
->getContext());
1946 FunctionCallee Fn
= M
->getOrInsertFunction("__safestack_pointer_address",
1947 StackPtrTy
->getPointerTo(0));
1948 return IRB
.CreateCall(Fn
);
1951 //===----------------------------------------------------------------------===//
1952 // Loop Strength Reduction hooks
1953 //===----------------------------------------------------------------------===//
1955 /// isLegalAddressingMode - Return true if the addressing mode represented
1956 /// by AM is legal for this target, for a load/store of the specified type.
1957 bool TargetLoweringBase::isLegalAddressingMode(const DataLayout
&DL
,
1958 const AddrMode
&AM
, Type
*Ty
,
1959 unsigned AS
, Instruction
*I
) const {
1960 // The default implementation of this implements a conservative RISCy, r+r and
1963 // Allows a sign-extended 16-bit immediate field.
1964 if (AM
.BaseOffs
<= -(1LL << 16) || AM
.BaseOffs
>= (1LL << 16)-1)
1967 // No global is ever allowed as a base.
1971 // Only support r+r,
1973 case 0: // "r+i" or just "i", depending on HasBaseReg.
1976 if (AM
.HasBaseReg
&& AM
.BaseOffs
) // "r+r+i" is not allowed.
1978 // Otherwise we have r+r or r+i.
1981 if (AM
.HasBaseReg
|| AM
.BaseOffs
) // 2*r+r or 2*r+i is not allowed.
1983 // Allow 2*r as r+r.
1985 default: // Don't allow n * r
1992 //===----------------------------------------------------------------------===//
1994 //===----------------------------------------------------------------------===//
1996 // For OpenBSD return its special guard variable. Otherwise return nullptr,
1997 // so that SelectionDAG handle SSP.
1998 Value
*TargetLoweringBase::getIRStackGuard(IRBuilderBase
&IRB
) const {
1999 if (getTargetMachine().getTargetTriple().isOSOpenBSD()) {
2000 Module
&M
= *IRB
.GetInsertBlock()->getParent()->getParent();
2001 PointerType
*PtrTy
= Type::getInt8PtrTy(M
.getContext());
2002 Constant
*C
= M
.getOrInsertGlobal("__guard_local", PtrTy
);
2003 if (GlobalVariable
*G
= dyn_cast_or_null
<GlobalVariable
>(C
))
2004 G
->setVisibility(GlobalValue::HiddenVisibility
);
2010 // Currently only support "standard" __stack_chk_guard.
2011 // TODO: add LOAD_STACK_GUARD support.
2012 void TargetLoweringBase::insertSSPDeclarations(Module
&M
) const {
2013 if (!M
.getNamedValue("__stack_chk_guard")) {
2014 auto *GV
= new GlobalVariable(M
, Type::getInt8PtrTy(M
.getContext()), false,
2015 GlobalVariable::ExternalLinkage
, nullptr,
2016 "__stack_chk_guard");
2018 // FreeBSD has "__stack_chk_guard" defined externally on libc.so
2019 if (M
.getDirectAccessExternalData() &&
2020 !TM
.getTargetTriple().isWindowsGNUEnvironment() &&
2021 !TM
.getTargetTriple().isOSFreeBSD() &&
2022 (!TM
.getTargetTriple().isOSDarwin() ||
2023 TM
.getRelocationModel() == Reloc::Static
))
2024 GV
->setDSOLocal(true);
2028 // Currently only support "standard" __stack_chk_guard.
2029 // TODO: add LOAD_STACK_GUARD support.
2030 Value
*TargetLoweringBase::getSDagStackGuard(const Module
&M
) const {
2031 return M
.getNamedValue("__stack_chk_guard");
2034 Function
*TargetLoweringBase::getSSPStackGuardCheck(const Module
&M
) const {
2038 unsigned TargetLoweringBase::getMinimumJumpTableEntries() const {
2039 return MinimumJumpTableEntries
;
2042 void TargetLoweringBase::setMinimumJumpTableEntries(unsigned Val
) {
2043 MinimumJumpTableEntries
= Val
;
2046 unsigned TargetLoweringBase::getMinimumJumpTableDensity(bool OptForSize
) const {
2047 return OptForSize
? OptsizeJumpTableDensity
: JumpTableDensity
;
2050 unsigned TargetLoweringBase::getMaximumJumpTableSize() const {
2051 return MaximumJumpTableSize
;
2054 void TargetLoweringBase::setMaximumJumpTableSize(unsigned Val
) {
2055 MaximumJumpTableSize
= Val
;
2058 bool TargetLoweringBase::isJumpTableRelative() const {
2059 return getTargetMachine().isPositionIndependent();
2062 Align
TargetLoweringBase::getPrefLoopAlignment(MachineLoop
*ML
) const {
2063 if (TM
.Options
.LoopAlignment
)
2064 return Align(TM
.Options
.LoopAlignment
);
2065 return PrefLoopAlignment
;
2068 unsigned TargetLoweringBase::getMaxPermittedBytesForAlignment(
2069 MachineBasicBlock
*MBB
) const {
2070 return MaxBytesForAlignment
;
2073 //===----------------------------------------------------------------------===//
2074 // Reciprocal Estimates
2075 //===----------------------------------------------------------------------===//
2077 /// Get the reciprocal estimate attribute string for a function that will
2078 /// override the target defaults.
2079 static StringRef
getRecipEstimateForFunc(MachineFunction
&MF
) {
2080 const Function
&F
= MF
.getFunction();
2081 return F
.getFnAttribute("reciprocal-estimates").getValueAsString();
2084 /// Construct a string for the given reciprocal operation of the given type.
2085 /// This string should match the corresponding option to the front-end's
2086 /// "-mrecip" flag assuming those strings have been passed through in an
2087 /// attribute string. For example, "vec-divf" for a division of a vXf32.
2088 static std::string
getReciprocalOpName(bool IsSqrt
, EVT VT
) {
2089 std::string Name
= VT
.isVector() ? "vec-" : "";
2091 Name
+= IsSqrt
? "sqrt" : "div";
2093 // TODO: Handle other float types?
2094 if (VT
.getScalarType() == MVT::f64
) {
2096 } else if (VT
.getScalarType() == MVT::f16
) {
2099 assert(VT
.getScalarType() == MVT::f32
&&
2100 "Unexpected FP type for reciprocal estimate");
2107 /// Return the character position and value (a single numeric character) of a
2108 /// customized refinement operation in the input string if it exists. Return
2109 /// false if there is no customized refinement step count.
2110 static bool parseRefinementStep(StringRef In
, size_t &Position
,
2112 const char RefStepToken
= ':';
2113 Position
= In
.find(RefStepToken
);
2114 if (Position
== StringRef::npos
)
2117 StringRef RefStepString
= In
.substr(Position
+ 1);
2118 // Allow exactly one numeric character for the additional refinement
2120 if (RefStepString
.size() == 1) {
2121 char RefStepChar
= RefStepString
[0];
2122 if (isDigit(RefStepChar
)) {
2123 Value
= RefStepChar
- '0';
2127 report_fatal_error("Invalid refinement step for -recip.");
2130 /// For the input attribute string, return one of the ReciprocalEstimate enum
2131 /// status values (enabled, disabled, or not specified) for this operation on
2132 /// the specified data type.
2133 static int getOpEnabled(bool IsSqrt
, EVT VT
, StringRef Override
) {
2134 if (Override
.empty())
2135 return TargetLoweringBase::ReciprocalEstimate::Unspecified
;
2137 SmallVector
<StringRef
, 4> OverrideVector
;
2138 Override
.split(OverrideVector
, ',');
2139 unsigned NumArgs
= OverrideVector
.size();
2141 // Check if "all", "none", or "default" was specified.
2143 // Look for an optional setting of the number of refinement steps needed
2144 // for this type of reciprocal operation.
2147 if (parseRefinementStep(Override
, RefPos
, RefSteps
)) {
2148 // Split the string for further processing.
2149 Override
= Override
.substr(0, RefPos
);
2152 // All reciprocal types are enabled.
2153 if (Override
== "all")
2154 return TargetLoweringBase::ReciprocalEstimate::Enabled
;
2156 // All reciprocal types are disabled.
2157 if (Override
== "none")
2158 return TargetLoweringBase::ReciprocalEstimate::Disabled
;
2160 // Target defaults for enablement are used.
2161 if (Override
== "default")
2162 return TargetLoweringBase::ReciprocalEstimate::Unspecified
;
2165 // The attribute string may omit the size suffix ('f'/'d').
2166 std::string VTName
= getReciprocalOpName(IsSqrt
, VT
);
2167 std::string VTNameNoSize
= VTName
;
2168 VTNameNoSize
.pop_back();
2169 static const char DisabledPrefix
= '!';
2171 for (StringRef RecipType
: OverrideVector
) {
2174 if (parseRefinementStep(RecipType
, RefPos
, RefSteps
))
2175 RecipType
= RecipType
.substr(0, RefPos
);
2177 // Ignore the disablement token for string matching.
2178 bool IsDisabled
= RecipType
[0] == DisabledPrefix
;
2180 RecipType
= RecipType
.substr(1);
2182 if (RecipType
.equals(VTName
) || RecipType
.equals(VTNameNoSize
))
2183 return IsDisabled
? TargetLoweringBase::ReciprocalEstimate::Disabled
2184 : TargetLoweringBase::ReciprocalEstimate::Enabled
;
2187 return TargetLoweringBase::ReciprocalEstimate::Unspecified
;
2190 /// For the input attribute string, return the customized refinement step count
2191 /// for this operation on the specified data type. If the step count does not
2192 /// exist, return the ReciprocalEstimate enum value for unspecified.
2193 static int getOpRefinementSteps(bool IsSqrt
, EVT VT
, StringRef Override
) {
2194 if (Override
.empty())
2195 return TargetLoweringBase::ReciprocalEstimate::Unspecified
;
2197 SmallVector
<StringRef
, 4> OverrideVector
;
2198 Override
.split(OverrideVector
, ',');
2199 unsigned NumArgs
= OverrideVector
.size();
2201 // Check if "all", "default", or "none" was specified.
2203 // Look for an optional setting of the number of refinement steps needed
2204 // for this type of reciprocal operation.
2207 if (!parseRefinementStep(Override
, RefPos
, RefSteps
))
2208 return TargetLoweringBase::ReciprocalEstimate::Unspecified
;
2210 // Split the string for further processing.
2211 Override
= Override
.substr(0, RefPos
);
2212 assert(Override
!= "none" &&
2213 "Disabled reciprocals, but specifed refinement steps?");
2215 // If this is a general override, return the specified number of steps.
2216 if (Override
== "all" || Override
== "default")
2220 // The attribute string may omit the size suffix ('f'/'d').
2221 std::string VTName
= getReciprocalOpName(IsSqrt
, VT
);
2222 std::string VTNameNoSize
= VTName
;
2223 VTNameNoSize
.pop_back();
2225 for (StringRef RecipType
: OverrideVector
) {
2228 if (!parseRefinementStep(RecipType
, RefPos
, RefSteps
))
2231 RecipType
= RecipType
.substr(0, RefPos
);
2232 if (RecipType
.equals(VTName
) || RecipType
.equals(VTNameNoSize
))
2236 return TargetLoweringBase::ReciprocalEstimate::Unspecified
;
2239 int TargetLoweringBase::getRecipEstimateSqrtEnabled(EVT VT
,
2240 MachineFunction
&MF
) const {
2241 return getOpEnabled(true, VT
, getRecipEstimateForFunc(MF
));
2244 int TargetLoweringBase::getRecipEstimateDivEnabled(EVT VT
,
2245 MachineFunction
&MF
) const {
2246 return getOpEnabled(false, VT
, getRecipEstimateForFunc(MF
));
2249 int TargetLoweringBase::getSqrtRefinementSteps(EVT VT
,
2250 MachineFunction
&MF
) const {
2251 return getOpRefinementSteps(true, VT
, getRecipEstimateForFunc(MF
));
2254 int TargetLoweringBase::getDivRefinementSteps(EVT VT
,
2255 MachineFunction
&MF
) const {
2256 return getOpRefinementSteps(false, VT
, getRecipEstimateForFunc(MF
));
2259 bool TargetLoweringBase::isLoadBitCastBeneficial(
2260 EVT LoadVT
, EVT BitcastVT
, const SelectionDAG
&DAG
,
2261 const MachineMemOperand
&MMO
) const {
2262 // Single-element vectors are scalarized, so we should generally avoid having
2263 // any memory operations on such types, as they would get scalarized too.
2264 if (LoadVT
.isFixedLengthVector() && BitcastVT
.isFixedLengthVector() &&
2265 BitcastVT
.getVectorNumElements() == 1)
2268 // Don't do if we could do an indexed load on the original type, but not on
2270 if (!LoadVT
.isSimple() || !BitcastVT
.isSimple())
2273 MVT LoadMVT
= LoadVT
.getSimpleVT();
2275 // Don't bother doing this if it's just going to be promoted again later, as
2276 // doing so might interfere with other combines.
2277 if (getOperationAction(ISD::LOAD
, LoadMVT
) == Promote
&&
2278 getTypeToPromoteTo(ISD::LOAD
, LoadMVT
) == BitcastVT
.getSimpleVT())
2282 return allowsMemoryAccess(*DAG
.getContext(), DAG
.getDataLayout(), BitcastVT
,
2287 void TargetLoweringBase::finalizeLowering(MachineFunction
&MF
) const {
2288 MF
.getRegInfo().freezeReservedRegs(MF
);
2291 MachineMemOperand::Flags
TargetLoweringBase::getLoadMemOperandFlags(
2292 const LoadInst
&LI
, const DataLayout
&DL
, AssumptionCache
*AC
,
2293 const TargetLibraryInfo
*LibInfo
) const {
2294 MachineMemOperand::Flags Flags
= MachineMemOperand::MOLoad
;
2295 if (LI
.isVolatile())
2296 Flags
|= MachineMemOperand::MOVolatile
;
2298 if (LI
.hasMetadata(LLVMContext::MD_nontemporal
))
2299 Flags
|= MachineMemOperand::MONonTemporal
;
2301 if (LI
.hasMetadata(LLVMContext::MD_invariant_load
))
2302 Flags
|= MachineMemOperand::MOInvariant
;
2304 if (isDereferenceableAndAlignedPointer(LI
.getPointerOperand(), LI
.getType(),
2305 LI
.getAlign(), DL
, &LI
, AC
,
2306 /*DT=*/nullptr, LibInfo
))
2307 Flags
|= MachineMemOperand::MODereferenceable
;
2309 Flags
|= getTargetMMOFlags(LI
);
2313 MachineMemOperand::Flags
2314 TargetLoweringBase::getStoreMemOperandFlags(const StoreInst
&SI
,
2315 const DataLayout
&DL
) const {
2316 MachineMemOperand::Flags Flags
= MachineMemOperand::MOStore
;
2318 if (SI
.isVolatile())
2319 Flags
|= MachineMemOperand::MOVolatile
;
2321 if (SI
.hasMetadata(LLVMContext::MD_nontemporal
))
2322 Flags
|= MachineMemOperand::MONonTemporal
;
2324 // FIXME: Not preserving dereferenceable
2325 Flags
|= getTargetMMOFlags(SI
);
2329 MachineMemOperand::Flags
2330 TargetLoweringBase::getAtomicMemOperandFlags(const Instruction
&AI
,
2331 const DataLayout
&DL
) const {
2332 auto Flags
= MachineMemOperand::MOLoad
| MachineMemOperand::MOStore
;
2334 if (const AtomicRMWInst
*RMW
= dyn_cast
<AtomicRMWInst
>(&AI
)) {
2335 if (RMW
->isVolatile())
2336 Flags
|= MachineMemOperand::MOVolatile
;
2337 } else if (const AtomicCmpXchgInst
*CmpX
= dyn_cast
<AtomicCmpXchgInst
>(&AI
)) {
2338 if (CmpX
->isVolatile())
2339 Flags
|= MachineMemOperand::MOVolatile
;
2341 llvm_unreachable("not an atomic instruction");
2343 // FIXME: Not preserving dereferenceable
2344 Flags
|= getTargetMMOFlags(AI
);
2348 Instruction
*TargetLoweringBase::emitLeadingFence(IRBuilderBase
&Builder
,
2350 AtomicOrdering Ord
) const {
2351 if (isReleaseOrStronger(Ord
) && Inst
->hasAtomicStore())
2352 return Builder
.CreateFence(Ord
);
2357 Instruction
*TargetLoweringBase::emitTrailingFence(IRBuilderBase
&Builder
,
2359 AtomicOrdering Ord
) const {
2360 if (isAcquireOrStronger(Ord
))
2361 return Builder
.CreateFence(Ord
);
2366 //===----------------------------------------------------------------------===//
2368 //===----------------------------------------------------------------------===//
2370 bool TargetLoweringBase::shouldLocalize(const MachineInstr
&MI
,
2371 const TargetTransformInfo
*TTI
) const {
2372 auto &MF
= *MI
.getMF();
2373 auto &MRI
= MF
.getRegInfo();
2374 // Assuming a spill and reload of a value has a cost of 1 instruction each,
2375 // this helper function computes the maximum number of uses we should consider
2376 // for remat. E.g. on arm64 global addresses take 2 insts to materialize. We
2377 // break even in terms of code size when the original MI has 2 users vs
2378 // choosing to potentially spill. Any more than 2 users we we have a net code
2379 // size increase. This doesn't take into account register pressure though.
2380 auto maxUses
= [](unsigned RematCost
) {
2381 // A cost of 1 means remats are basically free.
2383 return std::numeric_limits
<unsigned>::max();
2387 // Remat is too expensive, only sink if there's one user.
2390 llvm_unreachable("Unexpected remat cost");
2393 switch (MI
.getOpcode()) {
2396 // Constants-like instructions should be close to their users.
2397 // We don't want long live-ranges for them.
2398 case TargetOpcode::G_CONSTANT
:
2399 case TargetOpcode::G_FCONSTANT
:
2400 case TargetOpcode::G_FRAME_INDEX
:
2401 case TargetOpcode::G_INTTOPTR
:
2403 case TargetOpcode::G_GLOBAL_VALUE
: {
2404 unsigned RematCost
= TTI
->getGISelRematGlobalCost();
2405 Register Reg
= MI
.getOperand(0).getReg();
2406 unsigned MaxUses
= maxUses(RematCost
);
2407 if (MaxUses
== UINT_MAX
)
2408 return true; // Remats are "free" so always localize.
2409 return MRI
.hasAtMostUserInstrs(Reg
, MaxUses
);