1 //===-- IntrinsicLowering.cpp - Intrinsic Lowering default implementation -===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the IntrinsicLowering class.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/Constants.h"
15 #include "llvm/DerivedTypes.h"
16 #include "llvm/Module.h"
17 #include "llvm/Type.h"
18 #include "llvm/CodeGen/IntrinsicLowering.h"
19 #include "llvm/Support/CallSite.h"
20 #include "llvm/Support/ErrorHandling.h"
21 #include "llvm/Support/IRBuilder.h"
22 #include "llvm/Support/raw_ostream.h"
23 #include "llvm/Target/TargetData.h"
24 #include "llvm/ADT/SmallVector.h"
27 template <class ArgIt
>
28 static void EnsureFunctionExists(Module
&M
, const char *Name
,
29 ArgIt ArgBegin
, ArgIt ArgEnd
,
31 // Insert a correctly-typed definition now.
32 std::vector
<const Type
*> ParamTys
;
33 for (ArgIt I
= ArgBegin
; I
!= ArgEnd
; ++I
)
34 ParamTys
.push_back(I
->getType());
35 M
.getOrInsertFunction(Name
, FunctionType::get(RetTy
, ParamTys
, false));
38 static void EnsureFPIntrinsicsExist(Module
&M
, Function
*Fn
,
40 const char *DName
, const char *LDName
) {
41 // Insert definitions for all the floating point types.
42 switch((int)Fn
->arg_begin()->getType()->getTypeID()) {
44 EnsureFunctionExists(M
, FName
, Fn
->arg_begin(), Fn
->arg_end(),
45 Type::getFloatTy(M
.getContext()));
47 case Type::DoubleTyID
:
48 EnsureFunctionExists(M
, DName
, Fn
->arg_begin(), Fn
->arg_end(),
49 Type::getDoubleTy(M
.getContext()));
51 case Type::X86_FP80TyID
:
53 case Type::PPC_FP128TyID
:
54 EnsureFunctionExists(M
, LDName
, Fn
->arg_begin(), Fn
->arg_end(),
55 Fn
->arg_begin()->getType());
60 /// ReplaceCallWith - This function is used when we want to lower an intrinsic
61 /// call to a call of an external function. This handles hard cases such as
62 /// when there was already a prototype for the external function, and if that
63 /// prototype doesn't match the arguments we expect to pass in.
64 template <class ArgIt
>
65 static CallInst
*ReplaceCallWith(const char *NewFn
, CallInst
*CI
,
66 ArgIt ArgBegin
, ArgIt ArgEnd
,
68 // If we haven't already looked up this function, check to see if the
69 // program already contains a function with this name.
70 Module
*M
= CI
->getParent()->getParent()->getParent();
71 // Get or insert the definition now.
72 std::vector
<const Type
*> ParamTys
;
73 for (ArgIt I
= ArgBegin
; I
!= ArgEnd
; ++I
)
74 ParamTys
.push_back((*I
)->getType());
75 Constant
* FCache
= M
->getOrInsertFunction(NewFn
,
76 FunctionType::get(RetTy
, ParamTys
, false));
78 IRBuilder
<> Builder(CI
->getParent(), CI
);
79 SmallVector
<Value
*, 8> Args(ArgBegin
, ArgEnd
);
80 CallInst
*NewCI
= Builder
.CreateCall(FCache
, Args
.begin(), Args
.end());
81 NewCI
->setName(CI
->getName());
83 CI
->replaceAllUsesWith(NewCI
);
87 // VisualStudio defines setjmp as _setjmp
88 #if defined(_MSC_VER) && defined(setjmp) && \
89 !defined(setjmp_undefined_for_msvc)
90 # pragma push_macro("setjmp")
92 # define setjmp_undefined_for_msvc
95 void IntrinsicLowering::AddPrototypes(Module
&M
) {
96 LLVMContext
&Context
= M
.getContext();
97 for (Module::iterator I
= M
.begin(), E
= M
.end(); I
!= E
; ++I
)
98 if (I
->isDeclaration() && !I
->use_empty())
99 switch (I
->getIntrinsicID()) {
101 case Intrinsic::setjmp
:
102 EnsureFunctionExists(M
, "setjmp", I
->arg_begin(), I
->arg_end(),
103 Type::getInt32Ty(M
.getContext()));
105 case Intrinsic::longjmp
:
106 EnsureFunctionExists(M
, "longjmp", I
->arg_begin(), I
->arg_end(),
107 Type::getVoidTy(M
.getContext()));
109 case Intrinsic::siglongjmp
:
110 EnsureFunctionExists(M
, "abort", I
->arg_end(), I
->arg_end(),
111 Type::getVoidTy(M
.getContext()));
113 case Intrinsic::memcpy
:
114 M
.getOrInsertFunction("memcpy",
115 Type::getInt8PtrTy(Context
),
116 Type::getInt8PtrTy(Context
),
117 Type::getInt8PtrTy(Context
),
118 TD
.getIntPtrType(Context
), (Type
*)0);
120 case Intrinsic::memmove
:
121 M
.getOrInsertFunction("memmove",
122 Type::getInt8PtrTy(Context
),
123 Type::getInt8PtrTy(Context
),
124 Type::getInt8PtrTy(Context
),
125 TD
.getIntPtrType(Context
), (Type
*)0);
127 case Intrinsic::memset
:
128 M
.getOrInsertFunction("memset",
129 Type::getInt8PtrTy(Context
),
130 Type::getInt8PtrTy(Context
),
131 Type::getInt32Ty(M
.getContext()),
132 TD
.getIntPtrType(Context
), (Type
*)0);
134 case Intrinsic::sqrt
:
135 EnsureFPIntrinsicsExist(M
, I
, "sqrtf", "sqrt", "sqrtl");
138 EnsureFPIntrinsicsExist(M
, I
, "sinf", "sin", "sinl");
141 EnsureFPIntrinsicsExist(M
, I
, "cosf", "cos", "cosl");
144 EnsureFPIntrinsicsExist(M
, I
, "powf", "pow", "powl");
147 EnsureFPIntrinsicsExist(M
, I
, "logf", "log", "logl");
149 case Intrinsic::log2
:
150 EnsureFPIntrinsicsExist(M
, I
, "log2f", "log2", "log2l");
152 case Intrinsic::log10
:
153 EnsureFPIntrinsicsExist(M
, I
, "log10f", "log10", "log10l");
156 EnsureFPIntrinsicsExist(M
, I
, "expf", "exp", "expl");
158 case Intrinsic::exp2
:
159 EnsureFPIntrinsicsExist(M
, I
, "exp2f", "exp2", "exp2l");
164 /// LowerBSWAP - Emit the code to lower bswap of V before the specified
166 static Value
*LowerBSWAP(LLVMContext
&Context
, Value
*V
, Instruction
*IP
) {
167 assert(V
->getType()->isIntegerTy() && "Can't bswap a non-integer type!");
169 unsigned BitSize
= V
->getType()->getPrimitiveSizeInBits();
171 IRBuilder
<> Builder(IP
->getParent(), IP
);
174 default: llvm_unreachable("Unhandled type size of value to byteswap!");
176 Value
*Tmp1
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 8),
178 Value
*Tmp2
= Builder
.CreateLShr(V
, ConstantInt::get(V
->getType(), 8),
180 V
= Builder
.CreateOr(Tmp1
, Tmp2
, "bswap.i16");
184 Value
*Tmp4
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 24),
186 Value
*Tmp3
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 8),
188 Value
*Tmp2
= Builder
.CreateLShr(V
, ConstantInt::get(V
->getType(), 8),
190 Value
*Tmp1
= Builder
.CreateLShr(V
,ConstantInt::get(V
->getType(), 24),
192 Tmp3
= Builder
.CreateAnd(Tmp3
,
193 ConstantInt::get(Type::getInt32Ty(Context
), 0xFF0000),
195 Tmp2
= Builder
.CreateAnd(Tmp2
,
196 ConstantInt::get(Type::getInt32Ty(Context
), 0xFF00),
198 Tmp4
= Builder
.CreateOr(Tmp4
, Tmp3
, "bswap.or1");
199 Tmp2
= Builder
.CreateOr(Tmp2
, Tmp1
, "bswap.or2");
200 V
= Builder
.CreateOr(Tmp4
, Tmp2
, "bswap.i32");
204 Value
*Tmp8
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 56),
206 Value
*Tmp7
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 40),
208 Value
*Tmp6
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 24),
210 Value
*Tmp5
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 8),
212 Value
* Tmp4
= Builder
.CreateLShr(V
, ConstantInt::get(V
->getType(), 8),
214 Value
* Tmp3
= Builder
.CreateLShr(V
,
215 ConstantInt::get(V
->getType(), 24),
217 Value
* Tmp2
= Builder
.CreateLShr(V
,
218 ConstantInt::get(V
->getType(), 40),
220 Value
* Tmp1
= Builder
.CreateLShr(V
,
221 ConstantInt::get(V
->getType(), 56),
223 Tmp7
= Builder
.CreateAnd(Tmp7
,
224 ConstantInt::get(Type::getInt64Ty(Context
),
225 0xFF000000000000ULL
),
227 Tmp6
= Builder
.CreateAnd(Tmp6
,
228 ConstantInt::get(Type::getInt64Ty(Context
),
231 Tmp5
= Builder
.CreateAnd(Tmp5
,
232 ConstantInt::get(Type::getInt64Ty(Context
),
235 Tmp4
= Builder
.CreateAnd(Tmp4
,
236 ConstantInt::get(Type::getInt64Ty(Context
),
239 Tmp3
= Builder
.CreateAnd(Tmp3
,
240 ConstantInt::get(Type::getInt64Ty(Context
),
243 Tmp2
= Builder
.CreateAnd(Tmp2
,
244 ConstantInt::get(Type::getInt64Ty(Context
),
247 Tmp8
= Builder
.CreateOr(Tmp8
, Tmp7
, "bswap.or1");
248 Tmp6
= Builder
.CreateOr(Tmp6
, Tmp5
, "bswap.or2");
249 Tmp4
= Builder
.CreateOr(Tmp4
, Tmp3
, "bswap.or3");
250 Tmp2
= Builder
.CreateOr(Tmp2
, Tmp1
, "bswap.or4");
251 Tmp8
= Builder
.CreateOr(Tmp8
, Tmp6
, "bswap.or5");
252 Tmp4
= Builder
.CreateOr(Tmp4
, Tmp2
, "bswap.or6");
253 V
= Builder
.CreateOr(Tmp8
, Tmp4
, "bswap.i64");
260 /// LowerCTPOP - Emit the code to lower ctpop of V before the specified
262 static Value
*LowerCTPOP(LLVMContext
&Context
, Value
*V
, Instruction
*IP
) {
263 assert(V
->getType()->isIntegerTy() && "Can't ctpop a non-integer type!");
265 static const uint64_t MaskValues
[6] = {
266 0x5555555555555555ULL
, 0x3333333333333333ULL
,
267 0x0F0F0F0F0F0F0F0FULL
, 0x00FF00FF00FF00FFULL
,
268 0x0000FFFF0000FFFFULL
, 0x00000000FFFFFFFFULL
271 IRBuilder
<> Builder(IP
->getParent(), IP
);
273 unsigned BitSize
= V
->getType()->getPrimitiveSizeInBits();
274 unsigned WordSize
= (BitSize
+ 63) / 64;
275 Value
*Count
= ConstantInt::get(V
->getType(), 0);
277 for (unsigned n
= 0; n
< WordSize
; ++n
) {
278 Value
*PartValue
= V
;
279 for (unsigned i
= 1, ct
= 0; i
< (BitSize
>64 ? 64 : BitSize
);
281 Value
*MaskCst
= ConstantInt::get(V
->getType(), MaskValues
[ct
]);
282 Value
*LHS
= Builder
.CreateAnd(PartValue
, MaskCst
, "cppop.and1");
283 Value
*VShift
= Builder
.CreateLShr(PartValue
,
284 ConstantInt::get(V
->getType(), i
),
286 Value
*RHS
= Builder
.CreateAnd(VShift
, MaskCst
, "cppop.and2");
287 PartValue
= Builder
.CreateAdd(LHS
, RHS
, "ctpop.step");
289 Count
= Builder
.CreateAdd(PartValue
, Count
, "ctpop.part");
291 V
= Builder
.CreateLShr(V
, ConstantInt::get(V
->getType(), 64),
300 /// LowerCTLZ - Emit the code to lower ctlz of V before the specified
302 static Value
*LowerCTLZ(LLVMContext
&Context
, Value
*V
, Instruction
*IP
) {
304 IRBuilder
<> Builder(IP
->getParent(), IP
);
306 unsigned BitSize
= V
->getType()->getPrimitiveSizeInBits();
307 for (unsigned i
= 1; i
< BitSize
; i
<<= 1) {
308 Value
*ShVal
= ConstantInt::get(V
->getType(), i
);
309 ShVal
= Builder
.CreateLShr(V
, ShVal
, "ctlz.sh");
310 V
= Builder
.CreateOr(V
, ShVal
, "ctlz.step");
313 V
= Builder
.CreateNot(V
);
314 return LowerCTPOP(Context
, V
, IP
);
317 static void ReplaceFPIntrinsicWithCall(CallInst
*CI
, const char *Fname
,
319 const char *LDname
) {
321 switch (CI
->getArgOperand(0)->getType()->getTypeID()) {
322 default: llvm_unreachable("Invalid type in intrinsic");
323 case Type::FloatTyID
:
324 ReplaceCallWith(Fname
, CI
, CS
.arg_begin(), CS
.arg_end(),
325 Type::getFloatTy(CI
->getContext()));
327 case Type::DoubleTyID
:
328 ReplaceCallWith(Dname
, CI
, CS
.arg_begin(), CS
.arg_end(),
329 Type::getDoubleTy(CI
->getContext()));
331 case Type::X86_FP80TyID
:
332 case Type::FP128TyID
:
333 case Type::PPC_FP128TyID
:
334 ReplaceCallWith(LDname
, CI
, CS
.arg_begin(), CS
.arg_end(),
335 CI
->getArgOperand(0)->getType());
340 void IntrinsicLowering::LowerIntrinsicCall(CallInst
*CI
) {
341 IRBuilder
<> Builder(CI
->getParent(), CI
);
342 LLVMContext
&Context
= CI
->getContext();
344 const Function
*Callee
= CI
->getCalledFunction();
345 assert(Callee
&& "Cannot lower an indirect call!");
348 switch (Callee
->getIntrinsicID()) {
349 case Intrinsic::not_intrinsic
:
350 report_fatal_error("Cannot lower a call to a non-intrinsic function '"+
351 Callee
->getName() + "'!");
353 report_fatal_error("Code generator does not support intrinsic function '"+
354 Callee
->getName()+"'!");
356 // The setjmp/longjmp intrinsics should only exist in the code if it was
357 // never optimized (ie, right out of the CFE), or if it has been hacked on
358 // by the lowerinvoke pass. In both cases, the right thing to do is to
359 // convert the call to an explicit setjmp or longjmp call.
360 case Intrinsic::setjmp
: {
361 Value
*V
= ReplaceCallWith("setjmp", CI
, CS
.arg_begin(), CS
.arg_end(),
362 Type::getInt32Ty(Context
));
363 if (!CI
->getType()->isVoidTy())
364 CI
->replaceAllUsesWith(V
);
367 case Intrinsic::sigsetjmp
:
368 if (!CI
->getType()->isVoidTy())
369 CI
->replaceAllUsesWith(Constant::getNullValue(CI
->getType()));
372 case Intrinsic::longjmp
: {
373 ReplaceCallWith("longjmp", CI
, CS
.arg_begin(), CS
.arg_end(),
374 Type::getVoidTy(Context
));
378 case Intrinsic::siglongjmp
: {
379 // Insert the call to abort
380 ReplaceCallWith("abort", CI
, CS
.arg_end(), CS
.arg_end(),
381 Type::getVoidTy(Context
));
384 case Intrinsic::ctpop
:
385 CI
->replaceAllUsesWith(LowerCTPOP(Context
, CI
->getArgOperand(0), CI
));
388 case Intrinsic::bswap
:
389 CI
->replaceAllUsesWith(LowerBSWAP(Context
, CI
->getArgOperand(0), CI
));
392 case Intrinsic::ctlz
:
393 CI
->replaceAllUsesWith(LowerCTLZ(Context
, CI
->getArgOperand(0), CI
));
396 case Intrinsic::cttz
: {
397 // cttz(x) -> ctpop(~X & (X-1))
398 Value
*Src
= CI
->getArgOperand(0);
399 Value
*NotSrc
= Builder
.CreateNot(Src
);
400 NotSrc
->setName(Src
->getName() + ".not");
401 Value
*SrcM1
= ConstantInt::get(Src
->getType(), 1);
402 SrcM1
= Builder
.CreateSub(Src
, SrcM1
);
403 Src
= LowerCTPOP(Context
, Builder
.CreateAnd(NotSrc
, SrcM1
), CI
);
404 CI
->replaceAllUsesWith(Src
);
408 case Intrinsic::stacksave
:
409 case Intrinsic::stackrestore
: {
411 errs() << "WARNING: this target does not support the llvm.stack"
412 << (Callee
->getIntrinsicID() == Intrinsic::stacksave
?
413 "save" : "restore") << " intrinsic.\n";
415 if (Callee
->getIntrinsicID() == Intrinsic::stacksave
)
416 CI
->replaceAllUsesWith(Constant::getNullValue(CI
->getType()));
420 case Intrinsic::returnaddress
:
421 case Intrinsic::frameaddress
:
422 errs() << "WARNING: this target does not support the llvm."
423 << (Callee
->getIntrinsicID() == Intrinsic::returnaddress
?
424 "return" : "frame") << "address intrinsic.\n";
425 CI
->replaceAllUsesWith(ConstantPointerNull::get(
426 cast
<PointerType
>(CI
->getType())));
429 case Intrinsic::prefetch
:
430 break; // Simply strip out prefetches on unsupported architectures
432 case Intrinsic::pcmarker
:
433 break; // Simply strip out pcmarker on unsupported architectures
434 case Intrinsic::readcyclecounter
: {
435 errs() << "WARNING: this target does not support the llvm.readcyclecoun"
436 << "ter intrinsic. It is being lowered to a constant 0\n";
437 CI
->replaceAllUsesWith(ConstantInt::get(Type::getInt64Ty(Context
), 0));
441 case Intrinsic::dbg_declare
:
442 break; // Simply strip out debugging intrinsics
444 case Intrinsic::eh_exception
:
445 case Intrinsic::eh_selector
:
446 CI
->replaceAllUsesWith(Constant::getNullValue(CI
->getType()));
449 case Intrinsic::eh_typeid_for
:
450 // Return something different to eh_selector.
451 CI
->replaceAllUsesWith(ConstantInt::get(CI
->getType(), 1));
454 case Intrinsic::var_annotation
:
455 break; // Strip out annotate intrinsic
457 case Intrinsic::memcpy
: {
458 const IntegerType
*IntPtr
= TD
.getIntPtrType(Context
);
459 Value
*Size
= Builder
.CreateIntCast(CI
->getArgOperand(2), IntPtr
,
460 /* isSigned */ false);
462 Ops
[0] = CI
->getArgOperand(0);
463 Ops
[1] = CI
->getArgOperand(1);
465 ReplaceCallWith("memcpy", CI
, Ops
, Ops
+3, CI
->getArgOperand(0)->getType());
468 case Intrinsic::memmove
: {
469 const IntegerType
*IntPtr
= TD
.getIntPtrType(Context
);
470 Value
*Size
= Builder
.CreateIntCast(CI
->getArgOperand(2), IntPtr
,
471 /* isSigned */ false);
473 Ops
[0] = CI
->getArgOperand(0);
474 Ops
[1] = CI
->getArgOperand(1);
476 ReplaceCallWith("memmove", CI
, Ops
, Ops
+3, CI
->getArgOperand(0)->getType());
479 case Intrinsic::memset
: {
480 const IntegerType
*IntPtr
= TD
.getIntPtrType(Context
);
481 Value
*Size
= Builder
.CreateIntCast(CI
->getArgOperand(2), IntPtr
,
482 /* isSigned */ false);
484 Ops
[0] = CI
->getArgOperand(0);
485 // Extend the amount to i32.
486 Ops
[1] = Builder
.CreateIntCast(CI
->getArgOperand(1),
487 Type::getInt32Ty(Context
),
488 /* isSigned */ false);
490 ReplaceCallWith("memset", CI
, Ops
, Ops
+3, CI
->getArgOperand(0)->getType());
493 case Intrinsic::sqrt
: {
494 ReplaceFPIntrinsicWithCall(CI
, "sqrtf", "sqrt", "sqrtl");
497 case Intrinsic::log
: {
498 ReplaceFPIntrinsicWithCall(CI
, "logf", "log", "logl");
501 case Intrinsic::log2
: {
502 ReplaceFPIntrinsicWithCall(CI
, "log2f", "log2", "log2l");
505 case Intrinsic::log10
: {
506 ReplaceFPIntrinsicWithCall(CI
, "log10f", "log10", "log10l");
509 case Intrinsic::exp
: {
510 ReplaceFPIntrinsicWithCall(CI
, "expf", "exp", "expl");
513 case Intrinsic::exp2
: {
514 ReplaceFPIntrinsicWithCall(CI
, "exp2f", "exp2", "exp2l");
517 case Intrinsic::pow
: {
518 ReplaceFPIntrinsicWithCall(CI
, "powf", "pow", "powl");
521 case Intrinsic::flt_rounds
:
522 // Lower to "round to the nearest"
523 if (!CI
->getType()->isVoidTy())
524 CI
->replaceAllUsesWith(ConstantInt::get(CI
->getType(), 1));
526 case Intrinsic::invariant_start
:
527 case Intrinsic::lifetime_start
:
528 // Discard region information.
529 CI
->replaceAllUsesWith(UndefValue::get(CI
->getType()));
531 case Intrinsic::invariant_end
:
532 case Intrinsic::lifetime_end
:
533 // Discard region information.
537 assert(CI
->use_empty() &&
538 "Lowering should have eliminated any uses of the intrinsic call!");
539 CI
->eraseFromParent();
542 bool IntrinsicLowering::LowerToByteSwap(CallInst
*CI
) {
543 // Verify this is a simple bswap.
544 if (CI
->getNumArgOperands() != 1 ||
545 CI
->getType() != CI
->getArgOperand(0)->getType() ||
546 !CI
->getType()->isIntegerTy())
549 const IntegerType
*Ty
= dyn_cast
<IntegerType
>(CI
->getType());
553 // Okay, we can do this xform, do so now.
554 const Type
*Tys
[] = { Ty
};
555 Module
*M
= CI
->getParent()->getParent()->getParent();
556 Constant
*Int
= Intrinsic::getDeclaration(M
, Intrinsic::bswap
, Tys
, 1);
558 Value
*Op
= CI
->getArgOperand(0);
559 Op
= CallInst::Create(Int
, Op
, CI
->getName(), CI
);
561 CI
->replaceAllUsesWith(Op
);
562 CI
->eraseFromParent();