1 //===-- IntrinsicLowering.cpp - Intrinsic Lowering default implementation -===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the IntrinsicLowering class.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/Constants.h"
15 #include "llvm/DerivedTypes.h"
16 #include "llvm/Module.h"
17 #include "llvm/Type.h"
18 #include "llvm/CodeGen/IntrinsicLowering.h"
19 #include "llvm/Support/CallSite.h"
20 #include "llvm/Support/ErrorHandling.h"
21 #include "llvm/Support/IRBuilder.h"
22 #include "llvm/Support/raw_ostream.h"
23 #include "llvm/Target/TargetData.h"
24 #include "llvm/ADT/SmallVector.h"
27 template <class ArgIt
>
28 static void EnsureFunctionExists(Module
&M
, const char *Name
,
29 ArgIt ArgBegin
, ArgIt ArgEnd
,
31 // Insert a correctly-typed definition now.
32 std::vector
<const Type
*> ParamTys
;
33 for (ArgIt I
= ArgBegin
; I
!= ArgEnd
; ++I
)
34 ParamTys
.push_back(I
->getType());
35 M
.getOrInsertFunction(Name
, FunctionType::get(RetTy
, ParamTys
, false));
38 static void EnsureFPIntrinsicsExist(Module
&M
, Function
*Fn
,
40 const char *DName
, const char *LDName
) {
41 // Insert definitions for all the floating point types.
42 switch((int)Fn
->arg_begin()->getType()->getTypeID()) {
44 EnsureFunctionExists(M
, FName
, Fn
->arg_begin(), Fn
->arg_end(),
45 Type::getFloatTy(M
.getContext()));
47 case Type::DoubleTyID
:
48 EnsureFunctionExists(M
, DName
, Fn
->arg_begin(), Fn
->arg_end(),
49 Type::getDoubleTy(M
.getContext()));
51 case Type::X86_FP80TyID
:
53 case Type::PPC_FP128TyID
:
54 EnsureFunctionExists(M
, LDName
, Fn
->arg_begin(), Fn
->arg_end(),
55 Fn
->arg_begin()->getType());
60 /// ReplaceCallWith - This function is used when we want to lower an intrinsic
61 /// call to a call of an external function. This handles hard cases such as
62 /// when there was already a prototype for the external function, and if that
63 /// prototype doesn't match the arguments we expect to pass in.
64 template <class ArgIt
>
65 static CallInst
*ReplaceCallWith(const char *NewFn
, CallInst
*CI
,
66 ArgIt ArgBegin
, ArgIt ArgEnd
,
68 // If we haven't already looked up this function, check to see if the
69 // program already contains a function with this name.
70 Module
*M
= CI
->getParent()->getParent()->getParent();
71 // Get or insert the definition now.
72 std::vector
<const Type
*> ParamTys
;
73 for (ArgIt I
= ArgBegin
; I
!= ArgEnd
; ++I
)
74 ParamTys
.push_back((*I
)->getType());
75 Constant
* FCache
= M
->getOrInsertFunction(NewFn
,
76 FunctionType::get(RetTy
, ParamTys
, false));
78 IRBuilder
<> Builder(CI
->getParent(), CI
);
79 SmallVector
<Value
*, 8> Args(ArgBegin
, ArgEnd
);
80 CallInst
*NewCI
= Builder
.CreateCall(FCache
, Args
.begin(), Args
.end());
81 NewCI
->setName(CI
->getName());
83 CI
->replaceAllUsesWith(NewCI
);
87 // VisualStudio defines setjmp as _setjmp
88 #if defined(_MSC_VER) && defined(setjmp) && \
89 !defined(setjmp_undefined_for_msvc)
90 # pragma push_macro("setjmp")
92 # define setjmp_undefined_for_msvc
95 void IntrinsicLowering::AddPrototypes(Module
&M
) {
96 LLVMContext
&Context
= M
.getContext();
97 for (Module::iterator I
= M
.begin(), E
= M
.end(); I
!= E
; ++I
)
98 if (I
->isDeclaration() && !I
->use_empty())
99 switch (I
->getIntrinsicID()) {
101 case Intrinsic::setjmp
:
102 EnsureFunctionExists(M
, "setjmp", I
->arg_begin(), I
->arg_end(),
103 Type::getInt32Ty(M
.getContext()));
105 case Intrinsic::longjmp
:
106 EnsureFunctionExists(M
, "longjmp", I
->arg_begin(), I
->arg_end(),
107 Type::getVoidTy(M
.getContext()));
109 case Intrinsic::siglongjmp
:
110 EnsureFunctionExists(M
, "abort", I
->arg_end(), I
->arg_end(),
111 Type::getVoidTy(M
.getContext()));
113 case Intrinsic::memcpy
:
114 M
.getOrInsertFunction("memcpy",
115 Type::getInt8PtrTy(Context
),
116 Type::getInt8PtrTy(Context
),
117 Type::getInt8PtrTy(Context
),
118 TD
.getIntPtrType(Context
), (Type
*)0);
120 case Intrinsic::memmove
:
121 M
.getOrInsertFunction("memmove",
122 Type::getInt8PtrTy(Context
),
123 Type::getInt8PtrTy(Context
),
124 Type::getInt8PtrTy(Context
),
125 TD
.getIntPtrType(Context
), (Type
*)0);
127 case Intrinsic::memset
:
128 M
.getOrInsertFunction("memset",
129 Type::getInt8PtrTy(Context
),
130 Type::getInt8PtrTy(Context
),
131 Type::getInt32Ty(M
.getContext()),
132 TD
.getIntPtrType(Context
), (Type
*)0);
134 case Intrinsic::sqrt
:
135 EnsureFPIntrinsicsExist(M
, I
, "sqrtf", "sqrt", "sqrtl");
138 EnsureFPIntrinsicsExist(M
, I
, "sinf", "sin", "sinl");
141 EnsureFPIntrinsicsExist(M
, I
, "cosf", "cos", "cosl");
144 EnsureFPIntrinsicsExist(M
, I
, "powf", "pow", "powl");
147 EnsureFPIntrinsicsExist(M
, I
, "logf", "log", "logl");
149 case Intrinsic::log2
:
150 EnsureFPIntrinsicsExist(M
, I
, "log2f", "log2", "log2l");
152 case Intrinsic::log10
:
153 EnsureFPIntrinsicsExist(M
, I
, "log10f", "log10", "log10l");
156 EnsureFPIntrinsicsExist(M
, I
, "expf", "exp", "expl");
158 case Intrinsic::exp2
:
159 EnsureFPIntrinsicsExist(M
, I
, "exp2f", "exp2", "exp2l");
164 /// LowerBSWAP - Emit the code to lower bswap of V before the specified
166 static Value
*LowerBSWAP(LLVMContext
&Context
, Value
*V
, Instruction
*IP
) {
167 assert(V
->getType()->isIntegerTy() && "Can't bswap a non-integer type!");
169 unsigned BitSize
= V
->getType()->getPrimitiveSizeInBits();
171 IRBuilder
<> Builder(IP
->getParent(), IP
);
174 default: llvm_unreachable("Unhandled type size of value to byteswap!");
176 Value
*Tmp1
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 8),
178 Value
*Tmp2
= Builder
.CreateLShr(V
, ConstantInt::get(V
->getType(), 8),
180 V
= Builder
.CreateOr(Tmp1
, Tmp2
, "bswap.i16");
184 Value
*Tmp4
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 24),
186 Value
*Tmp3
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 8),
188 Value
*Tmp2
= Builder
.CreateLShr(V
, ConstantInt::get(V
->getType(), 8),
190 Value
*Tmp1
= Builder
.CreateLShr(V
,ConstantInt::get(V
->getType(), 24),
192 Tmp3
= Builder
.CreateAnd(Tmp3
,
193 ConstantInt::get(Type::getInt32Ty(Context
), 0xFF0000),
195 Tmp2
= Builder
.CreateAnd(Tmp2
,
196 ConstantInt::get(Type::getInt32Ty(Context
), 0xFF00),
198 Tmp4
= Builder
.CreateOr(Tmp4
, Tmp3
, "bswap.or1");
199 Tmp2
= Builder
.CreateOr(Tmp2
, Tmp1
, "bswap.or2");
200 V
= Builder
.CreateOr(Tmp4
, Tmp2
, "bswap.i32");
204 Value
*Tmp8
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 56),
206 Value
*Tmp7
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 40),
208 Value
*Tmp6
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 24),
210 Value
*Tmp5
= Builder
.CreateShl(V
, ConstantInt::get(V
->getType(), 8),
212 Value
* Tmp4
= Builder
.CreateLShr(V
, ConstantInt::get(V
->getType(), 8),
214 Value
* Tmp3
= Builder
.CreateLShr(V
,
215 ConstantInt::get(V
->getType(), 24),
217 Value
* Tmp2
= Builder
.CreateLShr(V
,
218 ConstantInt::get(V
->getType(), 40),
220 Value
* Tmp1
= Builder
.CreateLShr(V
,
221 ConstantInt::get(V
->getType(), 56),
223 Tmp7
= Builder
.CreateAnd(Tmp7
,
224 ConstantInt::get(Type::getInt64Ty(Context
),
225 0xFF000000000000ULL
),
227 Tmp6
= Builder
.CreateAnd(Tmp6
,
228 ConstantInt::get(Type::getInt64Ty(Context
),
231 Tmp5
= Builder
.CreateAnd(Tmp5
,
232 ConstantInt::get(Type::getInt64Ty(Context
),
235 Tmp4
= Builder
.CreateAnd(Tmp4
,
236 ConstantInt::get(Type::getInt64Ty(Context
),
239 Tmp3
= Builder
.CreateAnd(Tmp3
,
240 ConstantInt::get(Type::getInt64Ty(Context
),
243 Tmp2
= Builder
.CreateAnd(Tmp2
,
244 ConstantInt::get(Type::getInt64Ty(Context
),
247 Tmp8
= Builder
.CreateOr(Tmp8
, Tmp7
, "bswap.or1");
248 Tmp6
= Builder
.CreateOr(Tmp6
, Tmp5
, "bswap.or2");
249 Tmp4
= Builder
.CreateOr(Tmp4
, Tmp3
, "bswap.or3");
250 Tmp2
= Builder
.CreateOr(Tmp2
, Tmp1
, "bswap.or4");
251 Tmp8
= Builder
.CreateOr(Tmp8
, Tmp6
, "bswap.or5");
252 Tmp4
= Builder
.CreateOr(Tmp4
, Tmp2
, "bswap.or6");
253 V
= Builder
.CreateOr(Tmp8
, Tmp4
, "bswap.i64");
260 /// LowerCTPOP - Emit the code to lower ctpop of V before the specified
262 static Value
*LowerCTPOP(LLVMContext
&Context
, Value
*V
, Instruction
*IP
) {
263 assert(V
->getType()->isIntegerTy() && "Can't ctpop a non-integer type!");
265 static const uint64_t MaskValues
[6] = {
266 0x5555555555555555ULL
, 0x3333333333333333ULL
,
267 0x0F0F0F0F0F0F0F0FULL
, 0x00FF00FF00FF00FFULL
,
268 0x0000FFFF0000FFFFULL
, 0x00000000FFFFFFFFULL
271 IRBuilder
<> Builder(IP
->getParent(), IP
);
273 unsigned BitSize
= V
->getType()->getPrimitiveSizeInBits();
274 unsigned WordSize
= (BitSize
+ 63) / 64;
275 Value
*Count
= ConstantInt::get(V
->getType(), 0);
277 for (unsigned n
= 0; n
< WordSize
; ++n
) {
278 Value
*PartValue
= V
;
279 for (unsigned i
= 1, ct
= 0; i
< (BitSize
>64 ? 64 : BitSize
);
281 Value
*MaskCst
= ConstantInt::get(V
->getType(), MaskValues
[ct
]);
282 Value
*LHS
= Builder
.CreateAnd(PartValue
, MaskCst
, "cppop.and1");
283 Value
*VShift
= Builder
.CreateLShr(PartValue
,
284 ConstantInt::get(V
->getType(), i
),
286 Value
*RHS
= Builder
.CreateAnd(VShift
, MaskCst
, "cppop.and2");
287 PartValue
= Builder
.CreateAdd(LHS
, RHS
, "ctpop.step");
289 Count
= Builder
.CreateAdd(PartValue
, Count
, "ctpop.part");
291 V
= Builder
.CreateLShr(V
, ConstantInt::get(V
->getType(), 64),
300 /// LowerCTLZ - Emit the code to lower ctlz of V before the specified
302 static Value
*LowerCTLZ(LLVMContext
&Context
, Value
*V
, Instruction
*IP
) {
304 IRBuilder
<> Builder(IP
->getParent(), IP
);
306 unsigned BitSize
= V
->getType()->getPrimitiveSizeInBits();
307 for (unsigned i
= 1; i
< BitSize
; i
<<= 1) {
308 Value
*ShVal
= ConstantInt::get(V
->getType(), i
);
309 ShVal
= Builder
.CreateLShr(V
, ShVal
, "ctlz.sh");
310 V
= Builder
.CreateOr(V
, ShVal
, "ctlz.step");
313 V
= Builder
.CreateNot(V
);
314 return LowerCTPOP(Context
, V
, IP
);
317 static void ReplaceFPIntrinsicWithCall(CallInst
*CI
, const char *Fname
,
319 const char *LDname
) {
321 switch (CI
->getArgOperand(0)->getType()->getTypeID()) {
322 default: llvm_unreachable("Invalid type in intrinsic");
323 case Type::FloatTyID
:
324 ReplaceCallWith(Fname
, CI
, CS
.arg_begin(), CS
.arg_end(),
325 Type::getFloatTy(CI
->getContext()));
327 case Type::DoubleTyID
:
328 ReplaceCallWith(Dname
, CI
, CS
.arg_begin(), CS
.arg_end(),
329 Type::getDoubleTy(CI
->getContext()));
331 case Type::X86_FP80TyID
:
332 case Type::FP128TyID
:
333 case Type::PPC_FP128TyID
:
334 ReplaceCallWith(LDname
, CI
, CS
.arg_begin(), CS
.arg_end(),
335 CI
->getArgOperand(0)->getType());
340 void IntrinsicLowering::LowerIntrinsicCall(CallInst
*CI
) {
341 IRBuilder
<> Builder(CI
->getParent(), CI
);
342 LLVMContext
&Context
= CI
->getContext();
344 const Function
*Callee
= CI
->getCalledFunction();
345 assert(Callee
&& "Cannot lower an indirect call!");
348 switch (Callee
->getIntrinsicID()) {
349 case Intrinsic::not_intrinsic
:
350 report_fatal_error("Cannot lower a call to a non-intrinsic function '"+
351 Callee
->getName() + "'!");
353 report_fatal_error("Code generator does not support intrinsic function '"+
354 Callee
->getName()+"'!");
356 case Intrinsic::expect
: {
357 // Just replace __builtin_expect(exp, c) with EXP.
358 Value
*V
= CI
->getArgOperand(0);
359 CI
->replaceAllUsesWith(V
);
363 // The setjmp/longjmp intrinsics should only exist in the code if it was
364 // never optimized (ie, right out of the CFE), or if it has been hacked on
365 // by the lowerinvoke pass. In both cases, the right thing to do is to
366 // convert the call to an explicit setjmp or longjmp call.
367 case Intrinsic::setjmp
: {
368 Value
*V
= ReplaceCallWith("setjmp", CI
, CS
.arg_begin(), CS
.arg_end(),
369 Type::getInt32Ty(Context
));
370 if (!CI
->getType()->isVoidTy())
371 CI
->replaceAllUsesWith(V
);
374 case Intrinsic::sigsetjmp
:
375 if (!CI
->getType()->isVoidTy())
376 CI
->replaceAllUsesWith(Constant::getNullValue(CI
->getType()));
379 case Intrinsic::longjmp
: {
380 ReplaceCallWith("longjmp", CI
, CS
.arg_begin(), CS
.arg_end(),
381 Type::getVoidTy(Context
));
385 case Intrinsic::siglongjmp
: {
386 // Insert the call to abort
387 ReplaceCallWith("abort", CI
, CS
.arg_end(), CS
.arg_end(),
388 Type::getVoidTy(Context
));
391 case Intrinsic::ctpop
:
392 CI
->replaceAllUsesWith(LowerCTPOP(Context
, CI
->getArgOperand(0), CI
));
395 case Intrinsic::bswap
:
396 CI
->replaceAllUsesWith(LowerBSWAP(Context
, CI
->getArgOperand(0), CI
));
399 case Intrinsic::ctlz
:
400 CI
->replaceAllUsesWith(LowerCTLZ(Context
, CI
->getArgOperand(0), CI
));
403 case Intrinsic::cttz
: {
404 // cttz(x) -> ctpop(~X & (X-1))
405 Value
*Src
= CI
->getArgOperand(0);
406 Value
*NotSrc
= Builder
.CreateNot(Src
);
407 NotSrc
->setName(Src
->getName() + ".not");
408 Value
*SrcM1
= ConstantInt::get(Src
->getType(), 1);
409 SrcM1
= Builder
.CreateSub(Src
, SrcM1
);
410 Src
= LowerCTPOP(Context
, Builder
.CreateAnd(NotSrc
, SrcM1
), CI
);
411 CI
->replaceAllUsesWith(Src
);
415 case Intrinsic::stacksave
:
416 case Intrinsic::stackrestore
: {
418 errs() << "WARNING: this target does not support the llvm.stack"
419 << (Callee
->getIntrinsicID() == Intrinsic::stacksave
?
420 "save" : "restore") << " intrinsic.\n";
422 if (Callee
->getIntrinsicID() == Intrinsic::stacksave
)
423 CI
->replaceAllUsesWith(Constant::getNullValue(CI
->getType()));
427 case Intrinsic::returnaddress
:
428 case Intrinsic::frameaddress
:
429 errs() << "WARNING: this target does not support the llvm."
430 << (Callee
->getIntrinsicID() == Intrinsic::returnaddress
?
431 "return" : "frame") << "address intrinsic.\n";
432 CI
->replaceAllUsesWith(ConstantPointerNull::get(
433 cast
<PointerType
>(CI
->getType())));
436 case Intrinsic::prefetch
:
437 break; // Simply strip out prefetches on unsupported architectures
439 case Intrinsic::pcmarker
:
440 break; // Simply strip out pcmarker on unsupported architectures
441 case Intrinsic::readcyclecounter
: {
442 errs() << "WARNING: this target does not support the llvm.readcyclecoun"
443 << "ter intrinsic. It is being lowered to a constant 0\n";
444 CI
->replaceAllUsesWith(ConstantInt::get(Type::getInt64Ty(Context
), 0));
448 case Intrinsic::dbg_declare
:
449 break; // Simply strip out debugging intrinsics
451 case Intrinsic::eh_exception
:
452 case Intrinsic::eh_selector
:
453 CI
->replaceAllUsesWith(Constant::getNullValue(CI
->getType()));
456 case Intrinsic::eh_typeid_for
:
457 // Return something different to eh_selector.
458 CI
->replaceAllUsesWith(ConstantInt::get(CI
->getType(), 1));
461 case Intrinsic::var_annotation
:
462 break; // Strip out annotate intrinsic
464 case Intrinsic::memcpy
: {
465 const IntegerType
*IntPtr
= TD
.getIntPtrType(Context
);
466 Value
*Size
= Builder
.CreateIntCast(CI
->getArgOperand(2), IntPtr
,
467 /* isSigned */ false);
469 Ops
[0] = CI
->getArgOperand(0);
470 Ops
[1] = CI
->getArgOperand(1);
472 ReplaceCallWith("memcpy", CI
, Ops
, Ops
+3, CI
->getArgOperand(0)->getType());
475 case Intrinsic::memmove
: {
476 const IntegerType
*IntPtr
= TD
.getIntPtrType(Context
);
477 Value
*Size
= Builder
.CreateIntCast(CI
->getArgOperand(2), IntPtr
,
478 /* isSigned */ false);
480 Ops
[0] = CI
->getArgOperand(0);
481 Ops
[1] = CI
->getArgOperand(1);
483 ReplaceCallWith("memmove", CI
, Ops
, Ops
+3, CI
->getArgOperand(0)->getType());
486 case Intrinsic::memset
: {
487 const IntegerType
*IntPtr
= TD
.getIntPtrType(Context
);
488 Value
*Size
= Builder
.CreateIntCast(CI
->getArgOperand(2), IntPtr
,
489 /* isSigned */ false);
491 Ops
[0] = CI
->getArgOperand(0);
492 // Extend the amount to i32.
493 Ops
[1] = Builder
.CreateIntCast(CI
->getArgOperand(1),
494 Type::getInt32Ty(Context
),
495 /* isSigned */ false);
497 ReplaceCallWith("memset", CI
, Ops
, Ops
+3, CI
->getArgOperand(0)->getType());
500 case Intrinsic::sqrt
: {
501 ReplaceFPIntrinsicWithCall(CI
, "sqrtf", "sqrt", "sqrtl");
504 case Intrinsic::log
: {
505 ReplaceFPIntrinsicWithCall(CI
, "logf", "log", "logl");
508 case Intrinsic::log2
: {
509 ReplaceFPIntrinsicWithCall(CI
, "log2f", "log2", "log2l");
512 case Intrinsic::log10
: {
513 ReplaceFPIntrinsicWithCall(CI
, "log10f", "log10", "log10l");
516 case Intrinsic::exp
: {
517 ReplaceFPIntrinsicWithCall(CI
, "expf", "exp", "expl");
520 case Intrinsic::exp2
: {
521 ReplaceFPIntrinsicWithCall(CI
, "exp2f", "exp2", "exp2l");
524 case Intrinsic::pow
: {
525 ReplaceFPIntrinsicWithCall(CI
, "powf", "pow", "powl");
528 case Intrinsic::flt_rounds
:
529 // Lower to "round to the nearest"
530 if (!CI
->getType()->isVoidTy())
531 CI
->replaceAllUsesWith(ConstantInt::get(CI
->getType(), 1));
533 case Intrinsic::invariant_start
:
534 case Intrinsic::lifetime_start
:
535 // Discard region information.
536 CI
->replaceAllUsesWith(UndefValue::get(CI
->getType()));
538 case Intrinsic::invariant_end
:
539 case Intrinsic::lifetime_end
:
540 // Discard region information.
544 assert(CI
->use_empty() &&
545 "Lowering should have eliminated any uses of the intrinsic call!");
546 CI
->eraseFromParent();
549 bool IntrinsicLowering::LowerToByteSwap(CallInst
*CI
) {
550 // Verify this is a simple bswap.
551 if (CI
->getNumArgOperands() != 1 ||
552 CI
->getType() != CI
->getArgOperand(0)->getType() ||
553 !CI
->getType()->isIntegerTy())
556 const IntegerType
*Ty
= dyn_cast
<IntegerType
>(CI
->getType());
560 // Okay, we can do this xform, do so now.
561 const Type
*Tys
[] = { Ty
};
562 Module
*M
= CI
->getParent()->getParent()->getParent();
563 Constant
*Int
= Intrinsic::getDeclaration(M
, Intrinsic::bswap
, Tys
, 1);
565 Value
*Op
= CI
->getArgOperand(0);
566 Op
= CallInst::Create(Int
, Op
, CI
->getName(), CI
);
568 CI
->replaceAllUsesWith(Op
);
569 CI
->eraseFromParent();