1 //===- ValueEnumerator.cpp - Number values and types for bitcode writer ---===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file implements the ValueEnumerator class.
11 //===----------------------------------------------------------------------===//
13 #include "ValueEnumerator.h"
14 #include "llvm/ADT/SmallVector.h"
15 #include "llvm/Config/llvm-config.h"
16 #include "llvm/IR/Argument.h"
17 #include "llvm/IR/BasicBlock.h"
18 #include "llvm/IR/Constant.h"
19 #include "llvm/IR/DebugInfoMetadata.h"
20 #include "llvm/IR/DerivedTypes.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/GlobalAlias.h"
23 #include "llvm/IR/GlobalIFunc.h"
24 #include "llvm/IR/GlobalObject.h"
25 #include "llvm/IR/GlobalValue.h"
26 #include "llvm/IR/GlobalVariable.h"
27 #include "llvm/IR/Instruction.h"
28 #include "llvm/IR/Instructions.h"
29 #include "llvm/IR/Metadata.h"
30 #include "llvm/IR/Module.h"
31 #include "llvm/IR/Operator.h"
32 #include "llvm/IR/Type.h"
33 #include "llvm/IR/Use.h"
34 #include "llvm/IR/User.h"
35 #include "llvm/IR/Value.h"
36 #include "llvm/IR/ValueSymbolTable.h"
37 #include "llvm/Support/Casting.h"
38 #include "llvm/Support/Compiler.h"
39 #include "llvm/Support/Debug.h"
40 #include "llvm/Support/MathExtras.h"
41 #include "llvm/Support/raw_ostream.h"
52 DenseMap
<const Value
*, std::pair
<unsigned, bool>> IDs
;
53 unsigned LastGlobalValueID
= 0;
57 bool isGlobalValue(unsigned ID
) const {
58 return ID
<= LastGlobalValueID
;
61 unsigned size() const { return IDs
.size(); }
62 std::pair
<unsigned, bool> &operator[](const Value
*V
) { return IDs
[V
]; }
64 std::pair
<unsigned, bool> lookup(const Value
*V
) const {
68 void index(const Value
*V
) {
69 // Explicitly sequence get-size and insert-value operations to avoid UB.
70 unsigned ID
= IDs
.size() + 1;
75 } // end anonymous namespace
77 static void orderValue(const Value
*V
, OrderMap
&OM
) {
78 if (OM
.lookup(V
).first
)
81 if (const Constant
*C
= dyn_cast
<Constant
>(V
)) {
82 if (C
->getNumOperands()) {
83 for (const Value
*Op
: C
->operands())
84 if (!isa
<BasicBlock
>(Op
) && !isa
<GlobalValue
>(Op
))
86 if (auto *CE
= dyn_cast
<ConstantExpr
>(C
))
87 if (CE
->getOpcode() == Instruction::ShuffleVector
)
88 orderValue(CE
->getShuffleMaskForBitcode(), OM
);
92 // Note: we cannot cache this lookup above, since inserting into the map
93 // changes the map's size, and thus affects the other IDs.
97 static OrderMap
orderModule(const Module
&M
) {
98 // This needs to match the order used by ValueEnumerator::ValueEnumerator()
99 // and ValueEnumerator::incorporateFunction().
102 // Initializers of GlobalValues are processed in
103 // BitcodeReader::ResolveGlobalAndAliasInits(). Match the order there rather
104 // than ValueEnumerator, and match the code in predictValueUseListOrderImpl()
105 // by giving IDs in reverse order.
107 // Since GlobalValues never reference each other directly (just through
108 // initializers), their relative IDs only matter for determining order of
109 // uses in their initializers.
110 for (const GlobalVariable
&G
: reverse(M
.globals()))
112 for (const GlobalAlias
&A
: reverse(M
.aliases()))
114 for (const GlobalIFunc
&I
: reverse(M
.ifuncs()))
116 for (const Function
&F
: reverse(M
))
118 OM
.LastGlobalValueID
= OM
.size();
120 auto orderConstantValue
= [&OM
](const Value
*V
) {
121 if (isa
<Constant
>(V
) || isa
<InlineAsm
>(V
))
125 for (const Function
&F
: M
) {
126 if (F
.isDeclaration())
128 // Here we need to match the union of ValueEnumerator::incorporateFunction()
129 // and WriteFunction(). Basic blocks are implicitly declared before
130 // anything else (by declaring their size).
131 for (const BasicBlock
&BB
: F
)
134 // Metadata used by instructions is decoded before the actual instructions,
135 // so visit any constants used by it beforehand.
136 for (const BasicBlock
&BB
: F
)
137 for (const Instruction
&I
: BB
) {
138 auto OrderConstantFromMetadata
= [&](Metadata
*MD
) {
139 if (const auto *VAM
= dyn_cast
<ValueAsMetadata
>(MD
)) {
140 orderConstantValue(VAM
->getValue());
141 } else if (const auto *AL
= dyn_cast
<DIArgList
>(MD
)) {
142 for (const auto *VAM
: AL
->getArgs())
143 orderConstantValue(VAM
->getValue());
147 for (DbgVariableRecord
&DVR
: filterDbgVars(I
.getDbgRecordRange())) {
148 OrderConstantFromMetadata(DVR
.getRawLocation());
149 if (DVR
.isDbgAssign())
150 OrderConstantFromMetadata(DVR
.getRawAddress());
153 for (const Value
*V
: I
.operands()) {
154 if (const auto *MAV
= dyn_cast
<MetadataAsValue
>(V
))
155 OrderConstantFromMetadata(MAV
->getMetadata());
159 for (const Argument
&A
: F
.args())
161 for (const BasicBlock
&BB
: F
)
162 for (const Instruction
&I
: BB
) {
163 for (const Value
*Op
: I
.operands())
164 orderConstantValue(Op
);
165 if (auto *SVI
= dyn_cast
<ShuffleVectorInst
>(&I
))
166 orderValue(SVI
->getShuffleMaskForBitcode(), OM
);
173 static void predictValueUseListOrderImpl(const Value
*V
, const Function
*F
,
174 unsigned ID
, const OrderMap
&OM
,
175 UseListOrderStack
&Stack
) {
176 // Predict use-list order for this one.
177 using Entry
= std::pair
<const Use
*, unsigned>;
178 SmallVector
<Entry
, 64> List
;
179 for (const Use
&U
: V
->uses())
180 // Check if this user will be serialized.
181 if (OM
.lookup(U
.getUser()).first
)
182 List
.push_back(std::make_pair(&U
, List
.size()));
185 // We may have lost some users.
188 bool IsGlobalValue
= OM
.isGlobalValue(ID
);
189 llvm::sort(List
, [&](const Entry
&L
, const Entry
&R
) {
190 const Use
*LU
= L
.first
;
191 const Use
*RU
= R
.first
;
195 auto LID
= OM
.lookup(LU
->getUser()).first
;
196 auto RID
= OM
.lookup(RU
->getUser()).first
;
198 // If ID is 4, then expect: 7 6 5 1 2 3.
201 if (!IsGlobalValue
) // GlobalValue uses don't get reversed.
207 if (!IsGlobalValue
) // GlobalValue uses don't get reversed.
212 // LID and RID are equal, so we have different operands of the same user.
213 // Assume operands are added in order for all instructions.
215 if (!IsGlobalValue
) // GlobalValue uses don't get reversed.
216 return LU
->getOperandNo() < RU
->getOperandNo();
217 return LU
->getOperandNo() > RU
->getOperandNo();
220 if (llvm::is_sorted(List
, llvm::less_second()))
221 // Order is already correct.
224 // Store the shuffle.
225 Stack
.emplace_back(V
, F
, List
.size());
226 assert(List
.size() == Stack
.back().Shuffle
.size() && "Wrong size");
227 for (size_t I
= 0, E
= List
.size(); I
!= E
; ++I
)
228 Stack
.back().Shuffle
[I
] = List
[I
].second
;
231 static void predictValueUseListOrder(const Value
*V
, const Function
*F
,
232 OrderMap
&OM
, UseListOrderStack
&Stack
) {
233 auto &IDPair
= OM
[V
];
234 assert(IDPair
.first
&& "Unmapped value");
236 // Already predicted.
239 // Do the actual prediction.
240 IDPair
.second
= true;
241 if (!V
->use_empty() && std::next(V
->use_begin()) != V
->use_end())
242 predictValueUseListOrderImpl(V
, F
, IDPair
.first
, OM
, Stack
);
244 // Recursive descent into constants.
245 if (const Constant
*C
= dyn_cast
<Constant
>(V
)) {
246 if (C
->getNumOperands()) { // Visit GlobalValues.
247 for (const Value
*Op
: C
->operands())
248 if (isa
<Constant
>(Op
)) // Visit GlobalValues.
249 predictValueUseListOrder(Op
, F
, OM
, Stack
);
250 if (auto *CE
= dyn_cast
<ConstantExpr
>(C
))
251 if (CE
->getOpcode() == Instruction::ShuffleVector
)
252 predictValueUseListOrder(CE
->getShuffleMaskForBitcode(), F
, OM
,
258 static UseListOrderStack
predictUseListOrder(const Module
&M
) {
259 OrderMap OM
= orderModule(M
);
261 // Use-list orders need to be serialized after all the users have been added
262 // to a value, or else the shuffles will be incomplete. Store them per
263 // function in a stack.
265 // Aside from function order, the order of values doesn't matter much here.
266 UseListOrderStack Stack
;
268 // We want to visit the functions backward now so we can list function-local
269 // constants in the last Function they're used in. Module-level constants
270 // have already been visited above.
271 for (const Function
&F
: llvm::reverse(M
)) {
272 auto PredictValueOrderFromMetadata
= [&](Metadata
*MD
) {
273 if (const auto *VAM
= dyn_cast
<ValueAsMetadata
>(MD
)) {
274 predictValueUseListOrder(VAM
->getValue(), &F
, OM
, Stack
);
275 } else if (const auto *AL
= dyn_cast
<DIArgList
>(MD
)) {
276 for (const auto *VAM
: AL
->getArgs())
277 predictValueUseListOrder(VAM
->getValue(), &F
, OM
, Stack
);
280 if (F
.isDeclaration())
282 for (const BasicBlock
&BB
: F
)
283 predictValueUseListOrder(&BB
, &F
, OM
, Stack
);
284 for (const Argument
&A
: F
.args())
285 predictValueUseListOrder(&A
, &F
, OM
, Stack
);
286 for (const BasicBlock
&BB
: F
) {
287 for (const Instruction
&I
: BB
) {
288 for (DbgVariableRecord
&DVR
: filterDbgVars(I
.getDbgRecordRange())) {
289 PredictValueOrderFromMetadata(DVR
.getRawLocation());
290 if (DVR
.isDbgAssign())
291 PredictValueOrderFromMetadata(DVR
.getRawAddress());
293 for (const Value
*Op
: I
.operands()) {
294 if (isa
<Constant
>(*Op
) || isa
<InlineAsm
>(*Op
)) // Visit GlobalValues.
295 predictValueUseListOrder(Op
, &F
, OM
, Stack
);
296 if (const auto *MAV
= dyn_cast
<MetadataAsValue
>(Op
))
297 PredictValueOrderFromMetadata(MAV
->getMetadata());
299 if (auto *SVI
= dyn_cast
<ShuffleVectorInst
>(&I
))
300 predictValueUseListOrder(SVI
->getShuffleMaskForBitcode(), &F
, OM
,
302 predictValueUseListOrder(&I
, &F
, OM
, Stack
);
307 // Visit globals last, since the module-level use-list block will be seen
308 // before the function bodies are processed.
309 for (const GlobalVariable
&G
: M
.globals())
310 predictValueUseListOrder(&G
, nullptr, OM
, Stack
);
311 for (const Function
&F
: M
)
312 predictValueUseListOrder(&F
, nullptr, OM
, Stack
);
313 for (const GlobalAlias
&A
: M
.aliases())
314 predictValueUseListOrder(&A
, nullptr, OM
, Stack
);
315 for (const GlobalIFunc
&I
: M
.ifuncs())
316 predictValueUseListOrder(&I
, nullptr, OM
, Stack
);
317 for (const GlobalVariable
&G
: M
.globals())
318 if (G
.hasInitializer())
319 predictValueUseListOrder(G
.getInitializer(), nullptr, OM
, Stack
);
320 for (const GlobalAlias
&A
: M
.aliases())
321 predictValueUseListOrder(A
.getAliasee(), nullptr, OM
, Stack
);
322 for (const GlobalIFunc
&I
: M
.ifuncs())
323 predictValueUseListOrder(I
.getResolver(), nullptr, OM
, Stack
);
324 for (const Function
&F
: M
) {
325 for (const Use
&U
: F
.operands())
326 predictValueUseListOrder(U
.get(), nullptr, OM
, Stack
);
332 static bool isIntOrIntVectorValue(const std::pair
<const Value
*, unsigned> &V
) {
333 return V
.first
->getType()->isIntOrIntVectorTy();
336 ValueEnumerator::ValueEnumerator(const Module
&M
,
337 bool ShouldPreserveUseListOrder
)
338 : ShouldPreserveUseListOrder(ShouldPreserveUseListOrder
) {
339 if (ShouldPreserveUseListOrder
)
340 UseListOrders
= predictUseListOrder(M
);
342 // Enumerate the global variables.
343 for (const GlobalVariable
&GV
: M
.globals()) {
345 EnumerateType(GV
.getValueType());
348 // Enumerate the functions.
349 for (const Function
& F
: M
) {
351 EnumerateType(F
.getValueType());
352 EnumerateAttributes(F
.getAttributes());
355 // Enumerate the aliases.
356 for (const GlobalAlias
&GA
: M
.aliases()) {
358 EnumerateType(GA
.getValueType());
361 // Enumerate the ifuncs.
362 for (const GlobalIFunc
&GIF
: M
.ifuncs()) {
363 EnumerateValue(&GIF
);
364 EnumerateType(GIF
.getValueType());
367 // Remember what is the cutoff between globalvalue's and other constants.
368 unsigned FirstConstant
= Values
.size();
370 // Enumerate the global variable initializers and attributes.
371 for (const GlobalVariable
&GV
: M
.globals()) {
372 if (GV
.hasInitializer())
373 EnumerateValue(GV
.getInitializer());
374 if (GV
.hasAttributes())
375 EnumerateAttributes(GV
.getAttributesAsList(AttributeList::FunctionIndex
));
378 // Enumerate the aliasees.
379 for (const GlobalAlias
&GA
: M
.aliases())
380 EnumerateValue(GA
.getAliasee());
382 // Enumerate the ifunc resolvers.
383 for (const GlobalIFunc
&GIF
: M
.ifuncs())
384 EnumerateValue(GIF
.getResolver());
386 // Enumerate any optional Function data.
387 for (const Function
&F
: M
)
388 for (const Use
&U
: F
.operands())
389 EnumerateValue(U
.get());
391 // Enumerate the metadata type.
393 // TODO: Move this to ValueEnumerator::EnumerateOperandType() once bitcode
394 // only encodes the metadata type when it's used as a value.
395 EnumerateType(Type::getMetadataTy(M
.getContext()));
397 // Insert constants and metadata that are named at module level into the slot
398 // pool so that the module symbol table can refer to them...
399 EnumerateValueSymbolTable(M
.getValueSymbolTable());
400 EnumerateNamedMetadata(M
);
402 SmallVector
<std::pair
<unsigned, MDNode
*>, 8> MDs
;
403 for (const GlobalVariable
&GV
: M
.globals()) {
405 GV
.getAllMetadata(MDs
);
406 for (const auto &I
: MDs
)
407 // FIXME: Pass GV to EnumerateMetadata and arrange for the bitcode writer
408 // to write metadata to the global variable's own metadata block
410 EnumerateMetadata(nullptr, I
.second
);
413 // Enumerate types used by function bodies and argument lists.
414 for (const Function
&F
: M
) {
415 for (const Argument
&A
: F
.args())
416 EnumerateType(A
.getType());
418 // Enumerate metadata attached to this function.
420 F
.getAllMetadata(MDs
);
421 for (const auto &I
: MDs
)
422 EnumerateMetadata(F
.isDeclaration() ? nullptr : &F
, I
.second
);
424 for (const BasicBlock
&BB
: F
)
425 for (const Instruction
&I
: BB
) {
426 // Local metadata is enumerated during function-incorporation, but
427 // any ConstantAsMetadata arguments in a DIArgList should be examined
429 auto EnumerateNonLocalValuesFromMetadata
= [&](Metadata
*MD
) {
430 assert(MD
&& "Metadata unexpectedly null");
431 if (const auto *AL
= dyn_cast
<DIArgList
>(MD
)) {
432 for (const auto *VAM
: AL
->getArgs()) {
433 if (isa
<ConstantAsMetadata
>(VAM
))
434 EnumerateMetadata(&F
, VAM
);
439 if (!isa
<LocalAsMetadata
>(MD
))
440 EnumerateMetadata(&F
, MD
);
443 for (DbgRecord
&DR
: I
.getDbgRecordRange()) {
444 if (DbgLabelRecord
*DLR
= dyn_cast
<DbgLabelRecord
>(&DR
)) {
445 EnumerateMetadata(&F
, DLR
->getLabel());
446 EnumerateMetadata(&F
, &*DLR
->getDebugLoc());
449 // Enumerate non-local location metadata.
450 DbgVariableRecord
&DVR
= cast
<DbgVariableRecord
>(DR
);
451 EnumerateNonLocalValuesFromMetadata(DVR
.getRawLocation());
452 EnumerateMetadata(&F
, DVR
.getExpression());
453 EnumerateMetadata(&F
, DVR
.getVariable());
454 EnumerateMetadata(&F
, &*DVR
.getDebugLoc());
455 if (DVR
.isDbgAssign()) {
456 EnumerateNonLocalValuesFromMetadata(DVR
.getRawAddress());
457 EnumerateMetadata(&F
, DVR
.getAssignID());
458 EnumerateMetadata(&F
, DVR
.getAddressExpression());
461 for (const Use
&Op
: I
.operands()) {
462 auto *MD
= dyn_cast
<MetadataAsValue
>(&Op
);
464 EnumerateOperandType(Op
);
468 EnumerateNonLocalValuesFromMetadata(MD
->getMetadata());
470 if (auto *SVI
= dyn_cast
<ShuffleVectorInst
>(&I
))
471 EnumerateType(SVI
->getShuffleMaskForBitcode()->getType());
472 if (auto *GEP
= dyn_cast
<GetElementPtrInst
>(&I
))
473 EnumerateType(GEP
->getSourceElementType());
474 if (auto *AI
= dyn_cast
<AllocaInst
>(&I
))
475 EnumerateType(AI
->getAllocatedType());
476 EnumerateType(I
.getType());
477 if (const auto *Call
= dyn_cast
<CallBase
>(&I
)) {
478 EnumerateAttributes(Call
->getAttributes());
479 EnumerateType(Call
->getFunctionType());
482 // Enumerate metadata attached with this instruction.
484 I
.getAllMetadataOtherThanDebugLoc(MDs
);
485 for (unsigned i
= 0, e
= MDs
.size(); i
!= e
; ++i
)
486 EnumerateMetadata(&F
, MDs
[i
].second
);
488 // Don't enumerate the location directly -- it has a special record
489 // type -- but enumerate its operands.
490 if (DILocation
*L
= I
.getDebugLoc())
491 for (const Metadata
*Op
: L
->operands())
492 EnumerateMetadata(&F
, Op
);
496 // Optimize constant ordering.
497 OptimizeConstants(FirstConstant
, Values
.size());
499 // Organize metadata ordering.
503 unsigned ValueEnumerator::getInstructionID(const Instruction
*Inst
) const {
504 InstructionMapType::const_iterator I
= InstructionMap
.find(Inst
);
505 assert(I
!= InstructionMap
.end() && "Instruction is not mapped!");
509 unsigned ValueEnumerator::getComdatID(const Comdat
*C
) const {
510 unsigned ComdatID
= Comdats
.idFor(C
);
511 assert(ComdatID
&& "Comdat not found!");
515 void ValueEnumerator::setInstructionID(const Instruction
*I
) {
516 InstructionMap
[I
] = InstructionCount
++;
519 unsigned ValueEnumerator::getValueID(const Value
*V
) const {
520 if (auto *MD
= dyn_cast
<MetadataAsValue
>(V
))
521 return getMetadataID(MD
->getMetadata());
523 ValueMapType::const_iterator I
= ValueMap
.find(V
);
524 assert(I
!= ValueMap
.end() && "Value not in slotcalculator!");
528 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
529 LLVM_DUMP_METHOD
void ValueEnumerator::dump() const {
530 print(dbgs(), ValueMap
, "Default");
532 print(dbgs(), MetadataMap
, "MetaData");
537 void ValueEnumerator::print(raw_ostream
&OS
, const ValueMapType
&Map
,
538 const char *Name
) const {
539 OS
<< "Map Name: " << Name
<< "\n";
540 OS
<< "Size: " << Map
.size() << "\n";
541 for (const auto &I
: Map
) {
542 const Value
*V
= I
.first
;
544 OS
<< "Value: " << V
->getName();
546 OS
<< "Value: [null]\n";
550 OS
<< " Uses(" << V
->getNumUses() << "):";
551 for (const Use
&U
: V
->uses()) {
552 if (&U
!= &*V
->use_begin())
555 OS
<< " " << U
->getName();
564 void ValueEnumerator::print(raw_ostream
&OS
, const MetadataMapType
&Map
,
565 const char *Name
) const {
566 OS
<< "Map Name: " << Name
<< "\n";
567 OS
<< "Size: " << Map
.size() << "\n";
568 for (const auto &I
: Map
) {
569 const Metadata
*MD
= I
.first
;
570 OS
<< "Metadata: slot = " << I
.second
.ID
<< "\n";
571 OS
<< "Metadata: function = " << I
.second
.F
<< "\n";
577 /// OptimizeConstants - Reorder constant pool for denser encoding.
578 void ValueEnumerator::OptimizeConstants(unsigned CstStart
, unsigned CstEnd
) {
579 if (CstStart
== CstEnd
|| CstStart
+1 == CstEnd
) return;
581 if (ShouldPreserveUseListOrder
)
582 // Optimizing constants makes the use-list order difficult to predict.
583 // Disable it for now when trying to preserve the order.
586 std::stable_sort(Values
.begin() + CstStart
, Values
.begin() + CstEnd
,
587 [this](const std::pair
<const Value
*, unsigned> &LHS
,
588 const std::pair
<const Value
*, unsigned> &RHS
) {
590 if (LHS
.first
->getType() != RHS
.first
->getType())
591 return getTypeID(LHS
.first
->getType()) < getTypeID(RHS
.first
->getType());
592 // Then by frequency.
593 return LHS
.second
> RHS
.second
;
596 // Ensure that integer and vector of integer constants are at the start of the
597 // constant pool. This is important so that GEP structure indices come before
598 // gep constant exprs.
599 std::stable_partition(Values
.begin() + CstStart
, Values
.begin() + CstEnd
,
600 isIntOrIntVectorValue
);
602 // Rebuild the modified portion of ValueMap.
603 for (; CstStart
!= CstEnd
; ++CstStart
)
604 ValueMap
[Values
[CstStart
].first
] = CstStart
+1;
607 /// EnumerateValueSymbolTable - Insert all of the values in the specified symbol
608 /// table into the values table.
609 void ValueEnumerator::EnumerateValueSymbolTable(const ValueSymbolTable
&VST
) {
610 for (ValueSymbolTable::const_iterator VI
= VST
.begin(), VE
= VST
.end();
612 EnumerateValue(VI
->getValue());
615 /// Insert all of the values referenced by named metadata in the specified
617 void ValueEnumerator::EnumerateNamedMetadata(const Module
&M
) {
618 for (const auto &I
: M
.named_metadata())
619 EnumerateNamedMDNode(&I
);
622 void ValueEnumerator::EnumerateNamedMDNode(const NamedMDNode
*MD
) {
623 for (const MDNode
*N
: MD
->operands())
624 EnumerateMetadata(nullptr, N
);
627 unsigned ValueEnumerator::getMetadataFunctionID(const Function
*F
) const {
628 return F
? getValueID(F
) + 1 : 0;
631 void ValueEnumerator::EnumerateMetadata(const Function
*F
, const Metadata
*MD
) {
632 EnumerateMetadata(getMetadataFunctionID(F
), MD
);
635 void ValueEnumerator::EnumerateFunctionLocalMetadata(
636 const Function
&F
, const LocalAsMetadata
*Local
) {
637 EnumerateFunctionLocalMetadata(getMetadataFunctionID(&F
), Local
);
640 void ValueEnumerator::EnumerateFunctionLocalListMetadata(
641 const Function
&F
, const DIArgList
*ArgList
) {
642 EnumerateFunctionLocalListMetadata(getMetadataFunctionID(&F
), ArgList
);
645 void ValueEnumerator::dropFunctionFromMetadata(
646 MetadataMapType::value_type
&FirstMD
) {
647 SmallVector
<const MDNode
*, 64> Worklist
;
648 auto push
= [&Worklist
](MetadataMapType::value_type
&MD
) {
649 auto &Entry
= MD
.second
;
651 // Nothing to do if this metadata isn't tagged.
655 // Drop the function tag.
658 // If this is has an ID and is an MDNode, then its operands have entries as
659 // well. We need to drop the function from them too.
661 if (auto *N
= dyn_cast
<MDNode
>(MD
.first
))
662 Worklist
.push_back(N
);
665 while (!Worklist
.empty())
666 for (const Metadata
*Op
: Worklist
.pop_back_val()->operands()) {
669 auto MD
= MetadataMap
.find(Op
);
670 if (MD
!= MetadataMap
.end())
675 void ValueEnumerator::EnumerateMetadata(unsigned F
, const Metadata
*MD
) {
676 // It's vital for reader efficiency that uniqued subgraphs are done in
677 // post-order; it's expensive when their operands have forward references.
678 // If a distinct node is referenced from a uniqued node, it'll be delayed
679 // until the uniqued subgraph has been completely traversed.
680 SmallVector
<const MDNode
*, 32> DelayedDistinctNodes
;
682 // Start by enumerating MD, and then work through its transitive operands in
683 // post-order. This requires a depth-first search.
684 SmallVector
<std::pair
<const MDNode
*, MDNode::op_iterator
>, 32> Worklist
;
685 if (const MDNode
*N
= enumerateMetadataImpl(F
, MD
))
686 Worklist
.push_back(std::make_pair(N
, N
->op_begin()));
688 while (!Worklist
.empty()) {
689 const MDNode
*N
= Worklist
.back().first
;
691 // Enumerate operands until we hit a new node. We need to traverse these
692 // nodes' operands before visiting the rest of N's operands.
693 MDNode::op_iterator I
= std::find_if(
694 Worklist
.back().second
, N
->op_end(),
695 [&](const Metadata
*MD
) { return enumerateMetadataImpl(F
, MD
); });
696 if (I
!= N
->op_end()) {
697 auto *Op
= cast
<MDNode
>(*I
);
698 Worklist
.back().second
= ++I
;
700 // Delay traversing Op if it's a distinct node and N is uniqued.
701 if (Op
->isDistinct() && !N
->isDistinct())
702 DelayedDistinctNodes
.push_back(Op
);
704 Worklist
.push_back(std::make_pair(Op
, Op
->op_begin()));
708 // All the operands have been visited. Now assign an ID.
711 MetadataMap
[N
].ID
= MDs
.size();
713 // Flush out any delayed distinct nodes; these are all the distinct nodes
714 // that are leaves in last uniqued subgraph.
715 if (Worklist
.empty() || Worklist
.back().first
->isDistinct()) {
716 for (const MDNode
*N
: DelayedDistinctNodes
)
717 Worklist
.push_back(std::make_pair(N
, N
->op_begin()));
718 DelayedDistinctNodes
.clear();
723 const MDNode
*ValueEnumerator::enumerateMetadataImpl(unsigned F
, const Metadata
*MD
) {
728 (isa
<MDNode
>(MD
) || isa
<MDString
>(MD
) || isa
<ConstantAsMetadata
>(MD
)) &&
729 "Invalid metadata kind");
731 auto Insertion
= MetadataMap
.insert(std::make_pair(MD
, MDIndex(F
)));
732 MDIndex
&Entry
= Insertion
.first
->second
;
733 if (!Insertion
.second
) {
734 // Already mapped. If F doesn't match the function tag, drop it.
735 if (Entry
.hasDifferentFunction(F
))
736 dropFunctionFromMetadata(*Insertion
.first
);
740 // Don't assign IDs to metadata nodes.
741 if (auto *N
= dyn_cast
<MDNode
>(MD
))
744 // Save the metadata.
746 Entry
.ID
= MDs
.size();
748 // Enumerate the constant, if any.
749 if (auto *C
= dyn_cast
<ConstantAsMetadata
>(MD
))
750 EnumerateValue(C
->getValue());
755 /// EnumerateFunctionLocalMetadata - Incorporate function-local metadata
756 /// information reachable from the metadata.
757 void ValueEnumerator::EnumerateFunctionLocalMetadata(
758 unsigned F
, const LocalAsMetadata
*Local
) {
759 assert(F
&& "Expected a function");
761 // Check to see if it's already in!
762 MDIndex
&Index
= MetadataMap
[Local
];
764 assert(Index
.F
== F
&& "Expected the same function");
768 MDs
.push_back(Local
);
770 Index
.ID
= MDs
.size();
772 EnumerateValue(Local
->getValue());
775 /// EnumerateFunctionLocalListMetadata - Incorporate function-local metadata
776 /// information reachable from the metadata.
777 void ValueEnumerator::EnumerateFunctionLocalListMetadata(
778 unsigned F
, const DIArgList
*ArgList
) {
779 assert(F
&& "Expected a function");
781 // Check to see if it's already in!
782 MDIndex
&Index
= MetadataMap
[ArgList
];
784 assert(Index
.F
== F
&& "Expected the same function");
788 for (ValueAsMetadata
*VAM
: ArgList
->getArgs()) {
789 if (isa
<LocalAsMetadata
>(VAM
)) {
790 assert(MetadataMap
.count(VAM
) &&
791 "LocalAsMetadata should be enumerated before DIArgList");
792 assert(MetadataMap
[VAM
].F
== F
&&
793 "Expected LocalAsMetadata in the same function");
795 assert(isa
<ConstantAsMetadata
>(VAM
) &&
796 "Expected LocalAsMetadata or ConstantAsMetadata");
797 assert(ValueMap
.count(VAM
->getValue()) &&
798 "Constant should be enumerated beforeDIArgList");
799 EnumerateMetadata(F
, VAM
);
803 MDs
.push_back(ArgList
);
805 Index
.ID
= MDs
.size();
808 static unsigned getMetadataTypeOrder(const Metadata
*MD
) {
809 // Strings are emitted in bulk and must come first.
810 if (isa
<MDString
>(MD
))
813 // ConstantAsMetadata doesn't reference anything. We may as well shuffle it
814 // to the front since we can detect it.
815 auto *N
= dyn_cast
<MDNode
>(MD
);
819 // The reader is fast forward references for distinct node operands, but slow
820 // when uniqued operands are unresolved.
821 return N
->isDistinct() ? 2 : 3;
824 void ValueEnumerator::organizeMetadata() {
825 assert(MetadataMap
.size() == MDs
.size() &&
826 "Metadata map and vector out of sync");
831 // Copy out the index information from MetadataMap in order to choose a new
833 SmallVector
<MDIndex
, 64> Order
;
834 Order
.reserve(MetadataMap
.size());
835 for (const Metadata
*MD
: MDs
)
836 Order
.push_back(MetadataMap
.lookup(MD
));
839 // - by function, then
840 // - by isa<MDString>
841 // and then sort by the original/current ID. Since the IDs are guaranteed to
842 // be unique, the result of llvm::sort will be deterministic. There's no need
843 // for std::stable_sort.
844 llvm::sort(Order
, [this](MDIndex LHS
, MDIndex RHS
) {
845 return std::make_tuple(LHS
.F
, getMetadataTypeOrder(LHS
.get(MDs
)), LHS
.ID
) <
846 std::make_tuple(RHS
.F
, getMetadataTypeOrder(RHS
.get(MDs
)), RHS
.ID
);
849 // Rebuild MDs, index the metadata ranges for each function in FunctionMDs,
850 // and fix up MetadataMap.
851 std::vector
<const Metadata
*> OldMDs
;
853 MDs
.reserve(OldMDs
.size());
854 for (unsigned I
= 0, E
= Order
.size(); I
!= E
&& !Order
[I
].F
; ++I
) {
855 auto *MD
= Order
[I
].get(OldMDs
);
857 MetadataMap
[MD
].ID
= I
+ 1;
858 if (isa
<MDString
>(MD
))
862 // Return early if there's nothing for the functions.
863 if (MDs
.size() == Order
.size())
866 // Build the function metadata ranges.
868 FunctionMDs
.reserve(OldMDs
.size());
870 for (unsigned I
= MDs
.size(), E
= Order
.size(), ID
= MDs
.size(); I
!= E
;
872 unsigned F
= Order
[I
].F
;
875 } else if (PrevF
!= F
) {
876 R
.Last
= FunctionMDs
.size();
877 std::swap(R
, FunctionMDInfo
[PrevF
]);
878 R
.First
= FunctionMDs
.size();
884 auto *MD
= Order
[I
].get(OldMDs
);
885 FunctionMDs
.push_back(MD
);
886 MetadataMap
[MD
].ID
= ++ID
;
887 if (isa
<MDString
>(MD
))
890 R
.Last
= FunctionMDs
.size();
891 FunctionMDInfo
[PrevF
] = R
;
894 void ValueEnumerator::incorporateFunctionMetadata(const Function
&F
) {
895 NumModuleMDs
= MDs
.size();
897 auto R
= FunctionMDInfo
.lookup(getValueID(&F
) + 1);
898 NumMDStrings
= R
.NumStrings
;
899 MDs
.insert(MDs
.end(), FunctionMDs
.begin() + R
.First
,
900 FunctionMDs
.begin() + R
.Last
);
903 void ValueEnumerator::EnumerateValue(const Value
*V
) {
904 assert(!V
->getType()->isVoidTy() && "Can't insert void values!");
905 assert(!isa
<MetadataAsValue
>(V
) && "EnumerateValue doesn't handle Metadata!");
907 // Check to see if it's already in!
908 unsigned &ValueID
= ValueMap
[V
];
910 // Increment use count.
911 Values
[ValueID
-1].second
++;
915 if (auto *GO
= dyn_cast
<GlobalObject
>(V
))
916 if (const Comdat
*C
= GO
->getComdat())
919 // Enumerate the type of this value.
920 EnumerateType(V
->getType());
922 if (const Constant
*C
= dyn_cast
<Constant
>(V
)) {
923 if (isa
<GlobalValue
>(C
)) {
924 // Initializers for globals are handled explicitly elsewhere.
925 } else if (C
->getNumOperands()) {
926 // If a constant has operands, enumerate them. This makes sure that if a
927 // constant has uses (for example an array of const ints), that they are
930 // We prefer to enumerate them with values before we enumerate the user
931 // itself. This makes it more likely that we can avoid forward references
932 // in the reader. We know that there can be no cycles in the constants
933 // graph that don't go through a global variable.
934 for (const Use
&U
: C
->operands())
935 if (!isa
<BasicBlock
>(U
)) // Don't enumerate BB operand to BlockAddress.
937 if (auto *CE
= dyn_cast
<ConstantExpr
>(C
)) {
938 if (CE
->getOpcode() == Instruction::ShuffleVector
)
939 EnumerateValue(CE
->getShuffleMaskForBitcode());
940 if (auto *GEP
= dyn_cast
<GEPOperator
>(CE
))
941 EnumerateType(GEP
->getSourceElementType());
944 // Finally, add the value. Doing this could make the ValueID reference be
945 // dangling, don't reuse it.
946 Values
.push_back(std::make_pair(V
, 1U));
947 ValueMap
[V
] = Values
.size();
953 Values
.push_back(std::make_pair(V
, 1U));
954 ValueID
= Values
.size();
958 void ValueEnumerator::EnumerateType(Type
*Ty
) {
959 unsigned *TypeID
= &TypeMap
[Ty
];
961 // We've already seen this type.
965 // If it is a non-anonymous struct, mark the type as being visited so that we
966 // don't recursively visit it. This is safe because we allow forward
967 // references of these in the bitcode reader.
968 if (StructType
*STy
= dyn_cast
<StructType
>(Ty
))
969 if (!STy
->isLiteral())
972 // Enumerate all of the subtypes before we enumerate this type. This ensures
973 // that the type will be enumerated in an order that can be directly built.
974 for (Type
*SubTy
: Ty
->subtypes())
975 EnumerateType(SubTy
);
977 // Refresh the TypeID pointer in case the table rehashed.
978 TypeID
= &TypeMap
[Ty
];
980 // Check to see if we got the pointer another way. This can happen when
981 // enumerating recursive types that hit the base case deeper than they start.
983 // If this is actually a struct that we are treating as forward ref'able,
984 // then emit the definition now that all of its contents are available.
985 if (*TypeID
&& *TypeID
!= ~0U)
988 // Add this type now that its contents are all happily enumerated.
991 *TypeID
= Types
.size();
994 // Enumerate the types for the specified value. If the value is a constant,
995 // walk through it, enumerating the types of the constant.
996 void ValueEnumerator::EnumerateOperandType(const Value
*V
) {
997 EnumerateType(V
->getType());
999 assert(!isa
<MetadataAsValue
>(V
) && "Unexpected metadata operand");
1001 const Constant
*C
= dyn_cast
<Constant
>(V
);
1005 // If this constant is already enumerated, ignore it, we know its type must
1007 if (ValueMap
.count(C
))
1010 // This constant may have operands, make sure to enumerate the types in
1012 for (const Value
*Op
: C
->operands()) {
1013 // Don't enumerate basic blocks here, this happens as operands to
1015 if (isa
<BasicBlock
>(Op
))
1018 EnumerateOperandType(Op
);
1020 if (auto *CE
= dyn_cast
<ConstantExpr
>(C
)) {
1021 if (CE
->getOpcode() == Instruction::ShuffleVector
)
1022 EnumerateOperandType(CE
->getShuffleMaskForBitcode());
1023 if (CE
->getOpcode() == Instruction::GetElementPtr
)
1024 EnumerateType(cast
<GEPOperator
>(CE
)->getSourceElementType());
1028 void ValueEnumerator::EnumerateAttributes(AttributeList PAL
) {
1029 if (PAL
.isEmpty()) return; // null is always 0.
1032 unsigned &Entry
= AttributeListMap
[PAL
];
1034 // Never saw this before, add it.
1035 AttributeLists
.push_back(PAL
);
1036 Entry
= AttributeLists
.size();
1039 // Do lookups for all attribute groups.
1040 for (unsigned i
: PAL
.indexes()) {
1041 AttributeSet AS
= PAL
.getAttributes(i
);
1042 if (!AS
.hasAttributes())
1044 IndexAndAttrSet Pair
= {i
, AS
};
1045 unsigned &Entry
= AttributeGroupMap
[Pair
];
1047 AttributeGroups
.push_back(Pair
);
1048 Entry
= AttributeGroups
.size();
1050 for (Attribute Attr
: AS
) {
1051 if (Attr
.isTypeAttribute())
1052 EnumerateType(Attr
.getValueAsType());
1058 void ValueEnumerator::incorporateFunction(const Function
&F
) {
1059 InstructionCount
= 0;
1060 NumModuleValues
= Values
.size();
1062 // Add global metadata to the function block. This doesn't include
1064 incorporateFunctionMetadata(F
);
1066 // Adding function arguments to the value table.
1067 for (const auto &I
: F
.args()) {
1069 if (I
.hasAttribute(Attribute::ByVal
))
1070 EnumerateType(I
.getParamByValType());
1071 else if (I
.hasAttribute(Attribute::StructRet
))
1072 EnumerateType(I
.getParamStructRetType());
1073 else if (I
.hasAttribute(Attribute::ByRef
))
1074 EnumerateType(I
.getParamByRefType());
1076 FirstFuncConstantID
= Values
.size();
1078 // Add all function-level constants to the value table.
1079 for (const BasicBlock
&BB
: F
) {
1080 for (const Instruction
&I
: BB
) {
1081 for (const Use
&OI
: I
.operands()) {
1082 if ((isa
<Constant
>(OI
) && !isa
<GlobalValue
>(OI
)) || isa
<InlineAsm
>(OI
))
1085 if (auto *SVI
= dyn_cast
<ShuffleVectorInst
>(&I
))
1086 EnumerateValue(SVI
->getShuffleMaskForBitcode());
1088 BasicBlocks
.push_back(&BB
);
1089 ValueMap
[&BB
] = BasicBlocks
.size();
1092 // Optimize the constant layout.
1093 OptimizeConstants(FirstFuncConstantID
, Values
.size());
1095 // Add the function's parameter attributes so they are available for use in
1096 // the function's instruction.
1097 EnumerateAttributes(F
.getAttributes());
1099 FirstInstID
= Values
.size();
1101 SmallVector
<LocalAsMetadata
*, 8> FnLocalMDVector
;
1102 SmallVector
<DIArgList
*, 8> ArgListMDVector
;
1104 auto AddFnLocalMetadata
= [&](Metadata
*MD
) {
1107 if (auto *Local
= dyn_cast
<LocalAsMetadata
>(MD
)) {
1108 // Enumerate metadata after the instructions they might refer to.
1109 FnLocalMDVector
.push_back(Local
);
1110 } else if (auto *ArgList
= dyn_cast
<DIArgList
>(MD
)) {
1111 ArgListMDVector
.push_back(ArgList
);
1112 for (ValueAsMetadata
*VMD
: ArgList
->getArgs()) {
1113 if (auto *Local
= dyn_cast
<LocalAsMetadata
>(VMD
)) {
1114 // Enumerate metadata after the instructions they might refer
1116 FnLocalMDVector
.push_back(Local
);
1122 // Add all of the instructions.
1123 for (const BasicBlock
&BB
: F
) {
1124 for (const Instruction
&I
: BB
) {
1125 for (const Use
&OI
: I
.operands()) {
1126 if (auto *MD
= dyn_cast
<MetadataAsValue
>(&OI
))
1127 AddFnLocalMetadata(MD
->getMetadata());
1129 /// RemoveDIs: Add non-instruction function-local metadata uses.
1130 for (DbgVariableRecord
&DVR
: filterDbgVars(I
.getDbgRecordRange())) {
1131 assert(DVR
.getRawLocation() &&
1132 "DbgVariableRecord location unexpectedly null");
1133 AddFnLocalMetadata(DVR
.getRawLocation());
1134 if (DVR
.isDbgAssign()) {
1135 assert(DVR
.getRawAddress() &&
1136 "DbgVariableRecord location unexpectedly null");
1137 AddFnLocalMetadata(DVR
.getRawAddress());
1140 if (!I
.getType()->isVoidTy())
1145 // Add all of the function-local metadata.
1146 for (const LocalAsMetadata
*Local
: FnLocalMDVector
) {
1147 // At this point, every local values have been incorporated, we shouldn't
1148 // have a metadata operand that references a value that hasn't been seen.
1149 assert(ValueMap
.count(Local
->getValue()) &&
1150 "Missing value for metadata operand");
1151 EnumerateFunctionLocalMetadata(F
, Local
);
1153 // DIArgList entries must come after function-local metadata, as it is not
1154 // possible to forward-reference them.
1155 for (const DIArgList
*ArgList
: ArgListMDVector
)
1156 EnumerateFunctionLocalListMetadata(F
, ArgList
);
1159 void ValueEnumerator::purgeFunction() {
1160 /// Remove purged values from the ValueMap.
1161 for (const auto &V
: llvm::drop_begin(Values
, NumModuleValues
))
1162 ValueMap
.erase(V
.first
);
1163 for (const Metadata
*MD
: llvm::drop_begin(MDs
, NumModuleMDs
))
1164 MetadataMap
.erase(MD
);
1165 for (const BasicBlock
*BB
: BasicBlocks
)
1168 Values
.resize(NumModuleValues
);
1169 MDs
.resize(NumModuleMDs
);
1170 BasicBlocks
.clear();
1174 static void IncorporateFunctionInfoGlobalBBIDs(const Function
*F
,
1175 DenseMap
<const BasicBlock
*, unsigned> &IDMap
) {
1176 unsigned Counter
= 0;
1177 for (const BasicBlock
&BB
: *F
)
1178 IDMap
[&BB
] = ++Counter
;
1181 /// getGlobalBasicBlockID - This returns the function-specific ID for the
1182 /// specified basic block. This is relatively expensive information, so it
1183 /// should only be used by rare constructs such as address-of-label.
1184 unsigned ValueEnumerator::getGlobalBasicBlockID(const BasicBlock
*BB
) const {
1185 unsigned &Idx
= GlobalBasicBlockIDs
[BB
];
1189 IncorporateFunctionInfoGlobalBBIDs(BB
->getParent(), GlobalBasicBlockIDs
);
1190 return getGlobalBasicBlockID(BB
);
1193 uint64_t ValueEnumerator::computeBitsRequiredForTypeIndices() const {
1194 return Log2_32_Ceil(getTypes().size() + 1);