1 //===- RDFGraph.cpp -------------------------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // Target-independent, SSA-based data flow graph for register data flow (RDF).
11 #include "llvm/CodeGen/RDFGraph.h"
12 #include "llvm/ADT/BitVector.h"
13 #include "llvm/ADT/STLExtras.h"
14 #include "llvm/ADT/SetVector.h"
15 #include "llvm/CodeGen/MachineBasicBlock.h"
16 #include "llvm/CodeGen/MachineDominanceFrontier.h"
17 #include "llvm/CodeGen/MachineDominators.h"
18 #include "llvm/CodeGen/MachineFunction.h"
19 #include "llvm/CodeGen/MachineInstr.h"
20 #include "llvm/CodeGen/MachineOperand.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/CodeGen/RDFRegisters.h"
23 #include "llvm/CodeGen/TargetInstrInfo.h"
24 #include "llvm/CodeGen/TargetLowering.h"
25 #include "llvm/CodeGen/TargetRegisterInfo.h"
26 #include "llvm/CodeGen/TargetSubtargetInfo.h"
27 #include "llvm/IR/Function.h"
28 #include "llvm/MC/LaneBitmask.h"
29 #include "llvm/MC/MCInstrDesc.h"
30 #include "llvm/Support/ErrorHandling.h"
31 #include "llvm/Support/raw_ostream.h"
44 // Printing functions. Have them here first, so that the rest of the code
49 raw_ostream
&operator<< (raw_ostream
&OS
, const PrintLaneMaskOpt
&P
) {
51 OS
<< ':' << PrintLaneMask(P
.Mask
);
55 raw_ostream
&operator<< (raw_ostream
&OS
, const Print
<RegisterRef
> &P
) {
56 auto &TRI
= P
.G
.getTRI();
57 if (P
.Obj
.Reg
> 0 && P
.Obj
.Reg
< TRI
.getNumRegs())
58 OS
<< TRI
.getName(P
.Obj
.Reg
);
60 OS
<< '#' << P
.Obj
.Reg
;
61 OS
<< PrintLaneMaskOpt(P
.Obj
.Mask
);
65 raw_ostream
&operator<< (raw_ostream
&OS
, const Print
<NodeId
> &P
) {
66 auto NA
= P
.G
.addr
<NodeBase
*>(P
.Obj
);
67 uint16_t Attrs
= NA
.Addr
->getAttrs();
68 uint16_t Kind
= NodeAttrs::kind(Attrs
);
69 uint16_t Flags
= NodeAttrs::flags(Attrs
);
70 switch (NodeAttrs::type(Attrs
)) {
73 case NodeAttrs::Func
: OS
<< 'f'; break;
74 case NodeAttrs::Block
: OS
<< 'b'; break;
75 case NodeAttrs::Stmt
: OS
<< 's'; break;
76 case NodeAttrs::Phi
: OS
<< 'p'; break;
77 default: OS
<< "c?"; break;
81 if (Flags
& NodeAttrs::Undef
)
83 if (Flags
& NodeAttrs::Dead
)
85 if (Flags
& NodeAttrs::Preserving
)
87 if (Flags
& NodeAttrs::Clobbering
)
90 case NodeAttrs::Use
: OS
<< 'u'; break;
91 case NodeAttrs::Def
: OS
<< 'd'; break;
92 case NodeAttrs::Block
: OS
<< 'b'; break;
93 default: OS
<< "r?"; break;
101 if (Flags
& NodeAttrs::Shadow
)
106 static void printRefHeader(raw_ostream
&OS
, const NodeAddr
<RefNode
*> RA
,
107 const DataFlowGraph
&G
) {
108 OS
<< Print(RA
.Id
, G
) << '<'
109 << Print(RA
.Addr
->getRegRef(G
), G
) << '>';
110 if (RA
.Addr
->getFlags() & NodeAttrs::Fixed
)
114 raw_ostream
&operator<< (raw_ostream
&OS
, const Print
<NodeAddr
<DefNode
*>> &P
) {
115 printRefHeader(OS
, P
.Obj
, P
.G
);
117 if (NodeId N
= P
.Obj
.Addr
->getReachingDef())
120 if (NodeId N
= P
.Obj
.Addr
->getReachedDef())
123 if (NodeId N
= P
.Obj
.Addr
->getReachedUse())
126 if (NodeId N
= P
.Obj
.Addr
->getSibling())
131 raw_ostream
&operator<< (raw_ostream
&OS
, const Print
<NodeAddr
<UseNode
*>> &P
) {
132 printRefHeader(OS
, P
.Obj
, P
.G
);
134 if (NodeId N
= P
.Obj
.Addr
->getReachingDef())
137 if (NodeId N
= P
.Obj
.Addr
->getSibling())
142 raw_ostream
&operator<< (raw_ostream
&OS
,
143 const Print
<NodeAddr
<PhiUseNode
*>> &P
) {
144 printRefHeader(OS
, P
.Obj
, P
.G
);
146 if (NodeId N
= P
.Obj
.Addr
->getReachingDef())
149 if (NodeId N
= P
.Obj
.Addr
->getPredecessor())
152 if (NodeId N
= P
.Obj
.Addr
->getSibling())
157 raw_ostream
&operator<< (raw_ostream
&OS
, const Print
<NodeAddr
<RefNode
*>> &P
) {
158 switch (P
.Obj
.Addr
->getKind()) {
160 OS
<< PrintNode
<DefNode
*>(P
.Obj
, P
.G
);
163 if (P
.Obj
.Addr
->getFlags() & NodeAttrs::PhiRef
)
164 OS
<< PrintNode
<PhiUseNode
*>(P
.Obj
, P
.G
);
166 OS
<< PrintNode
<UseNode
*>(P
.Obj
, P
.G
);
172 raw_ostream
&operator<< (raw_ostream
&OS
, const Print
<NodeList
> &P
) {
173 unsigned N
= P
.Obj
.size();
174 for (auto I
: P
.Obj
) {
175 OS
<< Print(I
.Id
, P
.G
);
182 raw_ostream
&operator<< (raw_ostream
&OS
, const Print
<NodeSet
> &P
) {
183 unsigned N
= P
.Obj
.size();
184 for (auto I
: P
.Obj
) {
194 template <typename T
>
196 PrintListV(const NodeList
&L
, const DataFlowGraph
&G
) : List(L
), G(G
) {}
199 const NodeList
&List
;
200 const DataFlowGraph
&G
;
203 template <typename T
>
204 raw_ostream
&operator<< (raw_ostream
&OS
, const PrintListV
<T
> &P
) {
205 unsigned N
= P
.List
.size();
206 for (NodeAddr
<T
> A
: P
.List
) {
207 OS
<< PrintNode
<T
>(A
, P
.G
);
214 } // end anonymous namespace
216 raw_ostream
&operator<< (raw_ostream
&OS
, const Print
<NodeAddr
<PhiNode
*>> &P
) {
217 OS
<< Print(P
.Obj
.Id
, P
.G
) << ": phi ["
218 << PrintListV
<RefNode
*>(P
.Obj
.Addr
->members(P
.G
), P
.G
) << ']';
222 raw_ostream
&operator<<(raw_ostream
&OS
, const Print
<NodeAddr
<StmtNode
*>> &P
) {
223 const MachineInstr
&MI
= *P
.Obj
.Addr
->getCode();
224 unsigned Opc
= MI
.getOpcode();
225 OS
<< Print(P
.Obj
.Id
, P
.G
) << ": " << P
.G
.getTII().getName(Opc
);
226 // Print the target for calls and branches (for readability).
227 if (MI
.isCall() || MI
.isBranch()) {
228 MachineInstr::const_mop_iterator T
=
229 llvm::find_if(MI
.operands(),
230 [] (const MachineOperand
&Op
) -> bool {
231 return Op
.isMBB() || Op
.isGlobal() || Op
.isSymbol();
233 if (T
!= MI
.operands_end()) {
236 OS
<< printMBBReference(*T
->getMBB());
237 else if (T
->isGlobal())
238 OS
<< T
->getGlobal()->getName();
239 else if (T
->isSymbol())
240 OS
<< T
->getSymbolName();
243 OS
<< " [" << PrintListV
<RefNode
*>(P
.Obj
.Addr
->members(P
.G
), P
.G
) << ']';
247 raw_ostream
&operator<< (raw_ostream
&OS
,
248 const Print
<NodeAddr
<InstrNode
*>> &P
) {
249 switch (P
.Obj
.Addr
->getKind()) {
251 OS
<< PrintNode
<PhiNode
*>(P
.Obj
, P
.G
);
253 case NodeAttrs::Stmt
:
254 OS
<< PrintNode
<StmtNode
*>(P
.Obj
, P
.G
);
257 OS
<< "instr? " << Print(P
.Obj
.Id
, P
.G
);
263 raw_ostream
&operator<< (raw_ostream
&OS
,
264 const Print
<NodeAddr
<BlockNode
*>> &P
) {
265 MachineBasicBlock
*BB
= P
.Obj
.Addr
->getCode();
266 unsigned NP
= BB
->pred_size();
268 auto PrintBBs
= [&OS
] (std::vector
<int> Ns
) -> void {
269 unsigned N
= Ns
.size();
277 OS
<< Print(P
.Obj
.Id
, P
.G
) << ": --- " << printMBBReference(*BB
)
278 << " --- preds(" << NP
<< "): ";
279 for (MachineBasicBlock
*B
: BB
->predecessors())
280 Ns
.push_back(B
->getNumber());
283 unsigned NS
= BB
->succ_size();
284 OS
<< " succs(" << NS
<< "): ";
286 for (MachineBasicBlock
*B
: BB
->successors())
287 Ns
.push_back(B
->getNumber());
291 for (auto I
: P
.Obj
.Addr
->members(P
.G
))
292 OS
<< PrintNode
<InstrNode
*>(I
, P
.G
) << '\n';
296 raw_ostream
&operator<<(raw_ostream
&OS
, const Print
<NodeAddr
<FuncNode
*>> &P
) {
297 OS
<< "DFG dump:[\n" << Print(P
.Obj
.Id
, P
.G
) << ": Function: "
298 << P
.Obj
.Addr
->getCode()->getName() << '\n';
299 for (auto I
: P
.Obj
.Addr
->members(P
.G
))
300 OS
<< PrintNode
<BlockNode
*>(I
, P
.G
) << '\n';
305 raw_ostream
&operator<< (raw_ostream
&OS
, const Print
<RegisterSet
> &P
) {
308 OS
<< ' ' << Print(I
, P
.G
);
313 raw_ostream
&operator<< (raw_ostream
&OS
, const Print
<RegisterAggr
> &P
) {
318 raw_ostream
&operator<< (raw_ostream
&OS
,
319 const Print
<DataFlowGraph::DefStack
> &P
) {
320 for (auto I
= P
.Obj
.top(), E
= P
.Obj
.bottom(); I
!= E
; ) {
321 OS
<< Print(I
->Id
, P
.G
)
322 << '<' << Print(I
->Addr
->getRegRef(P
.G
), P
.G
) << '>';
330 } // end namespace rdf
331 } // end namespace llvm
333 // Node allocation functions.
335 // Node allocator is like a slab memory allocator: it allocates blocks of
336 // memory in sizes that are multiples of the size of a node. Each block has
337 // the same size. Nodes are allocated from the currently active block, and
338 // when it becomes full, a new one is created.
339 // There is a mapping scheme between node id and its location in a block,
340 // and within that block is described in the header file.
342 void NodeAllocator::startNewBlock() {
343 void *T
= MemPool
.Allocate(NodesPerBlock
*NodeMemSize
, NodeMemSize
);
344 char *P
= static_cast<char*>(T
);
346 // Check if the block index is still within the allowed range, i.e. less
347 // than 2^N, where N is the number of bits in NodeId for the block index.
348 // BitsPerIndex is the number of bits per node index.
349 assert((Blocks
.size() < ((size_t)1 << (8*sizeof(NodeId
)-BitsPerIndex
))) &&
350 "Out of bits for block index");
354 bool NodeAllocator::needNewBlock() {
358 char *ActiveBegin
= Blocks
.back();
359 uint32_t Index
= (ActiveEnd
-ActiveBegin
)/NodeMemSize
;
360 return Index
>= NodesPerBlock
;
363 NodeAddr
<NodeBase
*> NodeAllocator::New() {
367 uint32_t ActiveB
= Blocks
.size()-1;
368 uint32_t Index
= (ActiveEnd
- Blocks
[ActiveB
])/NodeMemSize
;
369 NodeAddr
<NodeBase
*> NA
= { reinterpret_cast<NodeBase
*>(ActiveEnd
),
370 makeId(ActiveB
, Index
) };
371 ActiveEnd
+= NodeMemSize
;
375 NodeId
NodeAllocator::id(const NodeBase
*P
) const {
376 uintptr_t A
= reinterpret_cast<uintptr_t>(P
);
377 for (unsigned i
= 0, n
= Blocks
.size(); i
!= n
; ++i
) {
378 uintptr_t B
= reinterpret_cast<uintptr_t>(Blocks
[i
]);
379 if (A
< B
|| A
>= B
+ NodesPerBlock
*NodeMemSize
)
381 uint32_t Idx
= (A
-B
)/NodeMemSize
;
382 return makeId(i
, Idx
);
384 llvm_unreachable("Invalid node address");
387 void NodeAllocator::clear() {
393 // Insert node NA after "this" in the circular chain.
394 void NodeBase::append(NodeAddr
<NodeBase
*> NA
) {
396 // If NA is already "next", do nothing.
403 // Fundamental node manipulator functions.
405 // Obtain the register reference from a reference node.
406 RegisterRef
RefNode::getRegRef(const DataFlowGraph
&G
) const {
407 assert(NodeAttrs::type(Attrs
) == NodeAttrs::Ref
);
408 if (NodeAttrs::flags(Attrs
) & NodeAttrs::PhiRef
)
409 return G
.unpack(Ref
.PR
);
410 assert(Ref
.Op
!= nullptr);
411 return G
.makeRegRef(*Ref
.Op
);
414 // Set the register reference in the reference node directly (for references
416 void RefNode::setRegRef(RegisterRef RR
, DataFlowGraph
&G
) {
417 assert(NodeAttrs::type(Attrs
) == NodeAttrs::Ref
);
418 assert(NodeAttrs::flags(Attrs
) & NodeAttrs::PhiRef
);
422 // Set the register reference in the reference node based on a machine
423 // operand (for references in statement nodes).
424 void RefNode::setRegRef(MachineOperand
*Op
, DataFlowGraph
&G
) {
425 assert(NodeAttrs::type(Attrs
) == NodeAttrs::Ref
);
426 assert(!(NodeAttrs::flags(Attrs
) & NodeAttrs::PhiRef
));
431 // Get the owner of a given reference node.
432 NodeAddr
<NodeBase
*> RefNode::getOwner(const DataFlowGraph
&G
) {
433 NodeAddr
<NodeBase
*> NA
= G
.addr
<NodeBase
*>(getNext());
435 while (NA
.Addr
!= this) {
436 if (NA
.Addr
->getType() == NodeAttrs::Code
)
438 NA
= G
.addr
<NodeBase
*>(NA
.Addr
->getNext());
440 llvm_unreachable("No owner in circular list");
443 // Connect the def node to the reaching def node.
444 void DefNode::linkToDef(NodeId Self
, NodeAddr
<DefNode
*> DA
) {
446 Ref
.Sib
= DA
.Addr
->getReachedDef();
447 DA
.Addr
->setReachedDef(Self
);
450 // Connect the use node to the reaching def node.
451 void UseNode::linkToDef(NodeId Self
, NodeAddr
<DefNode
*> DA
) {
453 Ref
.Sib
= DA
.Addr
->getReachedUse();
454 DA
.Addr
->setReachedUse(Self
);
457 // Get the first member of the code node.
458 NodeAddr
<NodeBase
*> CodeNode::getFirstMember(const DataFlowGraph
&G
) const {
459 if (Code
.FirstM
== 0)
460 return NodeAddr
<NodeBase
*>();
461 return G
.addr
<NodeBase
*>(Code
.FirstM
);
464 // Get the last member of the code node.
465 NodeAddr
<NodeBase
*> CodeNode::getLastMember(const DataFlowGraph
&G
) const {
467 return NodeAddr
<NodeBase
*>();
468 return G
.addr
<NodeBase
*>(Code
.LastM
);
471 // Add node NA at the end of the member list of the given code node.
472 void CodeNode::addMember(NodeAddr
<NodeBase
*> NA
, const DataFlowGraph
&G
) {
473 NodeAddr
<NodeBase
*> ML
= getLastMember(G
);
478 NodeId Self
= G
.id(this);
479 NA
.Addr
->setNext(Self
);
484 // Add node NA after member node MA in the given code node.
485 void CodeNode::addMemberAfter(NodeAddr
<NodeBase
*> MA
, NodeAddr
<NodeBase
*> NA
,
486 const DataFlowGraph
&G
) {
488 if (Code
.LastM
== MA
.Id
)
492 // Remove member node NA from the given code node.
493 void CodeNode::removeMember(NodeAddr
<NodeBase
*> NA
, const DataFlowGraph
&G
) {
494 NodeAddr
<NodeBase
*> MA
= getFirstMember(G
);
497 // Special handling if the member to remove is the first member.
498 if (MA
.Id
== NA
.Id
) {
499 if (Code
.LastM
== MA
.Id
) {
500 // If it is the only member, set both first and last to 0.
501 Code
.FirstM
= Code
.LastM
= 0;
503 // Otherwise, advance the first member.
504 Code
.FirstM
= MA
.Addr
->getNext();
509 while (MA
.Addr
!= this) {
510 NodeId MX
= MA
.Addr
->getNext();
512 MA
.Addr
->setNext(NA
.Addr
->getNext());
513 // If the member to remove happens to be the last one, update the
515 if (Code
.LastM
== NA
.Id
)
519 MA
= G
.addr
<NodeBase
*>(MX
);
521 llvm_unreachable("No such member");
524 // Return the list of all members of the code node.
525 NodeList
CodeNode::members(const DataFlowGraph
&G
) const {
526 static auto True
= [] (NodeAddr
<NodeBase
*>) -> bool { return true; };
527 return members_if(True
, G
);
530 // Return the owner of the given instr node.
531 NodeAddr
<NodeBase
*> InstrNode::getOwner(const DataFlowGraph
&G
) {
532 NodeAddr
<NodeBase
*> NA
= G
.addr
<NodeBase
*>(getNext());
534 while (NA
.Addr
!= this) {
535 assert(NA
.Addr
->getType() == NodeAttrs::Code
);
536 if (NA
.Addr
->getKind() == NodeAttrs::Block
)
538 NA
= G
.addr
<NodeBase
*>(NA
.Addr
->getNext());
540 llvm_unreachable("No owner in circular list");
543 // Add the phi node PA to the given block node.
544 void BlockNode::addPhi(NodeAddr
<PhiNode
*> PA
, const DataFlowGraph
&G
) {
545 NodeAddr
<NodeBase
*> M
= getFirstMember(G
);
551 assert(M
.Addr
->getType() == NodeAttrs::Code
);
552 if (M
.Addr
->getKind() == NodeAttrs::Stmt
) {
553 // If the first member of the block is a statement, insert the phi as
556 PA
.Addr
->setNext(M
.Id
);
558 // If the first member is a phi, find the last phi, and append PA to it.
559 assert(M
.Addr
->getKind() == NodeAttrs::Phi
);
560 NodeAddr
<NodeBase
*> MN
= M
;
563 MN
= G
.addr
<NodeBase
*>(M
.Addr
->getNext());
564 assert(MN
.Addr
->getType() == NodeAttrs::Code
);
565 } while (MN
.Addr
->getKind() == NodeAttrs::Phi
);
567 // M is the last phi.
568 addMemberAfter(M
, PA
, G
);
572 // Find the block node corresponding to the machine basic block BB in the
574 NodeAddr
<BlockNode
*> FuncNode::findBlock(const MachineBasicBlock
*BB
,
575 const DataFlowGraph
&G
) const {
576 auto EqBB
= [BB
] (NodeAddr
<NodeBase
*> NA
) -> bool {
577 return NodeAddr
<BlockNode
*>(NA
).Addr
->getCode() == BB
;
579 NodeList Ms
= members_if(EqBB
, G
);
582 return NodeAddr
<BlockNode
*>();
585 // Get the block node for the entry block in the given function.
586 NodeAddr
<BlockNode
*> FuncNode::getEntryBlock(const DataFlowGraph
&G
) {
587 MachineBasicBlock
*EntryB
= &getCode()->front();
588 return findBlock(EntryB
, G
);
591 // Target operand information.
594 // For a given instruction, check if there are any bits of RR that can remain
595 // unchanged across this def.
596 bool TargetOperandInfo::isPreserving(const MachineInstr
&In
, unsigned OpNum
)
598 return TII
.isPredicated(In
);
601 // Check if the definition of RR produces an unspecified value.
602 bool TargetOperandInfo::isClobbering(const MachineInstr
&In
, unsigned OpNum
)
604 const MachineOperand
&Op
= In
.getOperand(OpNum
);
609 if (Op
.isDef() && Op
.isDead())
614 // Check if the given instruction specifically requires
615 bool TargetOperandInfo::isFixedReg(const MachineInstr
&In
, unsigned OpNum
)
617 if (In
.isCall() || In
.isReturn() || In
.isInlineAsm())
619 // Check for a tail call.
621 for (const MachineOperand
&O
: In
.operands())
622 if (O
.isGlobal() || O
.isSymbol())
625 const MCInstrDesc
&D
= In
.getDesc();
626 if (!D
.getImplicitDefs() && !D
.getImplicitUses())
628 const MachineOperand
&Op
= In
.getOperand(OpNum
);
629 // If there is a sub-register, treat the operand as non-fixed. Currently,
630 // fixed registers are those that are listed in the descriptor as implicit
631 // uses or defs, and those lists do not allow sub-registers.
632 if (Op
.getSubReg() != 0)
634 Register Reg
= Op
.getReg();
635 const MCPhysReg
*ImpR
= Op
.isDef() ? D
.getImplicitDefs()
636 : D
.getImplicitUses();
646 // The data flow graph construction.
649 DataFlowGraph::DataFlowGraph(MachineFunction
&mf
, const TargetInstrInfo
&tii
,
650 const TargetRegisterInfo
&tri
, const MachineDominatorTree
&mdt
,
651 const MachineDominanceFrontier
&mdf
)
652 : DefaultTOI(std::make_unique
<TargetOperandInfo
>(tii
)), MF(mf
), TII(tii
),
653 TRI(tri
), PRI(tri
, mf
), MDT(mdt
), MDF(mdf
), TOI(*DefaultTOI
),
657 DataFlowGraph::DataFlowGraph(MachineFunction
&mf
, const TargetInstrInfo
&tii
,
658 const TargetRegisterInfo
&tri
, const MachineDominatorTree
&mdt
,
659 const MachineDominanceFrontier
&mdf
, const TargetOperandInfo
&toi
)
660 : MF(mf
), TII(tii
), TRI(tri
), PRI(tri
, mf
), MDT(mdt
), MDF(mdf
), TOI(toi
),
664 // The implementation of the definition stack.
665 // Each register reference has its own definition stack. In particular,
666 // for a register references "Reg" and "Reg:subreg" will each have their
667 // own definition stacks.
669 // Construct a stack iterator.
670 DataFlowGraph::DefStack::Iterator::Iterator(const DataFlowGraph::DefStack
&S
,
673 // Initialize to bottom.
677 // Initialize to the top, i.e. top-most non-delimiter (or 0, if empty).
678 Pos
= DS
.Stack
.size();
679 while (Pos
> 0 && DS
.isDelimiter(DS
.Stack
[Pos
-1]))
683 // Return the size of the stack, including block delimiters.
684 unsigned DataFlowGraph::DefStack::size() const {
686 for (auto I
= top(), E
= bottom(); I
!= E
; I
.down())
691 // Remove the top entry from the stack. Remove all intervening delimiters
692 // so that after this, the stack is either empty, or the top of the stack
693 // is a non-delimiter.
694 void DataFlowGraph::DefStack::pop() {
696 unsigned P
= nextDown(Stack
.size());
700 // Push a delimiter for block node N on the stack.
701 void DataFlowGraph::DefStack::start_block(NodeId N
) {
703 Stack
.push_back(NodeAddr
<DefNode
*>(nullptr, N
));
706 // Remove all nodes from the top of the stack, until the delimited for
707 // block node N is encountered. Remove the delimiter as well. In effect,
708 // this will remove from the stack all definitions from block N.
709 void DataFlowGraph::DefStack::clear_block(NodeId N
) {
711 unsigned P
= Stack
.size();
713 bool Found
= isDelimiter(Stack
[P
-1], N
);
718 // This will also remove the delimiter, if found.
722 // Move the stack iterator up by one.
723 unsigned DataFlowGraph::DefStack::nextUp(unsigned P
) const {
724 // Get the next valid position after P (skipping all delimiters).
725 // The input position P does not have to point to a non-delimiter.
726 unsigned SS
= Stack
.size();
731 IsDelim
= isDelimiter(Stack
[P
-1]);
732 } while (P
< SS
&& IsDelim
);
737 // Move the stack iterator down by one.
738 unsigned DataFlowGraph::DefStack::nextDown(unsigned P
) const {
739 // Get the preceding valid position before P (skipping all delimiters).
740 // The input position P does not have to point to a non-delimiter.
741 assert(P
> 0 && P
<= Stack
.size());
742 bool IsDelim
= isDelimiter(Stack
[P
-1]);
746 IsDelim
= isDelimiter(Stack
[P
-1]);
747 } while (P
> 0 && IsDelim
);
752 // Register information.
754 RegisterSet
DataFlowGraph::getLandingPadLiveIns() const {
756 const Function
&F
= MF
.getFunction();
757 const Constant
*PF
= F
.hasPersonalityFn() ? F
.getPersonalityFn()
759 const TargetLowering
&TLI
= *MF
.getSubtarget().getTargetLowering();
760 if (RegisterId R
= TLI
.getExceptionPointerRegister(PF
))
761 LR
.insert(RegisterRef(R
));
762 if (!isFuncletEHPersonality(classifyEHPersonality(PF
))) {
763 if (RegisterId R
= TLI
.getExceptionSelectorRegister(PF
))
764 LR
.insert(RegisterRef(R
));
769 // Node management functions.
771 // Get the pointer to the node with the id N.
772 NodeBase
*DataFlowGraph::ptr(NodeId N
) const {
775 return Memory
.ptr(N
);
778 // Get the id of the node at the address P.
779 NodeId
DataFlowGraph::id(const NodeBase
*P
) const {
785 // Allocate a new node and set the attributes to Attrs.
786 NodeAddr
<NodeBase
*> DataFlowGraph::newNode(uint16_t Attrs
) {
787 NodeAddr
<NodeBase
*> P
= Memory
.New();
789 P
.Addr
->setAttrs(Attrs
);
793 // Make a copy of the given node B, except for the data-flow links, which
795 NodeAddr
<NodeBase
*> DataFlowGraph::cloneNode(const NodeAddr
<NodeBase
*> B
) {
796 NodeAddr
<NodeBase
*> NA
= newNode(0);
797 memcpy(NA
.Addr
, B
.Addr
, sizeof(NodeBase
));
798 // Ref nodes need to have the data-flow links reset.
799 if (NA
.Addr
->getType() == NodeAttrs::Ref
) {
800 NodeAddr
<RefNode
*> RA
= NA
;
801 RA
.Addr
->setReachingDef(0);
802 RA
.Addr
->setSibling(0);
803 if (NA
.Addr
->getKind() == NodeAttrs::Def
) {
804 NodeAddr
<DefNode
*> DA
= NA
;
805 DA
.Addr
->setReachedDef(0);
806 DA
.Addr
->setReachedUse(0);
812 // Allocation routines for specific node types/kinds.
814 NodeAddr
<UseNode
*> DataFlowGraph::newUse(NodeAddr
<InstrNode
*> Owner
,
815 MachineOperand
&Op
, uint16_t Flags
) {
816 NodeAddr
<UseNode
*> UA
= newNode(NodeAttrs::Ref
| NodeAttrs::Use
| Flags
);
817 UA
.Addr
->setRegRef(&Op
, *this);
821 NodeAddr
<PhiUseNode
*> DataFlowGraph::newPhiUse(NodeAddr
<PhiNode
*> Owner
,
822 RegisterRef RR
, NodeAddr
<BlockNode
*> PredB
, uint16_t Flags
) {
823 NodeAddr
<PhiUseNode
*> PUA
= newNode(NodeAttrs::Ref
| NodeAttrs::Use
| Flags
);
824 assert(Flags
& NodeAttrs::PhiRef
);
825 PUA
.Addr
->setRegRef(RR
, *this);
826 PUA
.Addr
->setPredecessor(PredB
.Id
);
830 NodeAddr
<DefNode
*> DataFlowGraph::newDef(NodeAddr
<InstrNode
*> Owner
,
831 MachineOperand
&Op
, uint16_t Flags
) {
832 NodeAddr
<DefNode
*> DA
= newNode(NodeAttrs::Ref
| NodeAttrs::Def
| Flags
);
833 DA
.Addr
->setRegRef(&Op
, *this);
837 NodeAddr
<DefNode
*> DataFlowGraph::newDef(NodeAddr
<InstrNode
*> Owner
,
838 RegisterRef RR
, uint16_t Flags
) {
839 NodeAddr
<DefNode
*> DA
= newNode(NodeAttrs::Ref
| NodeAttrs::Def
| Flags
);
840 assert(Flags
& NodeAttrs::PhiRef
);
841 DA
.Addr
->setRegRef(RR
, *this);
845 NodeAddr
<PhiNode
*> DataFlowGraph::newPhi(NodeAddr
<BlockNode
*> Owner
) {
846 NodeAddr
<PhiNode
*> PA
= newNode(NodeAttrs::Code
| NodeAttrs::Phi
);
847 Owner
.Addr
->addPhi(PA
, *this);
851 NodeAddr
<StmtNode
*> DataFlowGraph::newStmt(NodeAddr
<BlockNode
*> Owner
,
853 NodeAddr
<StmtNode
*> SA
= newNode(NodeAttrs::Code
| NodeAttrs::Stmt
);
854 SA
.Addr
->setCode(MI
);
855 Owner
.Addr
->addMember(SA
, *this);
859 NodeAddr
<BlockNode
*> DataFlowGraph::newBlock(NodeAddr
<FuncNode
*> Owner
,
860 MachineBasicBlock
*BB
) {
861 NodeAddr
<BlockNode
*> BA
= newNode(NodeAttrs::Code
| NodeAttrs::Block
);
862 BA
.Addr
->setCode(BB
);
863 Owner
.Addr
->addMember(BA
, *this);
867 NodeAddr
<FuncNode
*> DataFlowGraph::newFunc(MachineFunction
*MF
) {
868 NodeAddr
<FuncNode
*> FA
= newNode(NodeAttrs::Code
| NodeAttrs::Func
);
869 FA
.Addr
->setCode(MF
);
873 // Build the data flow graph.
874 void DataFlowGraph::build(unsigned Options
) {
881 for (MachineBasicBlock
&B
: MF
) {
882 NodeAddr
<BlockNode
*> BA
= newBlock(Func
, &B
);
883 BlockNodes
.insert(std::make_pair(&B
, BA
));
884 for (MachineInstr
&I
: B
) {
885 if (I
.isDebugInstr())
891 NodeAddr
<BlockNode
*> EA
= Func
.Addr
->getEntryBlock(*this);
892 NodeList Blocks
= Func
.Addr
->members(*this);
894 // Collect information about block references.
896 for (NodeAddr
<BlockNode
*> BA
: Blocks
)
897 for (NodeAddr
<InstrNode
*> IA
: BA
.Addr
->members(*this))
898 for (NodeAddr
<RefNode
*> RA
: IA
.Addr
->members(*this))
899 AllRefs
.insert(RA
.Addr
->getRegRef(*this));
901 // Collect function live-ins and entry block live-ins.
902 MachineRegisterInfo
&MRI
= MF
.getRegInfo();
903 MachineBasicBlock
&EntryB
= *EA
.Addr
->getCode();
904 assert(EntryB
.pred_empty() && "Function entry block has predecessors");
905 for (std::pair
<unsigned,unsigned> P
: MRI
.liveins())
906 LiveIns
.insert(RegisterRef(P
.first
));
907 if (MRI
.tracksLiveness()) {
908 for (auto I
: EntryB
.liveins())
909 LiveIns
.insert(RegisterRef(I
.PhysReg
, I
.LaneMask
));
912 // Add function-entry phi nodes for the live-in registers.
913 //for (std::pair<RegisterId,LaneBitmask> P : LiveIns) {
914 for (auto I
= LiveIns
.rr_begin(), E
= LiveIns
.rr_end(); I
!= E
; ++I
) {
916 NodeAddr
<PhiNode
*> PA
= newPhi(EA
);
917 uint16_t PhiFlags
= NodeAttrs::PhiRef
| NodeAttrs::Preserving
;
918 NodeAddr
<DefNode
*> DA
= newDef(PA
, RR
, PhiFlags
);
919 PA
.Addr
->addMember(DA
, *this);
922 // Add phis for landing pads.
923 // Landing pads, unlike usual backs blocks, are not entered through
924 // branches in the program, or fall-throughs from other blocks. They
925 // are entered from the exception handling runtime and target's ABI
926 // may define certain registers as defined on entry to such a block.
927 RegisterSet EHRegs
= getLandingPadLiveIns();
928 if (!EHRegs
.empty()) {
929 for (NodeAddr
<BlockNode
*> BA
: Blocks
) {
930 const MachineBasicBlock
&B
= *BA
.Addr
->getCode();
934 // Prepare a list of NodeIds of the block's predecessors.
936 for (MachineBasicBlock
*PB
: B
.predecessors())
937 Preds
.push_back(findBlock(PB
));
939 // Build phi nodes for each live-in.
940 for (RegisterRef RR
: EHRegs
) {
941 NodeAddr
<PhiNode
*> PA
= newPhi(BA
);
942 uint16_t PhiFlags
= NodeAttrs::PhiRef
| NodeAttrs::Preserving
;
944 NodeAddr
<DefNode
*> DA
= newDef(PA
, RR
, PhiFlags
);
945 PA
.Addr
->addMember(DA
, *this);
946 // Add uses (no reaching defs for phi uses):
947 for (NodeAddr
<BlockNode
*> PBA
: Preds
) {
948 NodeAddr
<PhiUseNode
*> PUA
= newPhiUse(PA
, RR
, PBA
);
949 PA
.Addr
->addMember(PUA
, *this);
955 // Build a map "PhiM" which will contain, for each block, the set
956 // of references that will require phi definitions in that block.
958 for (NodeAddr
<BlockNode
*> BA
: Blocks
)
959 recordDefsForDF(PhiM
, BA
);
960 for (NodeAddr
<BlockNode
*> BA
: Blocks
)
961 buildPhis(PhiM
, AllRefs
, BA
);
963 // Link all the refs. This will recursively traverse the dominator tree.
965 linkBlockRefs(DM
, EA
);
967 // Finally, remove all unused phi nodes.
968 if (!(Options
& BuildOptions::KeepDeadPhis
))
972 RegisterRef
DataFlowGraph::makeRegRef(unsigned Reg
, unsigned Sub
) const {
973 assert(PhysicalRegisterInfo::isRegMaskId(Reg
) ||
974 Register::isPhysicalRegister(Reg
));
977 Reg
= TRI
.getSubReg(Reg
, Sub
);
978 return RegisterRef(Reg
);
981 RegisterRef
DataFlowGraph::makeRegRef(const MachineOperand
&Op
) const {
982 assert(Op
.isReg() || Op
.isRegMask());
984 return makeRegRef(Op
.getReg(), Op
.getSubReg());
985 return RegisterRef(PRI
.getRegMaskId(Op
.getRegMask()), LaneBitmask::getAll());
988 // For each stack in the map DefM, push the delimiter for block B on it.
989 void DataFlowGraph::markBlock(NodeId B
, DefStackMap
&DefM
) {
990 // Push block delimiters.
992 P
.second
.start_block(B
);
995 // Remove all definitions coming from block B from each stack in DefM.
996 void DataFlowGraph::releaseBlock(NodeId B
, DefStackMap
&DefM
) {
997 // Pop all defs from this block from the definition stack. Defs that were
998 // added to the map during the traversal of instructions will not have a
999 // delimiter, but for those, the whole stack will be emptied.
1000 for (auto &P
: DefM
)
1001 P
.second
.clear_block(B
);
1003 // Finally, remove empty stacks from the map.
1004 for (auto I
= DefM
.begin(), E
= DefM
.end(), NextI
= I
; I
!= E
; I
= NextI
) {
1005 NextI
= std::next(I
);
1006 // This preserves the validity of iterators other than I.
1007 if (I
->second
.empty())
1012 // Push all definitions from the instruction node IA to an appropriate
1014 void DataFlowGraph::pushAllDefs(NodeAddr
<InstrNode
*> IA
, DefStackMap
&DefM
) {
1015 pushClobbers(IA
, DefM
);
1019 // Push all definitions from the instruction node IA to an appropriate
1021 void DataFlowGraph::pushClobbers(NodeAddr
<InstrNode
*> IA
, DefStackMap
&DefM
) {
1023 std::set
<RegisterId
> Defined
;
1025 // The important objectives of this function are:
1026 // - to be able to handle instructions both while the graph is being
1027 // constructed, and after the graph has been constructed, and
1028 // - maintain proper ordering of definitions on the stack for each
1029 // register reference:
1030 // - if there are two or more related defs in IA (i.e. coming from
1031 // the same machine operand), then only push one def on the stack,
1032 // - if there are multiple unrelated defs of non-overlapping
1033 // subregisters of S, then the stack for S will have both (in an
1034 // unspecified order), but the order does not matter from the data-
1035 // -flow perspective.
1037 for (NodeAddr
<DefNode
*> DA
: IA
.Addr
->members_if(IsDef
, *this)) {
1038 if (Visited
.count(DA
.Id
))
1040 if (!(DA
.Addr
->getFlags() & NodeAttrs::Clobbering
))
1043 NodeList Rel
= getRelatedRefs(IA
, DA
);
1044 NodeAddr
<DefNode
*> PDA
= Rel
.front();
1045 RegisterRef RR
= PDA
.Addr
->getRegRef(*this);
1047 // Push the definition on the stack for the register and all aliases.
1048 // The def stack traversal in linkNodeUp will check the exact aliasing.
1049 DefM
[RR
.Reg
].push(DA
);
1050 Defined
.insert(RR
.Reg
);
1051 for (RegisterId A
: PRI
.getAliasSet(RR
.Reg
)) {
1052 // Check that we don't push the same def twice.
1053 assert(A
!= RR
.Reg
);
1054 if (!Defined
.count(A
))
1057 // Mark all the related defs as visited.
1058 for (NodeAddr
<NodeBase
*> T
: Rel
)
1059 Visited
.insert(T
.Id
);
1063 // Push all definitions from the instruction node IA to an appropriate
1065 void DataFlowGraph::pushDefs(NodeAddr
<InstrNode
*> IA
, DefStackMap
&DefM
) {
1068 std::set
<RegisterId
> Defined
;
1071 // The important objectives of this function are:
1072 // - to be able to handle instructions both while the graph is being
1073 // constructed, and after the graph has been constructed, and
1074 // - maintain proper ordering of definitions on the stack for each
1075 // register reference:
1076 // - if there are two or more related defs in IA (i.e. coming from
1077 // the same machine operand), then only push one def on the stack,
1078 // - if there are multiple unrelated defs of non-overlapping
1079 // subregisters of S, then the stack for S will have both (in an
1080 // unspecified order), but the order does not matter from the data-
1081 // -flow perspective.
1083 for (NodeAddr
<DefNode
*> DA
: IA
.Addr
->members_if(IsDef
, *this)) {
1084 if (Visited
.count(DA
.Id
))
1086 if (DA
.Addr
->getFlags() & NodeAttrs::Clobbering
)
1089 NodeList Rel
= getRelatedRefs(IA
, DA
);
1090 NodeAddr
<DefNode
*> PDA
= Rel
.front();
1091 RegisterRef RR
= PDA
.Addr
->getRegRef(*this);
1093 // Assert if the register is defined in two or more unrelated defs.
1094 // This could happen if there are two or more def operands defining it.
1095 if (!Defined
.insert(RR
.Reg
).second
) {
1096 MachineInstr
*MI
= NodeAddr
<StmtNode
*>(IA
).Addr
->getCode();
1097 dbgs() << "Multiple definitions of register: "
1098 << Print(RR
, *this) << " in\n " << *MI
<< "in "
1099 << printMBBReference(*MI
->getParent()) << '\n';
1100 llvm_unreachable(nullptr);
1103 // Push the definition on the stack for the register and all aliases.
1104 // The def stack traversal in linkNodeUp will check the exact aliasing.
1105 DefM
[RR
.Reg
].push(DA
);
1106 for (RegisterId A
: PRI
.getAliasSet(RR
.Reg
)) {
1107 // Check that we don't push the same def twice.
1108 assert(A
!= RR
.Reg
);
1111 // Mark all the related defs as visited.
1112 for (NodeAddr
<NodeBase
*> T
: Rel
)
1113 Visited
.insert(T
.Id
);
1117 // Return the list of all reference nodes related to RA, including RA itself.
1118 // See "getNextRelated" for the meaning of a "related reference".
1119 NodeList
DataFlowGraph::getRelatedRefs(NodeAddr
<InstrNode
*> IA
,
1120 NodeAddr
<RefNode
*> RA
) const {
1121 assert(IA
.Id
!= 0 && RA
.Id
!= 0);
1124 NodeId Start
= RA
.Id
;
1127 RA
= getNextRelated(IA
, RA
);
1128 } while (RA
.Id
!= 0 && RA
.Id
!= Start
);
1132 // Clear all information in the graph.
1133 void DataFlowGraph::reset() {
1136 Func
= NodeAddr
<FuncNode
*>();
1139 // Return the next reference node in the instruction node IA that is related
1140 // to RA. Conceptually, two reference nodes are related if they refer to the
1141 // same instance of a register access, but differ in flags or other minor
1142 // characteristics. Specific examples of related nodes are shadow reference
1144 // Return the equivalent of nullptr if there are no more related references.
1145 NodeAddr
<RefNode
*> DataFlowGraph::getNextRelated(NodeAddr
<InstrNode
*> IA
,
1146 NodeAddr
<RefNode
*> RA
) const {
1147 assert(IA
.Id
!= 0 && RA
.Id
!= 0);
1149 auto Related
= [this,RA
](NodeAddr
<RefNode
*> TA
) -> bool {
1150 if (TA
.Addr
->getKind() != RA
.Addr
->getKind())
1152 if (TA
.Addr
->getRegRef(*this) != RA
.Addr
->getRegRef(*this))
1156 auto RelatedStmt
= [&Related
,RA
](NodeAddr
<RefNode
*> TA
) -> bool {
1157 return Related(TA
) &&
1158 &RA
.Addr
->getOp() == &TA
.Addr
->getOp();
1160 auto RelatedPhi
= [&Related
,RA
](NodeAddr
<RefNode
*> TA
) -> bool {
1163 if (TA
.Addr
->getKind() != NodeAttrs::Use
)
1165 // For phi uses, compare predecessor blocks.
1166 const NodeAddr
<const PhiUseNode
*> TUA
= TA
;
1167 const NodeAddr
<const PhiUseNode
*> RUA
= RA
;
1168 return TUA
.Addr
->getPredecessor() == RUA
.Addr
->getPredecessor();
1171 RegisterRef RR
= RA
.Addr
->getRegRef(*this);
1172 if (IA
.Addr
->getKind() == NodeAttrs::Stmt
)
1173 return RA
.Addr
->getNextRef(RR
, RelatedStmt
, true, *this);
1174 return RA
.Addr
->getNextRef(RR
, RelatedPhi
, true, *this);
1177 // Find the next node related to RA in IA that satisfies condition P.
1178 // If such a node was found, return a pair where the second element is the
1179 // located node. If such a node does not exist, return a pair where the
1180 // first element is the element after which such a node should be inserted,
1181 // and the second element is a null-address.
1182 template <typename Predicate
>
1183 std::pair
<NodeAddr
<RefNode
*>,NodeAddr
<RefNode
*>>
1184 DataFlowGraph::locateNextRef(NodeAddr
<InstrNode
*> IA
, NodeAddr
<RefNode
*> RA
,
1185 Predicate P
) const {
1186 assert(IA
.Id
!= 0 && RA
.Id
!= 0);
1188 NodeAddr
<RefNode
*> NA
;
1189 NodeId Start
= RA
.Id
;
1191 NA
= getNextRelated(IA
, RA
);
1192 if (NA
.Id
== 0 || NA
.Id
== Start
)
1199 if (NA
.Id
!= 0 && NA
.Id
!= Start
)
1200 return std::make_pair(RA
, NA
);
1201 return std::make_pair(RA
, NodeAddr
<RefNode
*>());
1204 // Get the next shadow node in IA corresponding to RA, and optionally create
1205 // such a node if it does not exist.
1206 NodeAddr
<RefNode
*> DataFlowGraph::getNextShadow(NodeAddr
<InstrNode
*> IA
,
1207 NodeAddr
<RefNode
*> RA
, bool Create
) {
1208 assert(IA
.Id
!= 0 && RA
.Id
!= 0);
1210 uint16_t Flags
= RA
.Addr
->getFlags() | NodeAttrs::Shadow
;
1211 auto IsShadow
= [Flags
] (NodeAddr
<RefNode
*> TA
) -> bool {
1212 return TA
.Addr
->getFlags() == Flags
;
1214 auto Loc
= locateNextRef(IA
, RA
, IsShadow
);
1215 if (Loc
.second
.Id
!= 0 || !Create
)
1218 // Create a copy of RA and mark is as shadow.
1219 NodeAddr
<RefNode
*> NA
= cloneNode(RA
);
1220 NA
.Addr
->setFlags(Flags
| NodeAttrs::Shadow
);
1221 IA
.Addr
->addMemberAfter(Loc
.first
, NA
, *this);
1225 // Get the next shadow node in IA corresponding to RA. Return null-address
1226 // if such a node does not exist.
1227 NodeAddr
<RefNode
*> DataFlowGraph::getNextShadow(NodeAddr
<InstrNode
*> IA
,
1228 NodeAddr
<RefNode
*> RA
) const {
1229 assert(IA
.Id
!= 0 && RA
.Id
!= 0);
1230 uint16_t Flags
= RA
.Addr
->getFlags() | NodeAttrs::Shadow
;
1231 auto IsShadow
= [Flags
] (NodeAddr
<RefNode
*> TA
) -> bool {
1232 return TA
.Addr
->getFlags() == Flags
;
1234 return locateNextRef(IA
, RA
, IsShadow
).second
;
1237 // Create a new statement node in the block node BA that corresponds to
1238 // the machine instruction MI.
1239 void DataFlowGraph::buildStmt(NodeAddr
<BlockNode
*> BA
, MachineInstr
&In
) {
1240 NodeAddr
<StmtNode
*> SA
= newStmt(BA
, &In
);
1242 auto isCall
= [] (const MachineInstr
&In
) -> bool {
1246 if (In
.isBranch()) {
1247 for (const MachineOperand
&Op
: In
.operands())
1248 if (Op
.isGlobal() || Op
.isSymbol())
1250 // Assume indirect branches are calls. This is for the purpose of
1251 // keeping implicit operands, and so it won't hurt on intra-function
1252 // indirect branches.
1253 if (In
.isIndirectBranch())
1259 auto isDefUndef
= [this] (const MachineInstr
&In
, RegisterRef DR
) -> bool {
1260 // This instruction defines DR. Check if there is a use operand that
1261 // would make DR live on entry to the instruction.
1262 for (const MachineOperand
&Op
: In
.operands()) {
1263 if (!Op
.isReg() || Op
.getReg() == 0 || !Op
.isUse() || Op
.isUndef())
1265 RegisterRef UR
= makeRegRef(Op
);
1266 if (PRI
.alias(DR
, UR
))
1272 bool IsCall
= isCall(In
);
1273 unsigned NumOps
= In
.getNumOperands();
1275 // Avoid duplicate implicit defs. This will not detect cases of implicit
1276 // defs that define registers that overlap, but it is not clear how to
1277 // interpret that in the absence of explicit defs. Overlapping explicit
1278 // defs are likely illegal already.
1279 BitVector
DoneDefs(TRI
.getNumRegs());
1280 // Process explicit defs first.
1281 for (unsigned OpN
= 0; OpN
< NumOps
; ++OpN
) {
1282 MachineOperand
&Op
= In
.getOperand(OpN
);
1283 if (!Op
.isReg() || !Op
.isDef() || Op
.isImplicit())
1285 Register R
= Op
.getReg();
1286 if (!R
|| !Register::isPhysicalRegister(R
))
1288 uint16_t Flags
= NodeAttrs::None
;
1289 if (TOI
.isPreserving(In
, OpN
)) {
1290 Flags
|= NodeAttrs::Preserving
;
1291 // If the def is preserving, check if it is also undefined.
1292 if (isDefUndef(In
, makeRegRef(Op
)))
1293 Flags
|= NodeAttrs::Undef
;
1295 if (TOI
.isClobbering(In
, OpN
))
1296 Flags
|= NodeAttrs::Clobbering
;
1297 if (TOI
.isFixedReg(In
, OpN
))
1298 Flags
|= NodeAttrs::Fixed
;
1299 if (IsCall
&& Op
.isDead())
1300 Flags
|= NodeAttrs::Dead
;
1301 NodeAddr
<DefNode
*> DA
= newDef(SA
, Op
, Flags
);
1302 SA
.Addr
->addMember(DA
, *this);
1303 assert(!DoneDefs
.test(R
));
1307 // Process reg-masks (as clobbers).
1308 BitVector
DoneClobbers(TRI
.getNumRegs());
1309 for (unsigned OpN
= 0; OpN
< NumOps
; ++OpN
) {
1310 MachineOperand
&Op
= In
.getOperand(OpN
);
1311 if (!Op
.isRegMask())
1313 uint16_t Flags
= NodeAttrs::Clobbering
| NodeAttrs::Fixed
|
1315 NodeAddr
<DefNode
*> DA
= newDef(SA
, Op
, Flags
);
1316 SA
.Addr
->addMember(DA
, *this);
1317 // Record all clobbered registers in DoneDefs.
1318 const uint32_t *RM
= Op
.getRegMask();
1319 for (unsigned i
= 1, e
= TRI
.getNumRegs(); i
!= e
; ++i
)
1320 if (!(RM
[i
/32] & (1u << (i
%32))))
1321 DoneClobbers
.set(i
);
1324 // Process implicit defs, skipping those that have already been added
1326 for (unsigned OpN
= 0; OpN
< NumOps
; ++OpN
) {
1327 MachineOperand
&Op
= In
.getOperand(OpN
);
1328 if (!Op
.isReg() || !Op
.isDef() || !Op
.isImplicit())
1330 Register R
= Op
.getReg();
1331 if (!R
|| !Register::isPhysicalRegister(R
) || DoneDefs
.test(R
))
1333 RegisterRef RR
= makeRegRef(Op
);
1334 uint16_t Flags
= NodeAttrs::None
;
1335 if (TOI
.isPreserving(In
, OpN
)) {
1336 Flags
|= NodeAttrs::Preserving
;
1337 // If the def is preserving, check if it is also undefined.
1338 if (isDefUndef(In
, RR
))
1339 Flags
|= NodeAttrs::Undef
;
1341 if (TOI
.isClobbering(In
, OpN
))
1342 Flags
|= NodeAttrs::Clobbering
;
1343 if (TOI
.isFixedReg(In
, OpN
))
1344 Flags
|= NodeAttrs::Fixed
;
1345 if (IsCall
&& Op
.isDead()) {
1346 if (DoneClobbers
.test(R
))
1348 Flags
|= NodeAttrs::Dead
;
1350 NodeAddr
<DefNode
*> DA
= newDef(SA
, Op
, Flags
);
1351 SA
.Addr
->addMember(DA
, *this);
1355 for (unsigned OpN
= 0; OpN
< NumOps
; ++OpN
) {
1356 MachineOperand
&Op
= In
.getOperand(OpN
);
1357 if (!Op
.isReg() || !Op
.isUse())
1359 Register R
= Op
.getReg();
1360 if (!R
|| !Register::isPhysicalRegister(R
))
1362 uint16_t Flags
= NodeAttrs::None
;
1364 Flags
|= NodeAttrs::Undef
;
1365 if (TOI
.isFixedReg(In
, OpN
))
1366 Flags
|= NodeAttrs::Fixed
;
1367 NodeAddr
<UseNode
*> UA
= newUse(SA
, Op
, Flags
);
1368 SA
.Addr
->addMember(UA
, *this);
1372 // Scan all defs in the block node BA and record in PhiM the locations of
1373 // phi nodes corresponding to these defs.
1374 void DataFlowGraph::recordDefsForDF(BlockRefsMap
&PhiM
,
1375 NodeAddr
<BlockNode
*> BA
) {
1376 // Check all defs from block BA and record them in each block in BA's
1377 // iterated dominance frontier. This information will later be used to
1378 // create phi nodes.
1379 MachineBasicBlock
*BB
= BA
.Addr
->getCode();
1381 auto DFLoc
= MDF
.find(BB
);
1382 if (DFLoc
== MDF
.end() || DFLoc
->second
.empty())
1385 // Traverse all instructions in the block and collect the set of all
1386 // defined references. For each reference there will be a phi created
1387 // in the block's iterated dominance frontier.
1388 // This is done to make sure that each defined reference gets only one
1389 // phi node, even if it is defined multiple times.
1391 for (NodeAddr
<InstrNode
*> IA
: BA
.Addr
->members(*this))
1392 for (NodeAddr
<RefNode
*> RA
: IA
.Addr
->members_if(IsDef
, *this))
1393 Defs
.insert(RA
.Addr
->getRegRef(*this));
1395 // Calculate the iterated dominance frontier of BB.
1396 const MachineDominanceFrontier::DomSetType
&DF
= DFLoc
->second
;
1397 SetVector
<MachineBasicBlock
*> IDF(DF
.begin(), DF
.end());
1398 for (unsigned i
= 0; i
< IDF
.size(); ++i
) {
1399 auto F
= MDF
.find(IDF
[i
]);
1401 IDF
.insert(F
->second
.begin(), F
->second
.end());
1404 // Finally, add the set of defs to each block in the iterated dominance
1406 for (auto *DB
: IDF
) {
1407 NodeAddr
<BlockNode
*> DBA
= findBlock(DB
);
1408 PhiM
[DBA
.Id
].insert(Defs
.begin(), Defs
.end());
1412 // Given the locations of phi nodes in the map PhiM, create the phi nodes
1413 // that are located in the block node BA.
1414 void DataFlowGraph::buildPhis(BlockRefsMap
&PhiM
, RegisterSet
&AllRefs
,
1415 NodeAddr
<BlockNode
*> BA
) {
1416 // Check if this blocks has any DF defs, i.e. if there are any defs
1417 // that this block is in the iterated dominance frontier of.
1418 auto HasDF
= PhiM
.find(BA
.Id
);
1419 if (HasDF
== PhiM
.end() || HasDF
->second
.empty())
1422 // First, remove all R in Refs in such that there exists T in Refs
1423 // such that T covers R. In other words, only leave those refs that
1424 // are not covered by another ref (i.e. maximal with respect to covering).
1426 auto MaxCoverIn
= [this] (RegisterRef RR
, RegisterSet
&RRs
) -> RegisterRef
{
1427 for (RegisterRef I
: RRs
)
1428 if (I
!= RR
&& RegisterAggr::isCoverOf(I
, RR
, PRI
))
1434 for (RegisterRef I
: HasDF
->second
)
1435 MaxDF
.insert(MaxCoverIn(I
, HasDF
->second
));
1437 std::vector
<RegisterRef
> MaxRefs
;
1438 for (RegisterRef I
: MaxDF
)
1439 MaxRefs
.push_back(MaxCoverIn(I
, AllRefs
));
1441 // Now, for each R in MaxRefs, get the alias closure of R. If the closure
1442 // only has R in it, create a phi a def for R. Otherwise, create a phi,
1443 // and add a def for each S in the closure.
1445 // Sort the refs so that the phis will be created in a deterministic order.
1446 llvm::sort(MaxRefs
);
1447 // Remove duplicates.
1448 auto NewEnd
= std::unique(MaxRefs
.begin(), MaxRefs
.end());
1449 MaxRefs
.erase(NewEnd
, MaxRefs
.end());
1451 auto Aliased
= [this,&MaxRefs
](RegisterRef RR
,
1452 std::vector
<unsigned> &Closure
) -> bool {
1453 for (unsigned I
: Closure
)
1454 if (PRI
.alias(RR
, MaxRefs
[I
]))
1459 // Prepare a list of NodeIds of the block's predecessors.
1461 const MachineBasicBlock
*MBB
= BA
.Addr
->getCode();
1462 for (MachineBasicBlock
*PB
: MBB
->predecessors())
1463 Preds
.push_back(findBlock(PB
));
1465 while (!MaxRefs
.empty()) {
1466 // Put the first element in the closure, and then add all subsequent
1467 // elements from MaxRefs to it, if they alias at least one element
1468 // already in the closure.
1469 // ClosureIdx: vector of indices in MaxRefs of members of the closure.
1470 std::vector
<unsigned> ClosureIdx
= { 0 };
1471 for (unsigned i
= 1; i
!= MaxRefs
.size(); ++i
)
1472 if (Aliased(MaxRefs
[i
], ClosureIdx
))
1473 ClosureIdx
.push_back(i
);
1475 // Build a phi for the closure.
1476 unsigned CS
= ClosureIdx
.size();
1477 NodeAddr
<PhiNode
*> PA
= newPhi(BA
);
1480 for (unsigned X
= 0; X
!= CS
; ++X
) {
1481 RegisterRef RR
= MaxRefs
[ClosureIdx
[X
]];
1482 uint16_t PhiFlags
= NodeAttrs::PhiRef
| NodeAttrs::Preserving
;
1483 NodeAddr
<DefNode
*> DA
= newDef(PA
, RR
, PhiFlags
);
1484 PA
.Addr
->addMember(DA
, *this);
1487 for (NodeAddr
<BlockNode
*> PBA
: Preds
) {
1488 for (unsigned X
= 0; X
!= CS
; ++X
) {
1489 RegisterRef RR
= MaxRefs
[ClosureIdx
[X
]];
1490 NodeAddr
<PhiUseNode
*> PUA
= newPhiUse(PA
, RR
, PBA
);
1491 PA
.Addr
->addMember(PUA
, *this);
1495 // Erase from MaxRefs all elements in the closure.
1496 auto Begin
= MaxRefs
.begin();
1497 for (unsigned Idx
: llvm::reverse(ClosureIdx
))
1498 MaxRefs
.erase(Begin
+ Idx
);
1502 // Remove any unneeded phi nodes that were created during the build process.
1503 void DataFlowGraph::removeUnusedPhis() {
1504 // This will remove unused phis, i.e. phis where each def does not reach
1505 // any uses or other defs. This will not detect or remove circular phi
1506 // chains that are otherwise dead. Unused/dead phis are created during
1507 // the build process and this function is intended to remove these cases
1508 // that are easily determinable to be unnecessary.
1510 SetVector
<NodeId
> PhiQ
;
1511 for (NodeAddr
<BlockNode
*> BA
: Func
.Addr
->members(*this)) {
1512 for (auto P
: BA
.Addr
->members_if(IsPhi
, *this))
1516 static auto HasUsedDef
= [](NodeList
&Ms
) -> bool {
1517 for (NodeAddr
<NodeBase
*> M
: Ms
) {
1518 if (M
.Addr
->getKind() != NodeAttrs::Def
)
1520 NodeAddr
<DefNode
*> DA
= M
;
1521 if (DA
.Addr
->getReachedDef() != 0 || DA
.Addr
->getReachedUse() != 0)
1527 // Any phi, if it is removed, may affect other phis (make them dead).
1528 // For each removed phi, collect the potentially affected phis and add
1529 // them back to the queue.
1530 while (!PhiQ
.empty()) {
1531 auto PA
= addr
<PhiNode
*>(PhiQ
[0]);
1533 NodeList Refs
= PA
.Addr
->members(*this);
1534 if (HasUsedDef(Refs
))
1536 for (NodeAddr
<RefNode
*> RA
: Refs
) {
1537 if (NodeId RD
= RA
.Addr
->getReachingDef()) {
1538 auto RDA
= addr
<DefNode
*>(RD
);
1539 NodeAddr
<InstrNode
*> OA
= RDA
.Addr
->getOwner(*this);
1543 if (RA
.Addr
->isDef())
1544 unlinkDef(RA
, true);
1546 unlinkUse(RA
, true);
1548 NodeAddr
<BlockNode
*> BA
= PA
.Addr
->getOwner(*this);
1549 BA
.Addr
->removeMember(PA
, *this);
1553 // For a given reference node TA in an instruction node IA, connect the
1554 // reaching def of TA to the appropriate def node. Create any shadow nodes
1556 template <typename T
>
1557 void DataFlowGraph::linkRefUp(NodeAddr
<InstrNode
*> IA
, NodeAddr
<T
> TA
,
1561 RegisterRef RR
= TA
.Addr
->getRegRef(*this);
1564 // References from the def stack that have been examined so far.
1565 RegisterAggr
Defs(PRI
);
1567 for (auto I
= DS
.top(), E
= DS
.bottom(); I
!= E
; I
.down()) {
1568 RegisterRef QR
= I
->Addr
->getRegRef(*this);
1570 // Skip all defs that are aliased to any of the defs that we have already
1571 // seen. If this completes a cover of RR, stop the stack traversal.
1572 bool Alias
= Defs
.hasAliasOf(QR
);
1573 bool Cover
= Defs
.insert(QR
).hasCoverOf(RR
);
1580 // The reaching def.
1581 NodeAddr
<DefNode
*> RDA
= *I
;
1583 // Pick the reached node.
1587 // Mark the existing ref as "shadow" and create a new shadow.
1588 TAP
.Addr
->setFlags(TAP
.Addr
->getFlags() | NodeAttrs::Shadow
);
1589 TAP
= getNextShadow(IA
, TAP
, true);
1593 TAP
.Addr
->linkToDef(TAP
.Id
, RDA
);
1600 // Create data-flow links for all reference nodes in the statement node SA.
1601 template <typename Predicate
>
1602 void DataFlowGraph::linkStmtRefs(DefStackMap
&DefM
, NodeAddr
<StmtNode
*> SA
,
1608 // Link all nodes (upwards in the data-flow) with their reaching defs.
1609 for (NodeAddr
<RefNode
*> RA
: SA
.Addr
->members_if(P
, *this)) {
1610 uint16_t Kind
= RA
.Addr
->getKind();
1611 assert(Kind
== NodeAttrs::Def
|| Kind
== NodeAttrs::Use
);
1612 RegisterRef RR
= RA
.Addr
->getRegRef(*this);
1614 // Do not expect multiple defs of the same reference.
1615 assert(Kind
!= NodeAttrs::Def
|| !Defs
.count(RR
));
1619 auto F
= DefM
.find(RR
.Reg
);
1620 if (F
== DefM
.end())
1622 DefStack
&DS
= F
->second
;
1623 if (Kind
== NodeAttrs::Use
)
1624 linkRefUp
<UseNode
*>(SA
, RA
, DS
);
1625 else if (Kind
== NodeAttrs::Def
)
1626 linkRefUp
<DefNode
*>(SA
, RA
, DS
);
1628 llvm_unreachable("Unexpected node in instruction");
1632 // Create data-flow links for all instructions in the block node BA. This
1633 // will include updating any phi nodes in BA.
1634 void DataFlowGraph::linkBlockRefs(DefStackMap
&DefM
, NodeAddr
<BlockNode
*> BA
) {
1635 // Push block delimiters.
1636 markBlock(BA
.Id
, DefM
);
1638 auto IsClobber
= [] (NodeAddr
<RefNode
*> RA
) -> bool {
1639 return IsDef(RA
) && (RA
.Addr
->getFlags() & NodeAttrs::Clobbering
);
1641 auto IsNoClobber
= [] (NodeAddr
<RefNode
*> RA
) -> bool {
1642 return IsDef(RA
) && !(RA
.Addr
->getFlags() & NodeAttrs::Clobbering
);
1645 assert(BA
.Addr
&& "block node address is needed to create a data-flow link");
1646 // For each non-phi instruction in the block, link all the defs and uses
1647 // to their reaching defs. For any member of the block (including phis),
1648 // push the defs on the corresponding stacks.
1649 for (NodeAddr
<InstrNode
*> IA
: BA
.Addr
->members(*this)) {
1650 // Ignore phi nodes here. They will be linked part by part from the
1652 if (IA
.Addr
->getKind() == NodeAttrs::Stmt
) {
1653 linkStmtRefs(DefM
, IA
, IsUse
);
1654 linkStmtRefs(DefM
, IA
, IsClobber
);
1657 // Push the definitions on the stack.
1658 pushClobbers(IA
, DefM
);
1660 if (IA
.Addr
->getKind() == NodeAttrs::Stmt
)
1661 linkStmtRefs(DefM
, IA
, IsNoClobber
);
1666 // Recursively process all children in the dominator tree.
1667 MachineDomTreeNode
*N
= MDT
.getNode(BA
.Addr
->getCode());
1668 for (auto *I
: *N
) {
1669 MachineBasicBlock
*SB
= I
->getBlock();
1670 NodeAddr
<BlockNode
*> SBA
= findBlock(SB
);
1671 linkBlockRefs(DefM
, SBA
);
1674 // Link the phi uses from the successor blocks.
1675 auto IsUseForBA
= [BA
](NodeAddr
<NodeBase
*> NA
) -> bool {
1676 if (NA
.Addr
->getKind() != NodeAttrs::Use
)
1678 assert(NA
.Addr
->getFlags() & NodeAttrs::PhiRef
);
1679 NodeAddr
<PhiUseNode
*> PUA
= NA
;
1680 return PUA
.Addr
->getPredecessor() == BA
.Id
;
1683 RegisterSet EHLiveIns
= getLandingPadLiveIns();
1684 MachineBasicBlock
*MBB
= BA
.Addr
->getCode();
1686 for (MachineBasicBlock
*SB
: MBB
->successors()) {
1687 bool IsEHPad
= SB
->isEHPad();
1688 NodeAddr
<BlockNode
*> SBA
= findBlock(SB
);
1689 for (NodeAddr
<InstrNode
*> IA
: SBA
.Addr
->members_if(IsPhi
, *this)) {
1690 // Do not link phi uses for landing pad live-ins.
1692 // Find what register this phi is for.
1693 NodeAddr
<RefNode
*> RA
= IA
.Addr
->getFirstMember(*this);
1695 if (EHLiveIns
.count(RA
.Addr
->getRegRef(*this)))
1698 // Go over each phi use associated with MBB, and link it.
1699 for (auto U
: IA
.Addr
->members_if(IsUseForBA
, *this)) {
1700 NodeAddr
<PhiUseNode
*> PUA
= U
;
1701 RegisterRef RR
= PUA
.Addr
->getRegRef(*this);
1702 linkRefUp
<UseNode
*>(IA
, PUA
, DefM
[RR
.Reg
]);
1707 // Pop all defs from this block from the definition stacks.
1708 releaseBlock(BA
.Id
, DefM
);
1711 // Remove the use node UA from any data-flow and structural links.
1712 void DataFlowGraph::unlinkUseDF(NodeAddr
<UseNode
*> UA
) {
1713 NodeId RD
= UA
.Addr
->getReachingDef();
1714 NodeId Sib
= UA
.Addr
->getSibling();
1721 auto RDA
= addr
<DefNode
*>(RD
);
1722 auto TA
= addr
<UseNode
*>(RDA
.Addr
->getReachedUse());
1723 if (TA
.Id
== UA
.Id
) {
1724 RDA
.Addr
->setReachedUse(Sib
);
1728 while (TA
.Id
!= 0) {
1729 NodeId S
= TA
.Addr
->getSibling();
1731 TA
.Addr
->setSibling(UA
.Addr
->getSibling());
1734 TA
= addr
<UseNode
*>(S
);
1738 // Remove the def node DA from any data-flow and structural links.
1739 void DataFlowGraph::unlinkDefDF(NodeAddr
<DefNode
*> DA
) {
1747 // ... -- | DA | -- ... -- 0 : sibling chain of DA
1752 // | ... : Siblings (defs)
1756 // ... : sibling chain of reached uses
1758 NodeId RD
= DA
.Addr
->getReachingDef();
1760 // Visit all siblings of the reached def and reset their reaching defs.
1761 // Also, defs reached by DA are now "promoted" to being reached by RD,
1762 // so all of them will need to be spliced into the sibling chain where
1764 auto getAllNodes
= [this] (NodeId N
) -> NodeList
{
1767 auto RA
= addr
<RefNode
*>(N
);
1768 // Keep the nodes in the exact sibling order.
1770 N
= RA
.Addr
->getSibling();
1774 NodeList ReachedDefs
= getAllNodes(DA
.Addr
->getReachedDef());
1775 NodeList ReachedUses
= getAllNodes(DA
.Addr
->getReachedUse());
1778 for (NodeAddr
<RefNode
*> I
: ReachedDefs
)
1779 I
.Addr
->setSibling(0);
1780 for (NodeAddr
<RefNode
*> I
: ReachedUses
)
1781 I
.Addr
->setSibling(0);
1783 for (NodeAddr
<DefNode
*> I
: ReachedDefs
)
1784 I
.Addr
->setReachingDef(RD
);
1785 for (NodeAddr
<UseNode
*> I
: ReachedUses
)
1786 I
.Addr
->setReachingDef(RD
);
1788 NodeId Sib
= DA
.Addr
->getSibling();
1794 // Update the reaching def node and remove DA from the sibling list.
1795 auto RDA
= addr
<DefNode
*>(RD
);
1796 auto TA
= addr
<DefNode
*>(RDA
.Addr
->getReachedDef());
1797 if (TA
.Id
== DA
.Id
) {
1798 // If DA is the first reached def, just update the RD's reached def
1799 // to the DA's sibling.
1800 RDA
.Addr
->setReachedDef(Sib
);
1802 // Otherwise, traverse the sibling list of the reached defs and remove
1804 while (TA
.Id
!= 0) {
1805 NodeId S
= TA
.Addr
->getSibling();
1807 TA
.Addr
->setSibling(Sib
);
1810 TA
= addr
<DefNode
*>(S
);
1814 // Splice the DA's reached defs into the RDA's reached def chain.
1815 if (!ReachedDefs
.empty()) {
1816 auto Last
= NodeAddr
<DefNode
*>(ReachedDefs
.back());
1817 Last
.Addr
->setSibling(RDA
.Addr
->getReachedDef());
1818 RDA
.Addr
->setReachedDef(ReachedDefs
.front().Id
);
1820 // Splice the DA's reached uses into the RDA's reached use chain.
1821 if (!ReachedUses
.empty()) {
1822 auto Last
= NodeAddr
<UseNode
*>(ReachedUses
.back());
1823 Last
.Addr
->setSibling(RDA
.Addr
->getReachedUse());
1824 RDA
.Addr
->setReachedUse(ReachedUses
.front().Id
);