1 //===- DetectDeadLanes.cpp - SubRegister Lane Usage Analysis --*- C++ -*---===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// Analysis that tracks defined/used subregister lanes across COPY instructions
11 /// and instructions that get lowered to a COPY (PHI, REG_SEQUENCE,
12 /// INSERT_SUBREG, EXTRACT_SUBREG).
13 /// The information is used to detect dead definitions and the usage of
14 /// (completely) undefined values and mark the operands as such.
15 /// This pass is necessary because the dead/undef status is not obvious anymore
16 /// when subregisters are involved.
19 /// %0 = some definition
21 /// %2 = REG_SEQUENCE %0, sub0, %1, sub1
22 /// %3 = EXTRACT_SUBREG %2, sub1
24 /// The %0 definition is dead and %3 contains an undefined value.
26 //===----------------------------------------------------------------------===//
31 #include "llvm/ADT/BitVector.h"
32 #include "llvm/ADT/SetVector.h"
33 #include "llvm/CodeGen/MachineFunctionPass.h"
34 #include "llvm/CodeGen/MachineRegisterInfo.h"
35 #include "llvm/CodeGen/Passes.h"
36 #include "llvm/CodeGen/TargetRegisterInfo.h"
37 #include "llvm/CodeGen/TargetSubtargetInfo.h"
38 #include "llvm/InitializePasses.h"
39 #include "llvm/Pass.h"
40 #include "llvm/PassRegistry.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/raw_ostream.h"
46 #define DEBUG_TYPE "detect-dead-lanes"
50 /// Contains a bitmask of which lanes of a given virtual register are
51 /// defined and which ones are actually used.
53 LaneBitmask UsedLanes
;
54 LaneBitmask DefinedLanes
;
57 class DetectDeadLanes
: public MachineFunctionPass
{
59 bool runOnMachineFunction(MachineFunction
&MF
) override
;
62 DetectDeadLanes() : MachineFunctionPass(ID
) {}
64 StringRef
getPassName() const override
{ return "Detect Dead Lanes"; }
66 void getAnalysisUsage(AnalysisUsage
&AU
) const override
{
68 MachineFunctionPass::getAnalysisUsage(AU
);
72 /// Add used lane bits on the register used by operand \p MO. This translates
73 /// the bitmask based on the operands subregister, and puts the register into
74 /// the worklist if any new bits were added.
75 void addUsedLanesOnOperand(const MachineOperand
&MO
, LaneBitmask UsedLanes
);
77 /// Given a bitmask \p UsedLanes for the used lanes on a def output of a
78 /// COPY-like instruction determine the lanes used on the use operands
79 /// and call addUsedLanesOnOperand() for them.
80 void transferUsedLanesStep(const MachineInstr
&MI
, LaneBitmask UsedLanes
);
82 /// Given a use regiser operand \p Use and a mask of defined lanes, check
83 /// if the operand belongs to a lowersToCopies() instruction, transfer the
84 /// mask to the def and put the instruction into the worklist.
85 void transferDefinedLanesStep(const MachineOperand
&Use
,
86 LaneBitmask DefinedLanes
);
88 /// Given a mask \p DefinedLanes of lanes defined at operand \p OpNum
89 /// of COPY-like instruction, determine which lanes are defined at the output
91 LaneBitmask
transferDefinedLanes(const MachineOperand
&Def
, unsigned OpNum
,
92 LaneBitmask DefinedLanes
) const;
94 /// Given a mask \p UsedLanes used from the output of instruction \p MI
95 /// determine which lanes are used from operand \p MO of this instruction.
96 LaneBitmask
transferUsedLanes(const MachineInstr
&MI
, LaneBitmask UsedLanes
,
97 const MachineOperand
&MO
) const;
99 bool runOnce(MachineFunction
&MF
);
101 LaneBitmask
determineInitialDefinedLanes(unsigned Reg
);
102 LaneBitmask
determineInitialUsedLanes(unsigned Reg
);
104 bool isUndefRegAtInput(const MachineOperand
&MO
,
105 const VRegInfo
&RegInfo
) const;
107 bool isUndefInput(const MachineOperand
&MO
, bool *CrossCopy
) const;
109 const MachineRegisterInfo
*MRI
;
110 const TargetRegisterInfo
*TRI
;
112 void PutInWorklist(unsigned RegIdx
) {
113 if (WorklistMembers
.test(RegIdx
))
115 WorklistMembers
.set(RegIdx
);
116 Worklist
.push_back(RegIdx
);
120 /// Worklist containing virtreg indexes.
121 std::deque
<unsigned> Worklist
;
122 BitVector WorklistMembers
;
123 /// This bitvector is set for each vreg index where the vreg is defined
124 /// by an instruction where lowersToCopies()==true.
125 BitVector DefinedByCopy
;
128 } // end anonymous namespace
130 char DetectDeadLanes::ID
= 0;
131 char &llvm::DetectDeadLanesID
= DetectDeadLanes::ID
;
133 INITIALIZE_PASS(DetectDeadLanes
, DEBUG_TYPE
, "Detect Dead Lanes", false, false)
135 /// Returns true if \p MI will get lowered to a series of COPY instructions.
136 /// We call this a COPY-like instruction.
137 static bool lowersToCopies(const MachineInstr
&MI
) {
138 // Note: We could support instructions with MCInstrDesc::isRegSequenceLike(),
139 // isExtractSubRegLike(), isInsertSubregLike() in the future even though they
140 // are not lowered to a COPY.
141 switch (MI
.getOpcode()) {
142 case TargetOpcode::COPY
:
143 case TargetOpcode::PHI
:
144 case TargetOpcode::INSERT_SUBREG
:
145 case TargetOpcode::REG_SEQUENCE
:
146 case TargetOpcode::EXTRACT_SUBREG
:
152 static bool isCrossCopy(const MachineRegisterInfo
&MRI
,
153 const MachineInstr
&MI
,
154 const TargetRegisterClass
*DstRC
,
155 const MachineOperand
&MO
) {
156 assert(lowersToCopies(MI
));
157 Register SrcReg
= MO
.getReg();
158 const TargetRegisterClass
*SrcRC
= MRI
.getRegClass(SrcReg
);
162 unsigned SrcSubIdx
= MO
.getSubReg();
164 const TargetRegisterInfo
&TRI
= *MRI
.getTargetRegisterInfo();
165 unsigned DstSubIdx
= 0;
166 switch (MI
.getOpcode()) {
167 case TargetOpcode::INSERT_SUBREG
:
168 if (MI
.getOperandNo(&MO
) == 2)
169 DstSubIdx
= MI
.getOperand(3).getImm();
171 case TargetOpcode::REG_SEQUENCE
: {
172 unsigned OpNum
= MI
.getOperandNo(&MO
);
173 DstSubIdx
= MI
.getOperand(OpNum
+1).getImm();
176 case TargetOpcode::EXTRACT_SUBREG
: {
177 unsigned SubReg
= MI
.getOperand(2).getImm();
178 SrcSubIdx
= TRI
.composeSubRegIndices(SubReg
, SrcSubIdx
);
182 unsigned PreA
, PreB
; // Unused.
183 if (SrcSubIdx
&& DstSubIdx
)
184 return !TRI
.getCommonSuperRegClass(SrcRC
, SrcSubIdx
, DstRC
, DstSubIdx
, PreA
,
187 return !TRI
.getMatchingSuperRegClass(SrcRC
, DstRC
, SrcSubIdx
);
189 return !TRI
.getMatchingSuperRegClass(DstRC
, SrcRC
, DstSubIdx
);
190 return !TRI
.getCommonSubClass(SrcRC
, DstRC
);
193 void DetectDeadLanes::addUsedLanesOnOperand(const MachineOperand
&MO
,
194 LaneBitmask UsedLanes
) {
197 Register MOReg
= MO
.getReg();
198 if (!Register::isVirtualRegister(MOReg
))
201 unsigned MOSubReg
= MO
.getSubReg();
203 UsedLanes
= TRI
->composeSubRegIndexLaneMask(MOSubReg
, UsedLanes
);
204 UsedLanes
&= MRI
->getMaxLaneMaskForVReg(MOReg
);
206 unsigned MORegIdx
= Register::virtReg2Index(MOReg
);
207 VRegInfo
&MORegInfo
= VRegInfos
[MORegIdx
];
208 LaneBitmask PrevUsedLanes
= MORegInfo
.UsedLanes
;
209 // Any change at all?
210 if ((UsedLanes
& ~PrevUsedLanes
).none())
213 // Set UsedLanes and remember instruction for further propagation.
214 MORegInfo
.UsedLanes
= PrevUsedLanes
| UsedLanes
;
215 if (DefinedByCopy
.test(MORegIdx
))
216 PutInWorklist(MORegIdx
);
219 void DetectDeadLanes::transferUsedLanesStep(const MachineInstr
&MI
,
220 LaneBitmask UsedLanes
) {
221 for (const MachineOperand
&MO
: MI
.uses()) {
222 if (!MO
.isReg() || !Register::isVirtualRegister(MO
.getReg()))
224 LaneBitmask UsedOnMO
= transferUsedLanes(MI
, UsedLanes
, MO
);
225 addUsedLanesOnOperand(MO
, UsedOnMO
);
229 LaneBitmask
DetectDeadLanes::transferUsedLanes(const MachineInstr
&MI
,
230 LaneBitmask UsedLanes
,
231 const MachineOperand
&MO
) const {
232 unsigned OpNum
= MI
.getOperandNo(&MO
);
233 assert(lowersToCopies(MI
) &&
234 DefinedByCopy
[Register::virtReg2Index(MI
.getOperand(0).getReg())]);
236 switch (MI
.getOpcode()) {
237 case TargetOpcode::COPY
:
238 case TargetOpcode::PHI
:
240 case TargetOpcode::REG_SEQUENCE
: {
241 assert(OpNum
% 2 == 1);
242 unsigned SubIdx
= MI
.getOperand(OpNum
+ 1).getImm();
243 return TRI
->reverseComposeSubRegIndexLaneMask(SubIdx
, UsedLanes
);
245 case TargetOpcode::INSERT_SUBREG
: {
246 unsigned SubIdx
= MI
.getOperand(3).getImm();
247 LaneBitmask MO2UsedLanes
=
248 TRI
->reverseComposeSubRegIndexLaneMask(SubIdx
, UsedLanes
);
252 const MachineOperand
&Def
= MI
.getOperand(0);
253 Register DefReg
= Def
.getReg();
254 const TargetRegisterClass
*RC
= MRI
->getRegClass(DefReg
);
255 LaneBitmask MO1UsedLanes
;
256 if (RC
->CoveredBySubRegs
)
257 MO1UsedLanes
= UsedLanes
& ~TRI
->getSubRegIndexLaneMask(SubIdx
);
259 MO1UsedLanes
= RC
->LaneMask
;
264 case TargetOpcode::EXTRACT_SUBREG
: {
266 unsigned SubIdx
= MI
.getOperand(2).getImm();
267 return TRI
->composeSubRegIndexLaneMask(SubIdx
, UsedLanes
);
270 llvm_unreachable("function must be called with COPY-like instruction");
274 void DetectDeadLanes::transferDefinedLanesStep(const MachineOperand
&Use
,
275 LaneBitmask DefinedLanes
) {
278 // Check whether the operand writes a vreg and is part of a COPY-like
280 const MachineInstr
&MI
= *Use
.getParent();
281 if (MI
.getDesc().getNumDefs() != 1)
283 // FIXME: PATCHPOINT instructions announce a Def that does not always exist,
284 // they really need to be modeled differently!
285 if (MI
.getOpcode() == TargetOpcode::PATCHPOINT
)
287 const MachineOperand
&Def
= *MI
.defs().begin();
288 Register DefReg
= Def
.getReg();
289 if (!Register::isVirtualRegister(DefReg
))
291 unsigned DefRegIdx
= Register::virtReg2Index(DefReg
);
292 if (!DefinedByCopy
.test(DefRegIdx
))
295 unsigned OpNum
= MI
.getOperandNo(&Use
);
297 TRI
->reverseComposeSubRegIndexLaneMask(Use
.getSubReg(), DefinedLanes
);
298 DefinedLanes
= transferDefinedLanes(Def
, OpNum
, DefinedLanes
);
300 VRegInfo
&RegInfo
= VRegInfos
[DefRegIdx
];
301 LaneBitmask PrevDefinedLanes
= RegInfo
.DefinedLanes
;
302 // Any change at all?
303 if ((DefinedLanes
& ~PrevDefinedLanes
).none())
306 RegInfo
.DefinedLanes
= PrevDefinedLanes
| DefinedLanes
;
307 PutInWorklist(DefRegIdx
);
310 LaneBitmask
DetectDeadLanes::transferDefinedLanes(const MachineOperand
&Def
,
311 unsigned OpNum
, LaneBitmask DefinedLanes
) const {
312 const MachineInstr
&MI
= *Def
.getParent();
313 // Translate DefinedLanes if necessary.
314 switch (MI
.getOpcode()) {
315 case TargetOpcode::REG_SEQUENCE
: {
316 unsigned SubIdx
= MI
.getOperand(OpNum
+ 1).getImm();
317 DefinedLanes
= TRI
->composeSubRegIndexLaneMask(SubIdx
, DefinedLanes
);
318 DefinedLanes
&= TRI
->getSubRegIndexLaneMask(SubIdx
);
321 case TargetOpcode::INSERT_SUBREG
: {
322 unsigned SubIdx
= MI
.getOperand(3).getImm();
324 DefinedLanes
= TRI
->composeSubRegIndexLaneMask(SubIdx
, DefinedLanes
);
325 DefinedLanes
&= TRI
->getSubRegIndexLaneMask(SubIdx
);
327 assert(OpNum
== 1 && "INSERT_SUBREG must have two operands");
328 // Ignore lanes defined by operand 2.
329 DefinedLanes
&= ~TRI
->getSubRegIndexLaneMask(SubIdx
);
333 case TargetOpcode::EXTRACT_SUBREG
: {
334 unsigned SubIdx
= MI
.getOperand(2).getImm();
335 assert(OpNum
== 1 && "EXTRACT_SUBREG must have one register operand only");
336 DefinedLanes
= TRI
->reverseComposeSubRegIndexLaneMask(SubIdx
, DefinedLanes
);
339 case TargetOpcode::COPY
:
340 case TargetOpcode::PHI
:
343 llvm_unreachable("function must be called with COPY-like instruction");
346 assert(Def
.getSubReg() == 0 &&
347 "Should not have subregister defs in machine SSA phase");
348 DefinedLanes
&= MRI
->getMaxLaneMaskForVReg(Def
.getReg());
352 LaneBitmask
DetectDeadLanes::determineInitialDefinedLanes(unsigned Reg
) {
353 // Live-In or unused registers have no definition but are considered fully
355 if (!MRI
->hasOneDef(Reg
))
356 return LaneBitmask::getAll();
358 const MachineOperand
&Def
= *MRI
->def_begin(Reg
);
359 const MachineInstr
&DefMI
= *Def
.getParent();
360 if (lowersToCopies(DefMI
)) {
361 // Start optimisatically with no used or defined lanes for copy
362 // instructions. The following dataflow analysis will add more bits.
363 unsigned RegIdx
= Register::virtReg2Index(Reg
);
364 DefinedByCopy
.set(RegIdx
);
365 PutInWorklist(RegIdx
);
368 return LaneBitmask::getNone();
370 // COPY/PHI can copy across unrelated register classes (example: float/int)
371 // with incompatible subregister structure. Do not include these in the
372 // dataflow analysis since we cannot transfer lanemasks in a meaningful way.
373 const TargetRegisterClass
*DefRC
= MRI
->getRegClass(Reg
);
375 // Determine initially DefinedLanes.
376 LaneBitmask DefinedLanes
;
377 for (const MachineOperand
&MO
: DefMI
.uses()) {
378 if (!MO
.isReg() || !MO
.readsReg())
380 Register MOReg
= MO
.getReg();
384 LaneBitmask MODefinedLanes
;
385 if (Register::isPhysicalRegister(MOReg
)) {
386 MODefinedLanes
= LaneBitmask::getAll();
387 } else if (isCrossCopy(*MRI
, DefMI
, DefRC
, MO
)) {
388 MODefinedLanes
= LaneBitmask::getAll();
390 assert(Register::isVirtualRegister(MOReg
));
391 if (MRI
->hasOneDef(MOReg
)) {
392 const MachineOperand
&MODef
= *MRI
->def_begin(MOReg
);
393 const MachineInstr
&MODefMI
= *MODef
.getParent();
394 // Bits from copy-like operations will be added later.
395 if (lowersToCopies(MODefMI
) || MODefMI
.isImplicitDef())
398 unsigned MOSubReg
= MO
.getSubReg();
399 MODefinedLanes
= MRI
->getMaxLaneMaskForVReg(MOReg
);
400 MODefinedLanes
= TRI
->reverseComposeSubRegIndexLaneMask(
401 MOSubReg
, MODefinedLanes
);
404 unsigned OpNum
= DefMI
.getOperandNo(&MO
);
405 DefinedLanes
|= transferDefinedLanes(Def
, OpNum
, MODefinedLanes
);
409 if (DefMI
.isImplicitDef() || Def
.isDead())
410 return LaneBitmask::getNone();
412 assert(Def
.getSubReg() == 0 &&
413 "Should not have subregister defs in machine SSA phase");
414 return MRI
->getMaxLaneMaskForVReg(Reg
);
417 LaneBitmask
DetectDeadLanes::determineInitialUsedLanes(unsigned Reg
) {
418 LaneBitmask UsedLanes
= LaneBitmask::getNone();
419 for (const MachineOperand
&MO
: MRI
->use_nodbg_operands(Reg
)) {
423 const MachineInstr
&UseMI
= *MO
.getParent();
427 unsigned SubReg
= MO
.getSubReg();
428 if (lowersToCopies(UseMI
)) {
429 assert(UseMI
.getDesc().getNumDefs() == 1);
430 const MachineOperand
&Def
= *UseMI
.defs().begin();
431 Register DefReg
= Def
.getReg();
432 // The used lanes of COPY-like instruction operands are determined by the
433 // following dataflow analysis.
434 if (Register::isVirtualRegister(DefReg
)) {
435 // But ignore copies across incompatible register classes.
436 bool CrossCopy
= false;
437 if (lowersToCopies(UseMI
)) {
438 const TargetRegisterClass
*DstRC
= MRI
->getRegClass(DefReg
);
439 CrossCopy
= isCrossCopy(*MRI
, UseMI
, DstRC
, MO
);
441 LLVM_DEBUG(dbgs() << "Copy across incompatible classes: " << UseMI
);
449 // Shortcut: All lanes are used.
451 return MRI
->getMaxLaneMaskForVReg(Reg
);
453 UsedLanes
|= TRI
->getSubRegIndexLaneMask(SubReg
);
458 bool DetectDeadLanes::isUndefRegAtInput(const MachineOperand
&MO
,
459 const VRegInfo
&RegInfo
) const {
460 unsigned SubReg
= MO
.getSubReg();
461 LaneBitmask Mask
= TRI
->getSubRegIndexLaneMask(SubReg
);
462 return (RegInfo
.DefinedLanes
& RegInfo
.UsedLanes
& Mask
).none();
465 bool DetectDeadLanes::isUndefInput(const MachineOperand
&MO
,
466 bool *CrossCopy
) const {
469 const MachineInstr
&MI
= *MO
.getParent();
470 if (!lowersToCopies(MI
))
472 const MachineOperand
&Def
= MI
.getOperand(0);
473 Register DefReg
= Def
.getReg();
474 if (!Register::isVirtualRegister(DefReg
))
476 unsigned DefRegIdx
= Register::virtReg2Index(DefReg
);
477 if (!DefinedByCopy
.test(DefRegIdx
))
480 const VRegInfo
&DefRegInfo
= VRegInfos
[DefRegIdx
];
481 LaneBitmask UsedLanes
= transferUsedLanes(MI
, DefRegInfo
.UsedLanes
, MO
);
485 Register MOReg
= MO
.getReg();
486 if (Register::isVirtualRegister(MOReg
)) {
487 const TargetRegisterClass
*DstRC
= MRI
->getRegClass(DefReg
);
488 *CrossCopy
= isCrossCopy(*MRI
, MI
, DstRC
, MO
);
493 bool DetectDeadLanes::runOnce(MachineFunction
&MF
) {
494 // First pass: Populate defs/uses of vregs with initial values
495 unsigned NumVirtRegs
= MRI
->getNumVirtRegs();
496 for (unsigned RegIdx
= 0; RegIdx
< NumVirtRegs
; ++RegIdx
) {
497 unsigned Reg
= Register::index2VirtReg(RegIdx
);
499 // Determine used/defined lanes and add copy instructions to worklist.
500 VRegInfo
&Info
= VRegInfos
[RegIdx
];
501 Info
.DefinedLanes
= determineInitialDefinedLanes(Reg
);
502 Info
.UsedLanes
= determineInitialUsedLanes(Reg
);
505 // Iterate as long as defined lanes/used lanes keep changing.
506 while (!Worklist
.empty()) {
507 unsigned RegIdx
= Worklist
.front();
508 Worklist
.pop_front();
509 WorklistMembers
.reset(RegIdx
);
510 VRegInfo
&Info
= VRegInfos
[RegIdx
];
511 unsigned Reg
= Register::index2VirtReg(RegIdx
);
513 // Transfer UsedLanes to operands of DefMI (backwards dataflow).
514 MachineOperand
&Def
= *MRI
->def_begin(Reg
);
515 const MachineInstr
&MI
= *Def
.getParent();
516 transferUsedLanesStep(MI
, Info
.UsedLanes
);
517 // Transfer DefinedLanes to users of Reg (forward dataflow).
518 for (const MachineOperand
&MO
: MRI
->use_nodbg_operands(Reg
))
519 transferDefinedLanesStep(MO
, Info
.DefinedLanes
);
522 LLVM_DEBUG(dbgs() << "Defined/Used lanes:\n"; for (unsigned RegIdx
= 0;
523 RegIdx
< NumVirtRegs
;
525 unsigned Reg
= Register::index2VirtReg(RegIdx
);
526 const VRegInfo
&Info
= VRegInfos
[RegIdx
];
527 dbgs() << printReg(Reg
, nullptr)
528 << " Used: " << PrintLaneMask(Info
.UsedLanes
)
529 << " Def: " << PrintLaneMask(Info
.DefinedLanes
) << '\n';
533 // Mark operands as dead/unused.
534 for (MachineBasicBlock
&MBB
: MF
) {
535 for (MachineInstr
&MI
: MBB
) {
536 for (MachineOperand
&MO
: MI
.operands()) {
539 Register Reg
= MO
.getReg();
540 if (!Register::isVirtualRegister(Reg
))
542 unsigned RegIdx
= Register::virtReg2Index(Reg
);
543 const VRegInfo
&RegInfo
= VRegInfos
[RegIdx
];
544 if (MO
.isDef() && !MO
.isDead() && RegInfo
.UsedLanes
.none()) {
546 << "Marking operand '" << MO
<< "' as dead in " << MI
);
550 bool CrossCopy
= false;
551 if (isUndefRegAtInput(MO
, RegInfo
)) {
553 << "Marking operand '" << MO
<< "' as undef in " << MI
);
555 } else if (isUndefInput(MO
, &CrossCopy
)) {
557 << "Marking operand '" << MO
<< "' as undef in " << MI
);
570 bool DetectDeadLanes::runOnMachineFunction(MachineFunction
&MF
) {
571 // Don't bother if we won't track subregister liveness later. This pass is
572 // required for correctness if subregister liveness is enabled because the
573 // register coalescer cannot deal with hidden dead defs. However without
574 // subregister liveness enabled, the expected benefits of this pass are small
575 // so we safe the compile time.
576 MRI
= &MF
.getRegInfo();
577 if (!MRI
->subRegLivenessEnabled()) {
578 LLVM_DEBUG(dbgs() << "Skipping Detect dead lanes pass\n");
582 TRI
= MRI
->getTargetRegisterInfo();
584 unsigned NumVirtRegs
= MRI
->getNumVirtRegs();
585 VRegInfos
= new VRegInfo
[NumVirtRegs
];
586 WorklistMembers
.resize(NumVirtRegs
);
587 DefinedByCopy
.resize(NumVirtRegs
);
594 DefinedByCopy
.clear();
595 WorklistMembers
.clear();