1 //===- ShrinkWrap.cpp - Compute safe point for prolog/epilog insertion ----===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This pass looks for safe point where the prologue and epilogue can be
11 // The safe point for the prologue (resp. epilogue) is called Save
13 // A point is safe for prologue (resp. epilogue) if and only if
14 // it 1) dominates (resp. post-dominates) all the frame related operations and
15 // between 2) two executions of the Save (resp. Restore) point there is an
16 // execution of the Restore (resp. Save) point.
18 // For instance, the following points are safe:
19 // for (int i = 0; i < 10; ++i) {
24 // Indeed, the execution looks like Save -> Restore -> Save -> Restore ...
25 // And the following points are not:
26 // for (int i = 0; i < 10; ++i) {
30 // for (int i = 0; i < 10; ++i) {
34 // Indeed, the execution looks like Save -> Save -> ... -> Restore -> Restore.
36 // This pass also ensures that the safe points are 3) cheaper than the regular
37 // entry and exits blocks.
39 // Property #1 is ensured via the use of MachineDominatorTree and
40 // MachinePostDominatorTree.
41 // Property #2 is ensured via property #1 and MachineLoopInfo, i.e., both
42 // points must be in the same loop.
43 // Property #3 is ensured via the MachineBlockFrequencyInfo.
45 // If this pass found points matching all these properties, then
46 // MachineFrameInfo is updated with this information.
48 //===----------------------------------------------------------------------===//
50 #include "llvm/ADT/BitVector.h"
51 #include "llvm/ADT/PostOrderIterator.h"
52 #include "llvm/ADT/SetVector.h"
53 #include "llvm/ADT/SmallVector.h"
54 #include "llvm/ADT/Statistic.h"
55 #include "llvm/Analysis/CFG.h"
56 #include "llvm/CodeGen/MachineBasicBlock.h"
57 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
58 #include "llvm/CodeGen/MachineDominators.h"
59 #include "llvm/CodeGen/MachineFrameInfo.h"
60 #include "llvm/CodeGen/MachineFunction.h"
61 #include "llvm/CodeGen/MachineFunctionPass.h"
62 #include "llvm/CodeGen/MachineInstr.h"
63 #include "llvm/CodeGen/MachineLoopInfo.h"
64 #include "llvm/CodeGen/MachineOperand.h"
65 #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
66 #include "llvm/CodeGen/MachinePostDominators.h"
67 #include "llvm/CodeGen/RegisterClassInfo.h"
68 #include "llvm/CodeGen/RegisterScavenging.h"
69 #include "llvm/CodeGen/TargetFrameLowering.h"
70 #include "llvm/CodeGen/TargetInstrInfo.h"
71 #include "llvm/CodeGen/TargetLowering.h"
72 #include "llvm/CodeGen/TargetRegisterInfo.h"
73 #include "llvm/CodeGen/TargetSubtargetInfo.h"
74 #include "llvm/IR/Attributes.h"
75 #include "llvm/IR/Function.h"
76 #include "llvm/InitializePasses.h"
77 #include "llvm/MC/MCAsmInfo.h"
78 #include "llvm/Pass.h"
79 #include "llvm/Support/CommandLine.h"
80 #include "llvm/Support/Debug.h"
81 #include "llvm/Support/ErrorHandling.h"
82 #include "llvm/Support/raw_ostream.h"
83 #include "llvm/Target/TargetMachine.h"
90 #define DEBUG_TYPE "shrink-wrap"
92 STATISTIC(NumFunc
, "Number of functions");
93 STATISTIC(NumCandidates
, "Number of shrink-wrapping candidates");
94 STATISTIC(NumCandidatesDropped
,
95 "Number of shrink-wrapping candidates dropped because of frequency");
97 static cl::opt
<cl::boolOrDefault
>
98 EnableShrinkWrapOpt("enable-shrink-wrap", cl::Hidden
,
99 cl::desc("enable the shrink-wrapping pass"));
103 /// Class to determine where the safe point to insert the
104 /// prologue and epilogue are.
105 /// Unlike the paper from Fred C. Chow, PLDI'88, that introduces the
106 /// shrink-wrapping term for prologue/epilogue placement, this pass
107 /// does not rely on expensive data-flow analysis. Instead we use the
108 /// dominance properties and loop information to decide which point
109 /// are safe for such insertion.
110 class ShrinkWrap
: public MachineFunctionPass
{
111 /// Hold callee-saved information.
112 RegisterClassInfo RCI
;
113 MachineDominatorTree
*MDT
;
114 MachinePostDominatorTree
*MPDT
;
116 /// Current safe point found for the prologue.
117 /// The prologue will be inserted before the first instruction
118 /// in this basic block.
119 MachineBasicBlock
*Save
;
121 /// Current safe point found for the epilogue.
122 /// The epilogue will be inserted before the first terminator instruction
123 /// in this basic block.
124 MachineBasicBlock
*Restore
;
126 /// Hold the information of the basic block frequency.
127 /// Use to check the profitability of the new points.
128 MachineBlockFrequencyInfo
*MBFI
;
130 /// Hold the loop information. Used to determine if Save and Restore
131 /// are in the same loop.
132 MachineLoopInfo
*MLI
;
135 MachineOptimizationRemarkEmitter
*ORE
= nullptr;
137 /// Frequency of the Entry block.
140 /// Current opcode for frame setup.
141 unsigned FrameSetupOpcode
;
143 /// Current opcode for frame destroy.
144 unsigned FrameDestroyOpcode
;
146 /// Stack pointer register, used by llvm.{savestack,restorestack}
150 const MachineBasicBlock
*Entry
;
152 using SetOfRegs
= SmallSetVector
<unsigned, 16>;
154 /// Registers that need to be saved for the current function.
155 mutable SetOfRegs CurrentCSRs
;
157 /// Current MachineFunction.
158 MachineFunction
*MachineFunc
;
160 /// Check if \p MI uses or defines a callee-saved register or
161 /// a frame index. If this is the case, this means \p MI must happen
162 /// after Save and before Restore.
163 bool useOrDefCSROrFI(const MachineInstr
&MI
, RegScavenger
*RS
) const;
165 const SetOfRegs
&getCurrentCSRs(RegScavenger
*RS
) const {
166 if (CurrentCSRs
.empty()) {
168 const TargetFrameLowering
*TFI
=
169 MachineFunc
->getSubtarget().getFrameLowering();
171 TFI
->determineCalleeSaves(*MachineFunc
, SavedRegs
, RS
);
173 for (int Reg
= SavedRegs
.find_first(); Reg
!= -1;
174 Reg
= SavedRegs
.find_next(Reg
))
175 CurrentCSRs
.insert((unsigned)Reg
);
180 /// Update the Save and Restore points such that \p MBB is in
181 /// the region that is dominated by Save and post-dominated by Restore
182 /// and Save and Restore still match the safe point definition.
183 /// Such point may not exist and Save and/or Restore may be null after
185 void updateSaveRestorePoints(MachineBasicBlock
&MBB
, RegScavenger
*RS
);
187 /// Initialize the pass for \p MF.
188 void init(MachineFunction
&MF
) {
189 RCI
.runOnMachineFunction(MF
);
190 MDT
= &getAnalysis
<MachineDominatorTree
>();
191 MPDT
= &getAnalysis
<MachinePostDominatorTree
>();
194 MBFI
= &getAnalysis
<MachineBlockFrequencyInfo
>();
195 MLI
= &getAnalysis
<MachineLoopInfo
>();
196 ORE
= &getAnalysis
<MachineOptimizationRemarkEmitterPass
>().getORE();
197 EntryFreq
= MBFI
->getEntryFreq();
198 const TargetSubtargetInfo
&Subtarget
= MF
.getSubtarget();
199 const TargetInstrInfo
&TII
= *Subtarget
.getInstrInfo();
200 FrameSetupOpcode
= TII
.getCallFrameSetupOpcode();
201 FrameDestroyOpcode
= TII
.getCallFrameDestroyOpcode();
202 SP
= Subtarget
.getTargetLowering()->getStackPointerRegisterToSaveRestore();
210 /// Check whether or not Save and Restore points are still interesting for
212 bool ArePointsInteresting() const { return Save
!= Entry
&& Save
&& Restore
; }
214 /// Check if shrink wrapping is enabled for this target and function.
215 static bool isShrinkWrapEnabled(const MachineFunction
&MF
);
220 ShrinkWrap() : MachineFunctionPass(ID
) {
221 initializeShrinkWrapPass(*PassRegistry::getPassRegistry());
224 void getAnalysisUsage(AnalysisUsage
&AU
) const override
{
225 AU
.setPreservesAll();
226 AU
.addRequired
<MachineBlockFrequencyInfo
>();
227 AU
.addRequired
<MachineDominatorTree
>();
228 AU
.addRequired
<MachinePostDominatorTree
>();
229 AU
.addRequired
<MachineLoopInfo
>();
230 AU
.addRequired
<MachineOptimizationRemarkEmitterPass
>();
231 MachineFunctionPass::getAnalysisUsage(AU
);
234 MachineFunctionProperties
getRequiredProperties() const override
{
235 return MachineFunctionProperties().set(
236 MachineFunctionProperties::Property::NoVRegs
);
239 StringRef
getPassName() const override
{ return "Shrink Wrapping analysis"; }
241 /// Perform the shrink-wrapping analysis and update
242 /// the MachineFrameInfo attached to \p MF with the results.
243 bool runOnMachineFunction(MachineFunction
&MF
) override
;
246 } // end anonymous namespace
248 char ShrinkWrap::ID
= 0;
250 char &llvm::ShrinkWrapID
= ShrinkWrap::ID
;
252 INITIALIZE_PASS_BEGIN(ShrinkWrap
, DEBUG_TYPE
, "Shrink Wrap Pass", false, false)
253 INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo
)
254 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree
)
255 INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTree
)
256 INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo
)
257 INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass
)
258 INITIALIZE_PASS_END(ShrinkWrap
, DEBUG_TYPE
, "Shrink Wrap Pass", false, false)
260 bool ShrinkWrap::useOrDefCSROrFI(const MachineInstr
&MI
,
261 RegScavenger
*RS
) const {
262 // This prevents premature stack popping when occurs a indirect stack
263 // access. It is overly aggressive for the moment.
264 // TODO: - Obvious non-stack loads and store, such as global values,
265 // are known to not access the stack.
266 // - Further, data dependency and alias analysis can validate
267 // that load and stores never derive from the stack pointer.
268 if (MI
.mayLoadOrStore())
271 if (MI
.getOpcode() == FrameSetupOpcode
||
272 MI
.getOpcode() == FrameDestroyOpcode
) {
273 LLVM_DEBUG(dbgs() << "Frame instruction: " << MI
<< '\n');
276 const MachineFunction
*MF
= MI
.getParent()->getParent();
277 const TargetRegisterInfo
*TRI
= MF
->getSubtarget().getRegisterInfo();
278 for (const MachineOperand
&MO
: MI
.operands()) {
279 bool UseOrDefCSR
= false;
281 // Ignore instructions like DBG_VALUE which don't read/def the register.
282 if (!MO
.isDef() && !MO
.readsReg())
284 Register PhysReg
= MO
.getReg();
287 assert(Register::isPhysicalRegister(PhysReg
) && "Unallocated register?!");
288 // The stack pointer is not normally described as a callee-saved register
289 // in calling convention definitions, so we need to watch for it
290 // separately. An SP mentioned by a call instruction, we can ignore,
291 // though, as it's harmless and we do not want to effectively disable tail
292 // calls by forcing the restore point to post-dominate them.
293 // PPC's LR is also not normally described as a callee-saved register in
294 // calling convention definitions, so we need to watch for it, too. An LR
295 // mentioned implicitly by a return (or "branch to link register")
296 // instruction we can ignore, otherwise we may pessimize shrinkwrapping.
298 (!MI
.isCall() && PhysReg
== SP
) ||
299 RCI
.getLastCalleeSavedAlias(PhysReg
) ||
300 (!MI
.isReturn() && TRI
->isNonallocatableRegisterCalleeSave(PhysReg
));
301 } else if (MO
.isRegMask()) {
302 // Check if this regmask clobbers any of the CSRs.
303 for (unsigned Reg
: getCurrentCSRs(RS
)) {
304 if (MO
.clobbersPhysReg(Reg
)) {
310 // Skip FrameIndex operands in DBG_VALUE instructions.
311 if (UseOrDefCSR
|| (MO
.isFI() && !MI
.isDebugValue())) {
312 LLVM_DEBUG(dbgs() << "Use or define CSR(" << UseOrDefCSR
<< ") or FI("
313 << MO
.isFI() << "): " << MI
<< '\n');
320 /// Helper function to find the immediate (post) dominator.
321 template <typename ListOfBBs
, typename DominanceAnalysis
>
322 static MachineBasicBlock
*FindIDom(MachineBasicBlock
&Block
, ListOfBBs BBs
,
323 DominanceAnalysis
&Dom
) {
324 MachineBasicBlock
*IDom
= &Block
;
325 for (MachineBasicBlock
*BB
: BBs
) {
326 IDom
= Dom
.findNearestCommonDominator(IDom
, BB
);
335 void ShrinkWrap::updateSaveRestorePoints(MachineBasicBlock
&MBB
,
337 // Get rid of the easy cases first.
341 Save
= MDT
->findNearestCommonDominator(Save
, &MBB
);
346 else if (MPDT
->getNode(&MBB
)) // If the block is not in the post dom tree, it
347 // means the block never returns. If that's the
348 // case, we don't want to call
349 // `findNearestCommonDominator`, which will
351 Restore
= MPDT
->findNearestCommonDominator(Restore
, &MBB
);
353 Restore
= nullptr; // Abort, we can't find a restore point in this case.
355 // Make sure we would be able to insert the restore code before the
357 if (Restore
== &MBB
) {
358 for (const MachineInstr
&Terminator
: MBB
.terminators()) {
359 if (!useOrDefCSROrFI(Terminator
, RS
))
361 // One of the terminator needs to happen before the restore point.
362 if (MBB
.succ_empty()) {
363 Restore
= nullptr; // Abort, we can't find a restore point in this case.
366 // Look for a restore point that post-dominates all the successors.
367 // The immediate post-dominator is what we are looking for.
368 Restore
= FindIDom
<>(*Restore
, Restore
->successors(), *MPDT
);
375 dbgs() << "Restore point needs to be spanned on several blocks\n");
379 // Make sure Save and Restore are suitable for shrink-wrapping:
380 // 1. all path from Save needs to lead to Restore before exiting.
381 // 2. all path to Restore needs to go through Save from Entry.
382 // We achieve that by making sure that:
383 // A. Save dominates Restore.
384 // B. Restore post-dominates Save.
385 // C. Save and Restore are in the same loop.
386 bool SaveDominatesRestore
= false;
387 bool RestorePostDominatesSave
= false;
389 (!(SaveDominatesRestore
= MDT
->dominates(Save
, Restore
)) ||
390 !(RestorePostDominatesSave
= MPDT
->dominates(Restore
, Save
)) ||
391 // Post-dominance is not enough in loops to ensure that all uses/defs
392 // are after the prologue and before the epilogue at runtime.
401 // All the uses/defs of CSRs are dominated by Save and post-dominated
402 // by Restore. However, the CSRs uses are still reachable after
403 // Restore and before Save are executed.
405 // For now, just push the restore/save points outside of loops.
406 // FIXME: Refine the criteria to still find interesting cases
408 MLI
->getLoopFor(Save
) || MLI
->getLoopFor(Restore
))) {
410 if (!SaveDominatesRestore
) {
411 Save
= MDT
->findNearestCommonDominator(Save
, Restore
);
415 if (!RestorePostDominatesSave
)
416 Restore
= MPDT
->findNearestCommonDominator(Restore
, Save
);
419 if (Restore
&& (MLI
->getLoopFor(Save
) || MLI
->getLoopFor(Restore
))) {
420 if (MLI
->getLoopDepth(Save
) > MLI
->getLoopDepth(Restore
)) {
421 // Push Save outside of this loop if immediate dominator is different
422 // from save block. If immediate dominator is not different, bail out.
423 Save
= FindIDom
<>(*Save
, Save
->predecessors(), *MDT
);
427 // If the loop does not exit, there is no point in looking
428 // for a post-dominator outside the loop.
429 SmallVector
<MachineBasicBlock
*, 4> ExitBlocks
;
430 MLI
->getLoopFor(Restore
)->getExitingBlocks(ExitBlocks
);
431 // Push Restore outside of this loop.
432 // Look for the immediate post-dominator of the loop exits.
433 MachineBasicBlock
*IPdom
= Restore
;
434 for (MachineBasicBlock
*LoopExitBB
: ExitBlocks
) {
435 IPdom
= FindIDom
<>(*IPdom
, LoopExitBB
->successors(), *MPDT
);
439 // If the immediate post-dominator is not in a less nested loop,
440 // then we are stuck in a program with an infinite loop.
441 // In that case, we will not find a safe point, hence, bail out.
442 if (IPdom
&& MLI
->getLoopDepth(IPdom
) < MLI
->getLoopDepth(Restore
))
453 static bool giveUpWithRemarks(MachineOptimizationRemarkEmitter
*ORE
,
454 StringRef RemarkName
, StringRef RemarkMessage
,
455 const DiagnosticLocation
&Loc
,
456 const MachineBasicBlock
*MBB
) {
458 return MachineOptimizationRemarkMissed(DEBUG_TYPE
, RemarkName
, Loc
, MBB
)
462 LLVM_DEBUG(dbgs() << RemarkMessage
<< '\n');
466 bool ShrinkWrap::runOnMachineFunction(MachineFunction
&MF
) {
467 if (skipFunction(MF
.getFunction()) || MF
.empty() || !isShrinkWrapEnabled(MF
))
470 LLVM_DEBUG(dbgs() << "**** Analysing " << MF
.getName() << '\n');
474 ReversePostOrderTraversal
<MachineBasicBlock
*> RPOT(&*MF
.begin());
475 if (containsIrreducibleCFG
<MachineBasicBlock
*>(RPOT
, *MLI
)) {
476 // If MF is irreducible, a block may be in a loop without
477 // MachineLoopInfo reporting it. I.e., we may use the
478 // post-dominance property in loops, which lead to incorrect
479 // results. Moreover, we may miss that the prologue and
480 // epilogue are not in the same loop, leading to unbalanced
481 // construction/deconstruction of the stack frame.
482 return giveUpWithRemarks(ORE
, "UnsupportedIrreducibleCFG",
483 "Irreducible CFGs are not supported yet.",
484 MF
.getFunction().getSubprogram(), &MF
.front());
487 const TargetRegisterInfo
*TRI
= MF
.getSubtarget().getRegisterInfo();
488 std::unique_ptr
<RegScavenger
> RS(
489 TRI
->requiresRegisterScavenging(MF
) ? new RegScavenger() : nullptr);
491 for (MachineBasicBlock
&MBB
: MF
) {
492 LLVM_DEBUG(dbgs() << "Look into: " << MBB
.getNumber() << ' '
493 << MBB
.getName() << '\n');
495 if (MBB
.isEHFuncletEntry())
496 return giveUpWithRemarks(ORE
, "UnsupportedEHFunclets",
497 "EH Funclets are not supported yet.",
498 MBB
.front().getDebugLoc(), &MBB
);
500 if (MBB
.isEHPad() || MBB
.isInlineAsmBrIndirectTarget()) {
501 // Push the prologue and epilogue outside of the region that may throw (or
502 // jump out via inlineasm_br), by making sure that all the landing pads
503 // are at least at the boundary of the save and restore points. The
504 // problem is that a basic block can jump out from the middle in these
505 // cases, which we do not handle.
506 updateSaveRestorePoints(MBB
, RS
.get());
507 if (!ArePointsInteresting()) {
508 LLVM_DEBUG(dbgs() << "EHPad/inlineasm_br prevents shrink-wrapping\n");
514 for (const MachineInstr
&MI
: MBB
) {
515 if (!useOrDefCSROrFI(MI
, RS
.get()))
517 // Save (resp. restore) point must dominate (resp. post dominate)
518 // MI. Look for the proper basic block for those.
519 updateSaveRestorePoints(MBB
, RS
.get());
520 // If we are at a point where we cannot improve the placement of
521 // save/restore instructions, just give up.
522 if (!ArePointsInteresting()) {
523 LLVM_DEBUG(dbgs() << "No Shrink wrap candidate found\n");
526 // No need to look for other instructions, this basic block
527 // will already be part of the handled region.
531 if (!ArePointsInteresting()) {
532 // If the points are not interesting at this point, then they must be null
533 // because it means we did not encounter any frame/CSR related code.
534 // Otherwise, we would have returned from the previous loop.
535 assert(!Save
&& !Restore
&& "We miss a shrink-wrap opportunity?!");
536 LLVM_DEBUG(dbgs() << "Nothing to shrink-wrap\n");
540 LLVM_DEBUG(dbgs() << "\n ** Results **\nFrequency of the Entry: " << EntryFreq
543 const TargetFrameLowering
*TFI
= MF
.getSubtarget().getFrameLowering();
545 LLVM_DEBUG(dbgs() << "Shrink wrap candidates (#, Name, Freq):\nSave: "
546 << Save
->getNumber() << ' ' << Save
->getName() << ' '
547 << MBFI
->getBlockFreq(Save
).getFrequency()
548 << "\nRestore: " << Restore
->getNumber() << ' '
549 << Restore
->getName() << ' '
550 << MBFI
->getBlockFreq(Restore
).getFrequency() << '\n');
552 bool IsSaveCheap
, TargetCanUseSaveAsPrologue
= false;
553 if (((IsSaveCheap
= EntryFreq
>= MBFI
->getBlockFreq(Save
).getFrequency()) &&
554 EntryFreq
>= MBFI
->getBlockFreq(Restore
).getFrequency()) &&
555 ((TargetCanUseSaveAsPrologue
= TFI
->canUseAsPrologue(*Save
)) &&
556 TFI
->canUseAsEpilogue(*Restore
)))
559 dbgs() << "New points are too expensive or invalid for the target\n");
560 MachineBasicBlock
*NewBB
;
561 if (!IsSaveCheap
|| !TargetCanUseSaveAsPrologue
) {
562 Save
= FindIDom
<>(*Save
, Save
->predecessors(), *MDT
);
567 // Restore is expensive.
568 Restore
= FindIDom
<>(*Restore
, Restore
->successors(), *MPDT
);
573 updateSaveRestorePoints(*NewBB
, RS
.get());
574 } while (Save
&& Restore
);
576 if (!ArePointsInteresting()) {
577 ++NumCandidatesDropped
;
581 LLVM_DEBUG(dbgs() << "Final shrink wrap candidates:\nSave: "
582 << Save
->getNumber() << ' ' << Save
->getName()
583 << "\nRestore: " << Restore
->getNumber() << ' '
584 << Restore
->getName() << '\n');
586 MachineFrameInfo
&MFI
= MF
.getFrameInfo();
587 MFI
.setSavePoint(Save
);
588 MFI
.setRestorePoint(Restore
);
593 bool ShrinkWrap::isShrinkWrapEnabled(const MachineFunction
&MF
) {
594 const TargetFrameLowering
*TFI
= MF
.getSubtarget().getFrameLowering();
596 switch (EnableShrinkWrapOpt
) {
598 return TFI
->enableShrinkWrapping(MF
) &&
599 // Windows with CFI has some limitations that make it impossible
600 // to use shrink-wrapping.
601 !MF
.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
602 // Sanitizers look at the value of the stack at the location
603 // of the crash. Since a crash can happen anywhere, the
604 // frame must be lowered before anything else happen for the
605 // sanitizers to be able to get a correct stack frame.
606 !(MF
.getFunction().hasFnAttribute(Attribute::SanitizeAddress
) ||
607 MF
.getFunction().hasFnAttribute(Attribute::SanitizeThread
) ||
608 MF
.getFunction().hasFnAttribute(Attribute::SanitizeMemory
) ||
609 MF
.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress
));
610 // If EnableShrinkWrap is set, it takes precedence on whatever the
611 // target sets. The rational is that we assume we want to test
612 // something related to shrink-wrapping.
618 llvm_unreachable("Invalid shrink-wrapping state");