1 //===- X86AvoidStoreForwardingBlockis.cpp - Avoid HW Store Forward Block --===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // If a load follows a store and reloads data that the store has written to
10 // memory, Intel microarchitectures can in many cases forward the data directly
11 // from the store to the load, This "store forwarding" saves cycles by enabling
12 // the load to directly obtain the data instead of accessing the data from
14 // A "store forward block" occurs in cases that a store cannot be forwarded to
15 // the load. The most typical case of store forward block on Intel Core
16 // microarchitecture that a small store cannot be forwarded to a large load.
17 // The estimated penalty for a store forward block is ~13 cycles.
19 // This pass tries to recognize and handle cases where "store forward block"
20 // is created by the compiler when lowering memcpy calls to a sequence
21 // of a load and a store.
23 // The pass currently only handles cases where memcpy is lowered to
24 // XMM/YMM registers, it tries to break the memcpy into smaller copies.
25 // breaking the memcpy should be possible since there is no atomicity
26 // guarantee for loads and stores to XMM/YMM.
28 // It could be better for performance to solve the problem by loading
29 // to XMM/YMM then inserting the partial store before storing back from XMM/YMM
30 // to memory, but this will result in a more conservative optimization since it
31 // requires we prove that all memory accesses between the blocking store and the
32 // load must alias/don't alias before we can move the store, whereas the
33 // transformation done here is correct regardless to other memory accesses.
34 //===----------------------------------------------------------------------===//
36 #include "X86InstrInfo.h"
37 #include "X86Subtarget.h"
38 #include "llvm/CodeGen/MachineBasicBlock.h"
39 #include "llvm/CodeGen/MachineFunction.h"
40 #include "llvm/CodeGen/MachineFunctionPass.h"
41 #include "llvm/CodeGen/MachineInstr.h"
42 #include "llvm/CodeGen/MachineInstrBuilder.h"
43 #include "llvm/CodeGen/MachineOperand.h"
44 #include "llvm/CodeGen/MachineRegisterInfo.h"
45 #include "llvm/IR/DebugInfoMetadata.h"
46 #include "llvm/IR/DebugLoc.h"
47 #include "llvm/IR/Function.h"
48 #include "llvm/MC/MCInstrDesc.h"
52 #define DEBUG_TYPE "x86-avoid-SFB"
54 static cl::opt
<bool> DisableX86AvoidStoreForwardBlocks(
55 "x86-disable-avoid-SFB", cl::Hidden
,
56 cl::desc("X86: Disable Store Forwarding Blocks fixup."), cl::init(false));
58 static cl::opt
<unsigned> X86AvoidSFBInspectionLimit(
59 "x86-sfb-inspection-limit",
60 cl::desc("X86: Number of instructions backward to "
61 "inspect for store forwarding blocks."),
62 cl::init(20), cl::Hidden
);
66 using DisplacementSizeMap
= std::map
<int64_t, unsigned>;
68 class X86AvoidSFBPass
: public MachineFunctionPass
{
71 X86AvoidSFBPass() : MachineFunctionPass(ID
) { }
73 StringRef
getPassName() const override
{
74 return "X86 Avoid Store Forwarding Blocks";
77 bool runOnMachineFunction(MachineFunction
&MF
) override
;
79 void getAnalysisUsage(AnalysisUsage
&AU
) const override
{
80 MachineFunctionPass::getAnalysisUsage(AU
);
81 AU
.addRequired
<AAResultsWrapperPass
>();
85 MachineRegisterInfo
*MRI
;
86 const X86InstrInfo
*TII
;
87 const X86RegisterInfo
*TRI
;
88 SmallVector
<std::pair
<MachineInstr
*, MachineInstr
*>, 2>
89 BlockedLoadsStoresPairs
;
90 SmallVector
<MachineInstr
*, 2> ForRemoval
;
93 /// Returns couples of Load then Store to memory which look
95 void findPotentiallylBlockedCopies(MachineFunction
&MF
);
96 /// Break the memcpy's load and store into smaller copies
97 /// such that each memory load that was blocked by a smaller store
98 /// would now be copied separately.
99 void breakBlockedCopies(MachineInstr
*LoadInst
, MachineInstr
*StoreInst
,
100 const DisplacementSizeMap
&BlockingStoresDispSizeMap
);
101 /// Break a copy of size Size to smaller copies.
102 void buildCopies(int Size
, MachineInstr
*LoadInst
, int64_t LdDispImm
,
103 MachineInstr
*StoreInst
, int64_t StDispImm
,
104 int64_t LMMOffset
, int64_t SMMOffset
);
106 void buildCopy(MachineInstr
*LoadInst
, unsigned NLoadOpcode
, int64_t LoadDisp
,
107 MachineInstr
*StoreInst
, unsigned NStoreOpcode
,
108 int64_t StoreDisp
, unsigned Size
, int64_t LMMOffset
,
111 bool alias(const MachineMemOperand
&Op1
, const MachineMemOperand
&Op2
) const;
113 unsigned getRegSizeInBytes(MachineInstr
*Inst
);
116 } // end anonymous namespace
118 char X86AvoidSFBPass::ID
= 0;
120 INITIALIZE_PASS_BEGIN(X86AvoidSFBPass
, DEBUG_TYPE
, "Machine code sinking",
122 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass
)
123 INITIALIZE_PASS_END(X86AvoidSFBPass
, DEBUG_TYPE
, "Machine code sinking", false,
126 FunctionPass
*llvm::createX86AvoidStoreForwardingBlocks() {
127 return new X86AvoidSFBPass();
130 static bool isXMMLoadOpcode(unsigned Opcode
) {
131 return Opcode
== X86::MOVUPSrm
|| Opcode
== X86::MOVAPSrm
||
132 Opcode
== X86::VMOVUPSrm
|| Opcode
== X86::VMOVAPSrm
||
133 Opcode
== X86::VMOVUPDrm
|| Opcode
== X86::VMOVAPDrm
||
134 Opcode
== X86::VMOVDQUrm
|| Opcode
== X86::VMOVDQArm
||
135 Opcode
== X86::VMOVUPSZ128rm
|| Opcode
== X86::VMOVAPSZ128rm
||
136 Opcode
== X86::VMOVUPDZ128rm
|| Opcode
== X86::VMOVAPDZ128rm
||
137 Opcode
== X86::VMOVDQU64Z128rm
|| Opcode
== X86::VMOVDQA64Z128rm
||
138 Opcode
== X86::VMOVDQU32Z128rm
|| Opcode
== X86::VMOVDQA32Z128rm
;
140 static bool isYMMLoadOpcode(unsigned Opcode
) {
141 return Opcode
== X86::VMOVUPSYrm
|| Opcode
== X86::VMOVAPSYrm
||
142 Opcode
== X86::VMOVUPDYrm
|| Opcode
== X86::VMOVAPDYrm
||
143 Opcode
== X86::VMOVDQUYrm
|| Opcode
== X86::VMOVDQAYrm
||
144 Opcode
== X86::VMOVUPSZ256rm
|| Opcode
== X86::VMOVAPSZ256rm
||
145 Opcode
== X86::VMOVUPDZ256rm
|| Opcode
== X86::VMOVAPDZ256rm
||
146 Opcode
== X86::VMOVDQU64Z256rm
|| Opcode
== X86::VMOVDQA64Z256rm
||
147 Opcode
== X86::VMOVDQU32Z256rm
|| Opcode
== X86::VMOVDQA32Z256rm
;
150 static bool isPotentialBlockedMemCpyLd(unsigned Opcode
) {
151 return isXMMLoadOpcode(Opcode
) || isYMMLoadOpcode(Opcode
);
154 static bool isPotentialBlockedMemCpyPair(int LdOpcode
, int StOpcode
) {
158 return StOpcode
== X86::MOVUPSmr
|| StOpcode
== X86::MOVAPSmr
;
161 return StOpcode
== X86::VMOVUPSmr
|| StOpcode
== X86::VMOVAPSmr
;
164 return StOpcode
== X86::VMOVUPDmr
|| StOpcode
== X86::VMOVAPDmr
;
167 return StOpcode
== X86::VMOVDQUmr
|| StOpcode
== X86::VMOVDQAmr
;
168 case X86::VMOVUPSZ128rm
:
169 case X86::VMOVAPSZ128rm
:
170 return StOpcode
== X86::VMOVUPSZ128mr
|| StOpcode
== X86::VMOVAPSZ128mr
;
171 case X86::VMOVUPDZ128rm
:
172 case X86::VMOVAPDZ128rm
:
173 return StOpcode
== X86::VMOVUPDZ128mr
|| StOpcode
== X86::VMOVAPDZ128mr
;
174 case X86::VMOVUPSYrm
:
175 case X86::VMOVAPSYrm
:
176 return StOpcode
== X86::VMOVUPSYmr
|| StOpcode
== X86::VMOVAPSYmr
;
177 case X86::VMOVUPDYrm
:
178 case X86::VMOVAPDYrm
:
179 return StOpcode
== X86::VMOVUPDYmr
|| StOpcode
== X86::VMOVAPDYmr
;
180 case X86::VMOVDQUYrm
:
181 case X86::VMOVDQAYrm
:
182 return StOpcode
== X86::VMOVDQUYmr
|| StOpcode
== X86::VMOVDQAYmr
;
183 case X86::VMOVUPSZ256rm
:
184 case X86::VMOVAPSZ256rm
:
185 return StOpcode
== X86::VMOVUPSZ256mr
|| StOpcode
== X86::VMOVAPSZ256mr
;
186 case X86::VMOVUPDZ256rm
:
187 case X86::VMOVAPDZ256rm
:
188 return StOpcode
== X86::VMOVUPDZ256mr
|| StOpcode
== X86::VMOVAPDZ256mr
;
189 case X86::VMOVDQU64Z128rm
:
190 case X86::VMOVDQA64Z128rm
:
191 return StOpcode
== X86::VMOVDQU64Z128mr
|| StOpcode
== X86::VMOVDQA64Z128mr
;
192 case X86::VMOVDQU32Z128rm
:
193 case X86::VMOVDQA32Z128rm
:
194 return StOpcode
== X86::VMOVDQU32Z128mr
|| StOpcode
== X86::VMOVDQA32Z128mr
;
195 case X86::VMOVDQU64Z256rm
:
196 case X86::VMOVDQA64Z256rm
:
197 return StOpcode
== X86::VMOVDQU64Z256mr
|| StOpcode
== X86::VMOVDQA64Z256mr
;
198 case X86::VMOVDQU32Z256rm
:
199 case X86::VMOVDQA32Z256rm
:
200 return StOpcode
== X86::VMOVDQU32Z256mr
|| StOpcode
== X86::VMOVDQA32Z256mr
;
206 static bool isPotentialBlockingStoreInst(int Opcode
, int LoadOpcode
) {
208 PBlock
|= Opcode
== X86::MOV64mr
|| Opcode
== X86::MOV64mi32
||
209 Opcode
== X86::MOV32mr
|| Opcode
== X86::MOV32mi
||
210 Opcode
== X86::MOV16mr
|| Opcode
== X86::MOV16mi
||
211 Opcode
== X86::MOV8mr
|| Opcode
== X86::MOV8mi
;
212 if (isYMMLoadOpcode(LoadOpcode
))
213 PBlock
|= Opcode
== X86::VMOVUPSmr
|| Opcode
== X86::VMOVAPSmr
||
214 Opcode
== X86::VMOVUPDmr
|| Opcode
== X86::VMOVAPDmr
||
215 Opcode
== X86::VMOVDQUmr
|| Opcode
== X86::VMOVDQAmr
||
216 Opcode
== X86::VMOVUPSZ128mr
|| Opcode
== X86::VMOVAPSZ128mr
||
217 Opcode
== X86::VMOVUPDZ128mr
|| Opcode
== X86::VMOVAPDZ128mr
||
218 Opcode
== X86::VMOVDQU64Z128mr
||
219 Opcode
== X86::VMOVDQA64Z128mr
||
220 Opcode
== X86::VMOVDQU32Z128mr
|| Opcode
== X86::VMOVDQA32Z128mr
;
224 static const int MOV128SZ
= 16;
225 static const int MOV64SZ
= 8;
226 static const int MOV32SZ
= 4;
227 static const int MOV16SZ
= 2;
228 static const int MOV8SZ
= 1;
230 static unsigned getYMMtoXMMLoadOpcode(unsigned LoadOpcode
) {
231 switch (LoadOpcode
) {
232 case X86::VMOVUPSYrm
:
233 case X86::VMOVAPSYrm
:
234 return X86::VMOVUPSrm
;
235 case X86::VMOVUPDYrm
:
236 case X86::VMOVAPDYrm
:
237 return X86::VMOVUPDrm
;
238 case X86::VMOVDQUYrm
:
239 case X86::VMOVDQAYrm
:
240 return X86::VMOVDQUrm
;
241 case X86::VMOVUPSZ256rm
:
242 case X86::VMOVAPSZ256rm
:
243 return X86::VMOVUPSZ128rm
;
244 case X86::VMOVUPDZ256rm
:
245 case X86::VMOVAPDZ256rm
:
246 return X86::VMOVUPDZ128rm
;
247 case X86::VMOVDQU64Z256rm
:
248 case X86::VMOVDQA64Z256rm
:
249 return X86::VMOVDQU64Z128rm
;
250 case X86::VMOVDQU32Z256rm
:
251 case X86::VMOVDQA32Z256rm
:
252 return X86::VMOVDQU32Z128rm
;
254 llvm_unreachable("Unexpected Load Instruction Opcode");
259 static unsigned getYMMtoXMMStoreOpcode(unsigned StoreOpcode
) {
260 switch (StoreOpcode
) {
261 case X86::VMOVUPSYmr
:
262 case X86::VMOVAPSYmr
:
263 return X86::VMOVUPSmr
;
264 case X86::VMOVUPDYmr
:
265 case X86::VMOVAPDYmr
:
266 return X86::VMOVUPDmr
;
267 case X86::VMOVDQUYmr
:
268 case X86::VMOVDQAYmr
:
269 return X86::VMOVDQUmr
;
270 case X86::VMOVUPSZ256mr
:
271 case X86::VMOVAPSZ256mr
:
272 return X86::VMOVUPSZ128mr
;
273 case X86::VMOVUPDZ256mr
:
274 case X86::VMOVAPDZ256mr
:
275 return X86::VMOVUPDZ128mr
;
276 case X86::VMOVDQU64Z256mr
:
277 case X86::VMOVDQA64Z256mr
:
278 return X86::VMOVDQU64Z128mr
;
279 case X86::VMOVDQU32Z256mr
:
280 case X86::VMOVDQA32Z256mr
:
281 return X86::VMOVDQU32Z128mr
;
283 llvm_unreachable("Unexpected Load Instruction Opcode");
288 static int getAddrOffset(MachineInstr
*MI
) {
289 const MCInstrDesc
&Descl
= MI
->getDesc();
290 int AddrOffset
= X86II::getMemoryOperandNo(Descl
.TSFlags
);
291 assert(AddrOffset
!= -1 && "Expected Memory Operand");
292 AddrOffset
+= X86II::getOperandBias(Descl
);
296 static MachineOperand
&getBaseOperand(MachineInstr
*MI
) {
297 int AddrOffset
= getAddrOffset(MI
);
298 return MI
->getOperand(AddrOffset
+ X86::AddrBaseReg
);
301 static MachineOperand
&getDispOperand(MachineInstr
*MI
) {
302 int AddrOffset
= getAddrOffset(MI
);
303 return MI
->getOperand(AddrOffset
+ X86::AddrDisp
);
306 // Relevant addressing modes contain only base register and immediate
307 // displacement or frameindex and immediate displacement.
308 // TODO: Consider expanding to other addressing modes in the future
309 static bool isRelevantAddressingMode(MachineInstr
*MI
) {
310 int AddrOffset
= getAddrOffset(MI
);
311 MachineOperand
&Base
= getBaseOperand(MI
);
312 MachineOperand
&Disp
= getDispOperand(MI
);
313 MachineOperand
&Scale
= MI
->getOperand(AddrOffset
+ X86::AddrScaleAmt
);
314 MachineOperand
&Index
= MI
->getOperand(AddrOffset
+ X86::AddrIndexReg
);
315 MachineOperand
&Segment
= MI
->getOperand(AddrOffset
+ X86::AddrSegmentReg
);
317 if (!((Base
.isReg() && Base
.getReg() != X86::NoRegister
) || Base
.isFI()))
321 if (Scale
.getImm() != 1)
323 if (!(Index
.isReg() && Index
.getReg() == X86::NoRegister
))
325 if (!(Segment
.isReg() && Segment
.getReg() == X86::NoRegister
))
330 // Collect potentially blocking stores.
331 // Limit the number of instructions backwards we want to inspect
332 // since the effect of store block won't be visible if the store
333 // and load instructions have enough instructions in between to
334 // keep the core busy.
335 static SmallVector
<MachineInstr
*, 2>
336 findPotentialBlockers(MachineInstr
*LoadInst
) {
337 SmallVector
<MachineInstr
*, 2> PotentialBlockers
;
338 unsigned BlockCount
= 0;
339 const unsigned InspectionLimit
= X86AvoidSFBInspectionLimit
;
340 for (auto PBInst
= std::next(MachineBasicBlock::reverse_iterator(LoadInst
)),
341 E
= LoadInst
->getParent()->rend();
342 PBInst
!= E
; ++PBInst
) {
343 if (PBInst
->isMetaInstruction())
346 if (BlockCount
>= InspectionLimit
)
348 MachineInstr
&MI
= *PBInst
;
349 if (MI
.getDesc().isCall())
350 return PotentialBlockers
;
351 PotentialBlockers
.push_back(&MI
);
353 // If we didn't get to the instructions limit try predecessing blocks.
354 // Ideally we should traverse the predecessor blocks in depth with some
355 // coloring algorithm, but for now let's just look at the first order
357 if (BlockCount
< InspectionLimit
) {
358 MachineBasicBlock
*MBB
= LoadInst
->getParent();
359 int LimitLeft
= InspectionLimit
- BlockCount
;
360 for (MachineBasicBlock::pred_iterator PB
= MBB
->pred_begin(),
361 PE
= MBB
->pred_end();
363 MachineBasicBlock
*PMBB
= *PB
;
365 for (MachineBasicBlock::reverse_iterator PBInst
= PMBB
->rbegin(),
367 PBInst
!= PME
; ++PBInst
) {
368 if (PBInst
->isMetaInstruction())
371 if (PredCount
>= LimitLeft
)
373 if (PBInst
->getDesc().isCall())
375 PotentialBlockers
.push_back(&*PBInst
);
379 return PotentialBlockers
;
382 void X86AvoidSFBPass::buildCopy(MachineInstr
*LoadInst
, unsigned NLoadOpcode
,
383 int64_t LoadDisp
, MachineInstr
*StoreInst
,
384 unsigned NStoreOpcode
, int64_t StoreDisp
,
385 unsigned Size
, int64_t LMMOffset
,
387 MachineOperand
&LoadBase
= getBaseOperand(LoadInst
);
388 MachineOperand
&StoreBase
= getBaseOperand(StoreInst
);
389 MachineBasicBlock
*MBB
= LoadInst
->getParent();
390 MachineMemOperand
*LMMO
= *LoadInst
->memoperands_begin();
391 MachineMemOperand
*SMMO
= *StoreInst
->memoperands_begin();
393 Register Reg1
= MRI
->createVirtualRegister(
394 TII
->getRegClass(TII
->get(NLoadOpcode
), 0, TRI
, *(MBB
->getParent())));
395 MachineInstr
*NewLoad
=
396 BuildMI(*MBB
, LoadInst
, LoadInst
->getDebugLoc(), TII
->get(NLoadOpcode
),
400 .addReg(X86::NoRegister
)
402 .addReg(X86::NoRegister
)
404 MBB
->getParent()->getMachineMemOperand(LMMO
, LMMOffset
, Size
));
405 if (LoadBase
.isReg())
406 getBaseOperand(NewLoad
).setIsKill(false);
407 LLVM_DEBUG(NewLoad
->dump());
408 // If the load and store are consecutive, use the loadInst location to
409 // reduce register pressure.
410 MachineInstr
*StInst
= StoreInst
;
411 auto PrevInstrIt
= skipDebugInstructionsBackward(
412 std::prev(MachineBasicBlock::instr_iterator(StoreInst
)),
414 if (PrevInstrIt
.getNodePtr() == LoadInst
)
416 MachineInstr
*NewStore
=
417 BuildMI(*MBB
, StInst
, StInst
->getDebugLoc(), TII
->get(NStoreOpcode
))
420 .addReg(X86::NoRegister
)
422 .addReg(X86::NoRegister
)
425 MBB
->getParent()->getMachineMemOperand(SMMO
, SMMOffset
, Size
));
426 if (StoreBase
.isReg())
427 getBaseOperand(NewStore
).setIsKill(false);
428 MachineOperand
&StoreSrcVReg
= StoreInst
->getOperand(X86::AddrNumOperands
);
429 assert(StoreSrcVReg
.isReg() && "Expected virtual register");
430 NewStore
->getOperand(X86::AddrNumOperands
).setIsKill(StoreSrcVReg
.isKill());
431 LLVM_DEBUG(NewStore
->dump());
434 void X86AvoidSFBPass::buildCopies(int Size
, MachineInstr
*LoadInst
,
435 int64_t LdDispImm
, MachineInstr
*StoreInst
,
436 int64_t StDispImm
, int64_t LMMOffset
,
438 int LdDisp
= LdDispImm
;
439 int StDisp
= StDispImm
;
441 if ((Size
- MOV128SZ
>= 0) && isYMMLoadOpcode(LoadInst
->getOpcode())) {
442 Size
= Size
- MOV128SZ
;
443 buildCopy(LoadInst
, getYMMtoXMMLoadOpcode(LoadInst
->getOpcode()), LdDisp
,
444 StoreInst
, getYMMtoXMMStoreOpcode(StoreInst
->getOpcode()),
445 StDisp
, MOV128SZ
, LMMOffset
, SMMOffset
);
448 LMMOffset
+= MOV128SZ
;
449 SMMOffset
+= MOV128SZ
;
452 if (Size
- MOV64SZ
>= 0) {
453 Size
= Size
- MOV64SZ
;
454 buildCopy(LoadInst
, X86::MOV64rm
, LdDisp
, StoreInst
, X86::MOV64mr
, StDisp
,
455 MOV64SZ
, LMMOffset
, SMMOffset
);
458 LMMOffset
+= MOV64SZ
;
459 SMMOffset
+= MOV64SZ
;
462 if (Size
- MOV32SZ
>= 0) {
463 Size
= Size
- MOV32SZ
;
464 buildCopy(LoadInst
, X86::MOV32rm
, LdDisp
, StoreInst
, X86::MOV32mr
, StDisp
,
465 MOV32SZ
, LMMOffset
, SMMOffset
);
468 LMMOffset
+= MOV32SZ
;
469 SMMOffset
+= MOV32SZ
;
472 if (Size
- MOV16SZ
>= 0) {
473 Size
= Size
- MOV16SZ
;
474 buildCopy(LoadInst
, X86::MOV16rm
, LdDisp
, StoreInst
, X86::MOV16mr
, StDisp
,
475 MOV16SZ
, LMMOffset
, SMMOffset
);
478 LMMOffset
+= MOV16SZ
;
479 SMMOffset
+= MOV16SZ
;
482 if (Size
- MOV8SZ
>= 0) {
483 Size
= Size
- MOV8SZ
;
484 buildCopy(LoadInst
, X86::MOV8rm
, LdDisp
, StoreInst
, X86::MOV8mr
, StDisp
,
485 MOV8SZ
, LMMOffset
, SMMOffset
);
493 assert(Size
== 0 && "Wrong size division");
496 static void updateKillStatus(MachineInstr
*LoadInst
, MachineInstr
*StoreInst
) {
497 MachineOperand
&LoadBase
= getBaseOperand(LoadInst
);
498 MachineOperand
&StoreBase
= getBaseOperand(StoreInst
);
499 auto StorePrevNonDbgInstr
= skipDebugInstructionsBackward(
500 std::prev(MachineBasicBlock::instr_iterator(StoreInst
)),
501 LoadInst
->getParent()->instr_begin()).getNodePtr();
502 if (LoadBase
.isReg()) {
503 MachineInstr
*LastLoad
= LoadInst
->getPrevNode();
504 // If the original load and store to xmm/ymm were consecutive
505 // then the partial copies were also created in
506 // a consecutive order to reduce register pressure,
507 // and the location of the last load is before the last store.
508 if (StorePrevNonDbgInstr
== LoadInst
)
509 LastLoad
= LoadInst
->getPrevNode()->getPrevNode();
510 getBaseOperand(LastLoad
).setIsKill(LoadBase
.isKill());
512 if (StoreBase
.isReg()) {
513 MachineInstr
*StInst
= StoreInst
;
514 if (StorePrevNonDbgInstr
== LoadInst
)
516 getBaseOperand(StInst
->getPrevNode()).setIsKill(StoreBase
.isKill());
520 bool X86AvoidSFBPass::alias(const MachineMemOperand
&Op1
,
521 const MachineMemOperand
&Op2
) const {
522 if (!Op1
.getValue() || !Op2
.getValue())
525 int64_t MinOffset
= std::min(Op1
.getOffset(), Op2
.getOffset());
526 int64_t Overlapa
= Op1
.getSize() + Op1
.getOffset() - MinOffset
;
527 int64_t Overlapb
= Op2
.getSize() + Op2
.getOffset() - MinOffset
;
529 AliasResult AAResult
=
530 AA
->alias(MemoryLocation(Op1
.getValue(), Overlapa
, Op1
.getAAInfo()),
531 MemoryLocation(Op2
.getValue(), Overlapb
, Op2
.getAAInfo()));
532 return AAResult
!= NoAlias
;
535 void X86AvoidSFBPass::findPotentiallylBlockedCopies(MachineFunction
&MF
) {
537 for (auto &MI
: MBB
) {
538 if (!isPotentialBlockedMemCpyLd(MI
.getOpcode()))
540 int DefVR
= MI
.getOperand(0).getReg();
541 if (!MRI
->hasOneNonDBGUse(DefVR
))
543 for (auto UI
= MRI
->use_nodbg_begin(DefVR
), UE
= MRI
->use_nodbg_end();
545 MachineOperand
&StoreMO
= *UI
++;
546 MachineInstr
&StoreMI
= *StoreMO
.getParent();
547 // Skip cases where the memcpy may overlap.
548 if (StoreMI
.getParent() == MI
.getParent() &&
549 isPotentialBlockedMemCpyPair(MI
.getOpcode(), StoreMI
.getOpcode()) &&
550 isRelevantAddressingMode(&MI
) &&
551 isRelevantAddressingMode(&StoreMI
)) {
552 assert(MI
.hasOneMemOperand() &&
553 "Expected one memory operand for load instruction");
554 assert(StoreMI
.hasOneMemOperand() &&
555 "Expected one memory operand for store instruction");
556 if (!alias(**MI
.memoperands_begin(), **StoreMI
.memoperands_begin()))
557 BlockedLoadsStoresPairs
.push_back(std::make_pair(&MI
, &StoreMI
));
563 unsigned X86AvoidSFBPass::getRegSizeInBytes(MachineInstr
*LoadInst
) {
564 auto TRC
= TII
->getRegClass(TII
->get(LoadInst
->getOpcode()), 0, TRI
,
565 *LoadInst
->getParent()->getParent());
566 return TRI
->getRegSizeInBits(*TRC
) / 8;
569 void X86AvoidSFBPass::breakBlockedCopies(
570 MachineInstr
*LoadInst
, MachineInstr
*StoreInst
,
571 const DisplacementSizeMap
&BlockingStoresDispSizeMap
) {
572 int64_t LdDispImm
= getDispOperand(LoadInst
).getImm();
573 int64_t StDispImm
= getDispOperand(StoreInst
).getImm();
574 int64_t LMMOffset
= 0;
575 int64_t SMMOffset
= 0;
577 int64_t LdDisp1
= LdDispImm
;
579 int64_t StDisp1
= StDispImm
;
583 int64_t LdStDelta
= StDispImm
- LdDispImm
;
585 for (auto DispSizePair
: BlockingStoresDispSizeMap
) {
586 LdDisp2
= DispSizePair
.first
;
587 StDisp2
= DispSizePair
.first
+ LdStDelta
;
588 Size2
= DispSizePair
.second
;
589 // Avoid copying overlapping areas.
590 if (LdDisp2
< LdDisp1
) {
591 int OverlapDelta
= LdDisp1
- LdDisp2
;
592 LdDisp2
+= OverlapDelta
;
593 StDisp2
+= OverlapDelta
;
594 Size2
-= OverlapDelta
;
596 Size1
= LdDisp2
- LdDisp1
;
598 // Build a copy for the point until the current blocking store's
600 buildCopies(Size1
, LoadInst
, LdDisp1
, StoreInst
, StDisp1
, LMMOffset
,
602 // Build a copy for the current blocking store.
603 buildCopies(Size2
, LoadInst
, LdDisp2
, StoreInst
, StDisp2
, LMMOffset
+ Size1
,
605 LdDisp1
= LdDisp2
+ Size2
;
606 StDisp1
= StDisp2
+ Size2
;
607 LMMOffset
+= Size1
+ Size2
;
608 SMMOffset
+= Size1
+ Size2
;
610 unsigned Size3
= (LdDispImm
+ getRegSizeInBytes(LoadInst
)) - LdDisp1
;
611 buildCopies(Size3
, LoadInst
, LdDisp1
, StoreInst
, StDisp1
, LMMOffset
,
615 static bool hasSameBaseOpValue(MachineInstr
*LoadInst
,
616 MachineInstr
*StoreInst
) {
617 MachineOperand
&LoadBase
= getBaseOperand(LoadInst
);
618 MachineOperand
&StoreBase
= getBaseOperand(StoreInst
);
619 if (LoadBase
.isReg() != StoreBase
.isReg())
621 if (LoadBase
.isReg())
622 return LoadBase
.getReg() == StoreBase
.getReg();
623 return LoadBase
.getIndex() == StoreBase
.getIndex();
626 static bool isBlockingStore(int64_t LoadDispImm
, unsigned LoadSize
,
627 int64_t StoreDispImm
, unsigned StoreSize
) {
628 return ((StoreDispImm
>= LoadDispImm
) &&
629 (StoreDispImm
<= LoadDispImm
+ (LoadSize
- StoreSize
)));
632 // Keep track of all stores blocking a load
634 updateBlockingStoresDispSizeMap(DisplacementSizeMap
&BlockingStoresDispSizeMap
,
635 int64_t DispImm
, unsigned Size
) {
636 if (BlockingStoresDispSizeMap
.count(DispImm
)) {
637 // Choose the smallest blocking store starting at this displacement.
638 if (BlockingStoresDispSizeMap
[DispImm
] > Size
)
639 BlockingStoresDispSizeMap
[DispImm
] = Size
;
642 BlockingStoresDispSizeMap
[DispImm
] = Size
;
645 // Remove blocking stores contained in each other.
647 removeRedundantBlockingStores(DisplacementSizeMap
&BlockingStoresDispSizeMap
) {
648 if (BlockingStoresDispSizeMap
.size() <= 1)
651 SmallVector
<std::pair
<int64_t, unsigned>, 0> DispSizeStack
;
652 for (auto DispSizePair
: BlockingStoresDispSizeMap
) {
653 int64_t CurrDisp
= DispSizePair
.first
;
654 unsigned CurrSize
= DispSizePair
.second
;
655 while (DispSizeStack
.size()) {
656 int64_t PrevDisp
= DispSizeStack
.back().first
;
657 unsigned PrevSize
= DispSizeStack
.back().second
;
658 if (CurrDisp
+ CurrSize
> PrevDisp
+ PrevSize
)
660 DispSizeStack
.pop_back();
662 DispSizeStack
.push_back(DispSizePair
);
664 BlockingStoresDispSizeMap
.clear();
665 for (auto Disp
: DispSizeStack
)
666 BlockingStoresDispSizeMap
.insert(Disp
);
669 bool X86AvoidSFBPass::runOnMachineFunction(MachineFunction
&MF
) {
670 bool Changed
= false;
672 if (DisableX86AvoidStoreForwardBlocks
|| skipFunction(MF
.getFunction()) ||
673 !MF
.getSubtarget
<X86Subtarget
>().is64Bit())
676 MRI
= &MF
.getRegInfo();
677 assert(MRI
->isSSA() && "Expected MIR to be in SSA form");
678 TII
= MF
.getSubtarget
<X86Subtarget
>().getInstrInfo();
679 TRI
= MF
.getSubtarget
<X86Subtarget
>().getRegisterInfo();
680 AA
= &getAnalysis
<AAResultsWrapperPass
>().getAAResults();
681 LLVM_DEBUG(dbgs() << "Start X86AvoidStoreForwardBlocks\n";);
682 // Look for a load then a store to XMM/YMM which look like a memcpy
683 findPotentiallylBlockedCopies(MF
);
685 for (auto LoadStoreInstPair
: BlockedLoadsStoresPairs
) {
686 MachineInstr
*LoadInst
= LoadStoreInstPair
.first
;
687 int64_t LdDispImm
= getDispOperand(LoadInst
).getImm();
688 DisplacementSizeMap BlockingStoresDispSizeMap
;
690 SmallVector
<MachineInstr
*, 2> PotentialBlockers
=
691 findPotentialBlockers(LoadInst
);
692 for (auto PBInst
: PotentialBlockers
) {
693 if (!isPotentialBlockingStoreInst(PBInst
->getOpcode(),
694 LoadInst
->getOpcode()) ||
695 !isRelevantAddressingMode(PBInst
))
697 int64_t PBstDispImm
= getDispOperand(PBInst
).getImm();
698 assert(PBInst
->hasOneMemOperand() && "Expected One Memory Operand");
699 unsigned PBstSize
= (*PBInst
->memoperands_begin())->getSize();
700 // This check doesn't cover all cases, but it will suffice for now.
701 // TODO: take branch probability into consideration, if the blocking
702 // store is in an unreached block, breaking the memcopy could lose
704 if (hasSameBaseOpValue(LoadInst
, PBInst
) &&
705 isBlockingStore(LdDispImm
, getRegSizeInBytes(LoadInst
), PBstDispImm
,
707 updateBlockingStoresDispSizeMap(BlockingStoresDispSizeMap
, PBstDispImm
,
711 if (BlockingStoresDispSizeMap
.empty())
714 // We found a store forward block, break the memcpy's load and store
715 // into smaller copies such that each smaller store that was causing
716 // a store block would now be copied separately.
717 MachineInstr
*StoreInst
= LoadStoreInstPair
.second
;
718 LLVM_DEBUG(dbgs() << "Blocked load and store instructions: \n");
719 LLVM_DEBUG(LoadInst
->dump());
720 LLVM_DEBUG(StoreInst
->dump());
721 LLVM_DEBUG(dbgs() << "Replaced with:\n");
722 removeRedundantBlockingStores(BlockingStoresDispSizeMap
);
723 breakBlockedCopies(LoadInst
, StoreInst
, BlockingStoresDispSizeMap
);
724 updateKillStatus(LoadInst
, StoreInst
);
725 ForRemoval
.push_back(LoadInst
);
726 ForRemoval
.push_back(StoreInst
);
728 for (auto RemovedInst
: ForRemoval
) {
729 RemovedInst
->eraseFromParent();
732 BlockedLoadsStoresPairs
.clear();
733 LLVM_DEBUG(dbgs() << "End X86AvoidStoreForwardBlocks\n";);