1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/init.h>
5 #include <linux/spinlock.h>
7 #include <linux/interrupt.h>
8 #include <linux/export.h>
10 #include <linux/debugfs.h>
12 #include <asm/tlbflush.h>
13 #include <asm/mmu_context.h>
14 #include <asm/nospec-branch.h>
15 #include <asm/cache.h>
17 #include <asm/uv/uv.h>
19 #include "mm_internal.h"
22 * TLB flushing, formerly SMP-only
25 * These mean you can really definitely utterly forget about
26 * writing to user space from interrupts. (Its not allowed anyway).
28 * Optimizations Manfred Spraul <manfred@colorfullife.com>
30 * More scalable flush, from Andi Kleen
32 * Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi
36 * Use bit 0 to mangle the TIF_SPEC_IB state into the mm pointer which is
37 * stored in cpu_tlb_state.last_user_mm_ibpb.
39 #define LAST_USER_MM_IBPB 0x1UL
42 * We get here when we do something requiring a TLB invalidation
43 * but could not go invalidate all of the contexts. We do the
44 * necessary invalidation by clearing out the 'ctx_id' which
45 * forces a TLB flush when the context is loaded.
47 static void clear_asid_other(void)
52 * This is only expected to be set if we have disabled
53 * kernel _PAGE_GLOBAL pages.
55 if (!static_cpu_has(X86_FEATURE_PTI
)) {
60 for (asid
= 0; asid
< TLB_NR_DYN_ASIDS
; asid
++) {
61 /* Do not need to flush the current asid */
62 if (asid
== this_cpu_read(cpu_tlbstate
.loaded_mm_asid
))
65 * Make sure the next time we go to switch to
66 * this asid, we do a flush:
68 this_cpu_write(cpu_tlbstate
.ctxs
[asid
].ctx_id
, 0);
70 this_cpu_write(cpu_tlbstate
.invalidate_other
, false);
73 atomic64_t last_mm_ctx_id
= ATOMIC64_INIT(1);
76 static void choose_new_asid(struct mm_struct
*next
, u64 next_tlb_gen
,
77 u16
*new_asid
, bool *need_flush
)
81 if (!static_cpu_has(X86_FEATURE_PCID
)) {
87 if (this_cpu_read(cpu_tlbstate
.invalidate_other
))
90 for (asid
= 0; asid
< TLB_NR_DYN_ASIDS
; asid
++) {
91 if (this_cpu_read(cpu_tlbstate
.ctxs
[asid
].ctx_id
) !=
96 *need_flush
= (this_cpu_read(cpu_tlbstate
.ctxs
[asid
].tlb_gen
) <
102 * We don't currently own an ASID slot on this CPU.
105 *new_asid
= this_cpu_add_return(cpu_tlbstate
.next_asid
, 1) - 1;
106 if (*new_asid
>= TLB_NR_DYN_ASIDS
) {
108 this_cpu_write(cpu_tlbstate
.next_asid
, 1);
113 static void load_new_mm_cr3(pgd_t
*pgdir
, u16 new_asid
, bool need_flush
)
115 unsigned long new_mm_cr3
;
118 invalidate_user_asid(new_asid
);
119 new_mm_cr3
= build_cr3(pgdir
, new_asid
);
121 new_mm_cr3
= build_cr3_noflush(pgdir
, new_asid
);
125 * Caution: many callers of this function expect
126 * that load_cr3() is serializing and orders TLB
127 * fills with respect to the mm_cpumask writes.
129 write_cr3(new_mm_cr3
);
132 void leave_mm(int cpu
)
134 struct mm_struct
*loaded_mm
= this_cpu_read(cpu_tlbstate
.loaded_mm
);
137 * It's plausible that we're in lazy TLB mode while our mm is init_mm.
138 * If so, our callers still expect us to flush the TLB, but there
139 * aren't any user TLB entries in init_mm to worry about.
141 * This needs to happen before any other sanity checks due to
142 * intel_idle's shenanigans.
144 if (loaded_mm
== &init_mm
)
147 /* Warn if we're not lazy. */
148 WARN_ON(!this_cpu_read(cpu_tlbstate
.is_lazy
));
150 switch_mm(NULL
, &init_mm
, NULL
);
152 EXPORT_SYMBOL_GPL(leave_mm
);
154 void switch_mm(struct mm_struct
*prev
, struct mm_struct
*next
,
155 struct task_struct
*tsk
)
159 local_irq_save(flags
);
160 switch_mm_irqs_off(prev
, next
, tsk
);
161 local_irq_restore(flags
);
164 static void sync_current_stack_to_mm(struct mm_struct
*mm
)
166 unsigned long sp
= current_stack_pointer
;
167 pgd_t
*pgd
= pgd_offset(mm
, sp
);
169 if (pgtable_l5_enabled()) {
170 if (unlikely(pgd_none(*pgd
))) {
171 pgd_t
*pgd_ref
= pgd_offset_k(sp
);
173 set_pgd(pgd
, *pgd_ref
);
177 * "pgd" is faked. The top level entries are "p4d"s, so sync
178 * the p4d. This compiles to approximately the same code as
181 p4d_t
*p4d
= p4d_offset(pgd
, sp
);
183 if (unlikely(p4d_none(*p4d
))) {
184 pgd_t
*pgd_ref
= pgd_offset_k(sp
);
185 p4d_t
*p4d_ref
= p4d_offset(pgd_ref
, sp
);
187 set_p4d(p4d
, *p4d_ref
);
192 static inline unsigned long mm_mangle_tif_spec_ib(struct task_struct
*next
)
194 unsigned long next_tif
= task_thread_info(next
)->flags
;
195 unsigned long ibpb
= (next_tif
>> TIF_SPEC_IB
) & LAST_USER_MM_IBPB
;
197 return (unsigned long)next
->mm
| ibpb
;
200 static void cond_ibpb(struct task_struct
*next
)
202 if (!next
|| !next
->mm
)
206 * Both, the conditional and the always IBPB mode use the mm
207 * pointer to avoid the IBPB when switching between tasks of the
208 * same process. Using the mm pointer instead of mm->context.ctx_id
209 * opens a hypothetical hole vs. mm_struct reuse, which is more or
210 * less impossible to control by an attacker. Aside of that it
211 * would only affect the first schedule so the theoretically
212 * exposed data is not really interesting.
214 if (static_branch_likely(&switch_mm_cond_ibpb
)) {
215 unsigned long prev_mm
, next_mm
;
218 * This is a bit more complex than the always mode because
219 * it has to handle two cases:
221 * 1) Switch from a user space task (potential attacker)
222 * which has TIF_SPEC_IB set to a user space task
223 * (potential victim) which has TIF_SPEC_IB not set.
225 * 2) Switch from a user space task (potential attacker)
226 * which has TIF_SPEC_IB not set to a user space task
227 * (potential victim) which has TIF_SPEC_IB set.
229 * This could be done by unconditionally issuing IBPB when
230 * a task which has TIF_SPEC_IB set is either scheduled in
231 * or out. Though that results in two flushes when:
233 * - the same user space task is scheduled out and later
234 * scheduled in again and only a kernel thread ran in
237 * - a user space task belonging to the same process is
238 * scheduled in after a kernel thread ran in between
240 * - a user space task belonging to the same process is
241 * scheduled in immediately.
243 * Optimize this with reasonably small overhead for the
244 * above cases. Mangle the TIF_SPEC_IB bit into the mm
245 * pointer of the incoming task which is stored in
246 * cpu_tlbstate.last_user_mm_ibpb for comparison.
248 next_mm
= mm_mangle_tif_spec_ib(next
);
249 prev_mm
= this_cpu_read(cpu_tlbstate
.last_user_mm_ibpb
);
252 * Issue IBPB only if the mm's are different and one or
253 * both have the IBPB bit set.
255 if (next_mm
!= prev_mm
&&
256 (next_mm
| prev_mm
) & LAST_USER_MM_IBPB
)
257 indirect_branch_prediction_barrier();
259 this_cpu_write(cpu_tlbstate
.last_user_mm_ibpb
, next_mm
);
262 if (static_branch_unlikely(&switch_mm_always_ibpb
)) {
264 * Only flush when switching to a user space task with a
265 * different context than the user space task which ran
268 if (this_cpu_read(cpu_tlbstate
.last_user_mm
) != next
->mm
) {
269 indirect_branch_prediction_barrier();
270 this_cpu_write(cpu_tlbstate
.last_user_mm
, next
->mm
);
275 void switch_mm_irqs_off(struct mm_struct
*prev
, struct mm_struct
*next
,
276 struct task_struct
*tsk
)
278 struct mm_struct
*real_prev
= this_cpu_read(cpu_tlbstate
.loaded_mm
);
279 u16 prev_asid
= this_cpu_read(cpu_tlbstate
.loaded_mm_asid
);
280 bool was_lazy
= this_cpu_read(cpu_tlbstate
.is_lazy
);
281 unsigned cpu
= smp_processor_id();
287 * NB: The scheduler will call us with prev == next when switching
288 * from lazy TLB mode to normal mode if active_mm isn't changing.
289 * When this happens, we don't assume that CR3 (and hence
290 * cpu_tlbstate.loaded_mm) matches next.
292 * NB: leave_mm() calls us with prev == NULL and tsk == NULL.
295 /* We don't want flush_tlb_func_* to run concurrently with us. */
296 if (IS_ENABLED(CONFIG_PROVE_LOCKING
))
297 WARN_ON_ONCE(!irqs_disabled());
300 * Verify that CR3 is what we think it is. This will catch
301 * hypothetical buggy code that directly switches to swapper_pg_dir
302 * without going through leave_mm() / switch_mm_irqs_off() or that
303 * does something like write_cr3(read_cr3_pa()).
305 * Only do this check if CONFIG_DEBUG_VM=y because __read_cr3()
308 #ifdef CONFIG_DEBUG_VM
309 if (WARN_ON_ONCE(__read_cr3() != build_cr3(real_prev
->pgd
, prev_asid
))) {
311 * If we were to BUG here, we'd be very likely to kill
312 * the system so hard that we don't see the call trace.
313 * Try to recover instead by ignoring the error and doing
314 * a global flush to minimize the chance of corruption.
316 * (This is far from being a fully correct recovery.
317 * Architecturally, the CPU could prefetch something
318 * back into an incorrect ASID slot and leave it there
319 * to cause trouble down the road. It's better than
325 this_cpu_write(cpu_tlbstate
.is_lazy
, false);
328 * The membarrier system call requires a full memory barrier and
329 * core serialization before returning to user-space, after
330 * storing to rq->curr. Writing to CR3 provides that full
331 * memory barrier and core serializing instruction.
333 if (real_prev
== next
) {
334 VM_WARN_ON(this_cpu_read(cpu_tlbstate
.ctxs
[prev_asid
].ctx_id
) !=
335 next
->context
.ctx_id
);
338 * Even in lazy TLB mode, the CPU should stay set in the
339 * mm_cpumask. The TLB shootdown code can figure out from
340 * from cpu_tlbstate.is_lazy whether or not to send an IPI.
342 if (WARN_ON_ONCE(real_prev
!= &init_mm
&&
343 !cpumask_test_cpu(cpu
, mm_cpumask(next
))))
344 cpumask_set_cpu(cpu
, mm_cpumask(next
));
347 * If the CPU is not in lazy TLB mode, we are just switching
348 * from one thread in a process to another thread in the same
349 * process. No TLB flush required.
355 * Read the tlb_gen to check whether a flush is needed.
356 * If the TLB is up to date, just use it.
357 * The barrier synchronizes with the tlb_gen increment in
358 * the TLB shootdown code.
361 next_tlb_gen
= atomic64_read(&next
->context
.tlb_gen
);
362 if (this_cpu_read(cpu_tlbstate
.ctxs
[prev_asid
].tlb_gen
) ==
367 * TLB contents went out of date while we were in lazy
368 * mode. Fall through to the TLB switching code below.
370 new_asid
= prev_asid
;
374 * Avoid user/user BTB poisoning by flushing the branch
375 * predictor when switching between processes. This stops
376 * one process from doing Spectre-v2 attacks on another.
380 if (IS_ENABLED(CONFIG_VMAP_STACK
)) {
382 * If our current stack is in vmalloc space and isn't
383 * mapped in the new pgd, we'll double-fault. Forcibly
386 sync_current_stack_to_mm(next
);
390 * Stop remote flushes for the previous mm.
391 * Skip kernel threads; we never send init_mm TLB flushing IPIs,
392 * but the bitmap manipulation can cause cache line contention.
394 if (real_prev
!= &init_mm
) {
395 VM_WARN_ON_ONCE(!cpumask_test_cpu(cpu
,
396 mm_cpumask(real_prev
)));
397 cpumask_clear_cpu(cpu
, mm_cpumask(real_prev
));
401 * Start remote flushes and then read tlb_gen.
403 if (next
!= &init_mm
)
404 cpumask_set_cpu(cpu
, mm_cpumask(next
));
405 next_tlb_gen
= atomic64_read(&next
->context
.tlb_gen
);
407 choose_new_asid(next
, next_tlb_gen
, &new_asid
, &need_flush
);
409 /* Let nmi_uaccess_okay() know that we're changing CR3. */
410 this_cpu_write(cpu_tlbstate
.loaded_mm
, LOADED_MM_SWITCHING
);
415 this_cpu_write(cpu_tlbstate
.ctxs
[new_asid
].ctx_id
, next
->context
.ctx_id
);
416 this_cpu_write(cpu_tlbstate
.ctxs
[new_asid
].tlb_gen
, next_tlb_gen
);
417 load_new_mm_cr3(next
->pgd
, new_asid
, true);
420 * NB: This gets called via leave_mm() in the idle path
421 * where RCU functions differently. Tracing normally
422 * uses RCU, so we need to use the _rcuidle variant.
424 * (There is no good reason for this. The idle code should
425 * be rearranged to call this before rcu_idle_enter().)
427 trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH
, TLB_FLUSH_ALL
);
429 /* The new ASID is already up to date. */
430 load_new_mm_cr3(next
->pgd
, new_asid
, false);
432 /* See above wrt _rcuidle. */
433 trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH
, 0);
436 /* Make sure we write CR3 before loaded_mm. */
439 this_cpu_write(cpu_tlbstate
.loaded_mm
, next
);
440 this_cpu_write(cpu_tlbstate
.loaded_mm_asid
, new_asid
);
442 if (next
!= real_prev
) {
443 load_mm_cr4_irqsoff(next
);
444 switch_ldt(real_prev
, next
);
449 * Please ignore the name of this function. It should be called
450 * switch_to_kernel_thread().
452 * enter_lazy_tlb() is a hint from the scheduler that we are entering a
453 * kernel thread or other context without an mm. Acceptable implementations
454 * include doing nothing whatsoever, switching to init_mm, or various clever
455 * lazy tricks to try to minimize TLB flushes.
457 * The scheduler reserves the right to call enter_lazy_tlb() several times
458 * in a row. It will notify us that we're going back to a real mm by
459 * calling switch_mm_irqs_off().
461 void enter_lazy_tlb(struct mm_struct
*mm
, struct task_struct
*tsk
)
463 if (this_cpu_read(cpu_tlbstate
.loaded_mm
) == &init_mm
)
466 this_cpu_write(cpu_tlbstate
.is_lazy
, true);
470 * Call this when reinitializing a CPU. It fixes the following potential
473 * - The ASID changed from what cpu_tlbstate thinks it is (most likely
474 * because the CPU was taken down and came back up with CR3's PCID
475 * bits clear. CPU hotplug can do this.
477 * - The TLB contains junk in slots corresponding to inactive ASIDs.
479 * - The CPU went so far out to lunch that it may have missed a TLB
482 void initialize_tlbstate_and_flush(void)
485 struct mm_struct
*mm
= this_cpu_read(cpu_tlbstate
.loaded_mm
);
486 u64 tlb_gen
= atomic64_read(&init_mm
.context
.tlb_gen
);
487 unsigned long cr3
= __read_cr3();
489 /* Assert that CR3 already references the right mm. */
490 WARN_ON((cr3
& CR3_ADDR_MASK
) != __pa(mm
->pgd
));
493 * Assert that CR4.PCIDE is set if needed. (CR4.PCIDE initialization
494 * doesn't work like other CR4 bits because it can only be set from
497 WARN_ON(boot_cpu_has(X86_FEATURE_PCID
) &&
498 !(cr4_read_shadow() & X86_CR4_PCIDE
));
500 /* Force ASID 0 and force a TLB flush. */
501 write_cr3(build_cr3(mm
->pgd
, 0));
503 /* Reinitialize tlbstate. */
504 this_cpu_write(cpu_tlbstate
.last_user_mm_ibpb
, LAST_USER_MM_IBPB
);
505 this_cpu_write(cpu_tlbstate
.loaded_mm_asid
, 0);
506 this_cpu_write(cpu_tlbstate
.next_asid
, 1);
507 this_cpu_write(cpu_tlbstate
.ctxs
[0].ctx_id
, mm
->context
.ctx_id
);
508 this_cpu_write(cpu_tlbstate
.ctxs
[0].tlb_gen
, tlb_gen
);
510 for (i
= 1; i
< TLB_NR_DYN_ASIDS
; i
++)
511 this_cpu_write(cpu_tlbstate
.ctxs
[i
].ctx_id
, 0);
515 * flush_tlb_func_common()'s memory ordering requirement is that any
516 * TLB fills that happen after we flush the TLB are ordered after we
517 * read active_mm's tlb_gen. We don't need any explicit barriers
518 * because all x86 flush operations are serializing and the
519 * atomic64_read operation won't be reordered by the compiler.
521 static void flush_tlb_func_common(const struct flush_tlb_info
*f
,
522 bool local
, enum tlb_flush_reason reason
)
525 * We have three different tlb_gen values in here. They are:
527 * - mm_tlb_gen: the latest generation.
528 * - local_tlb_gen: the generation that this CPU has already caught
530 * - f->new_tlb_gen: the generation that the requester of the flush
531 * wants us to catch up to.
533 struct mm_struct
*loaded_mm
= this_cpu_read(cpu_tlbstate
.loaded_mm
);
534 u32 loaded_mm_asid
= this_cpu_read(cpu_tlbstate
.loaded_mm_asid
);
535 u64 mm_tlb_gen
= atomic64_read(&loaded_mm
->context
.tlb_gen
);
536 u64 local_tlb_gen
= this_cpu_read(cpu_tlbstate
.ctxs
[loaded_mm_asid
].tlb_gen
);
538 /* This code cannot presently handle being reentered. */
539 VM_WARN_ON(!irqs_disabled());
541 if (unlikely(loaded_mm
== &init_mm
))
544 VM_WARN_ON(this_cpu_read(cpu_tlbstate
.ctxs
[loaded_mm_asid
].ctx_id
) !=
545 loaded_mm
->context
.ctx_id
);
547 if (this_cpu_read(cpu_tlbstate
.is_lazy
)) {
549 * We're in lazy mode. We need to at least flush our
550 * paging-structure cache to avoid speculatively reading
551 * garbage into our TLB. Since switching to init_mm is barely
552 * slower than a minimal flush, just switch to init_mm.
554 * This should be rare, with native_flush_tlb_others skipping
555 * IPIs to lazy TLB mode CPUs.
557 switch_mm_irqs_off(NULL
, &init_mm
, NULL
);
561 if (unlikely(local_tlb_gen
== mm_tlb_gen
)) {
563 * There's nothing to do: we're already up to date. This can
564 * happen if two concurrent flushes happen -- the first flush to
565 * be handled can catch us all the way up, leaving no work for
568 trace_tlb_flush(reason
, 0);
572 WARN_ON_ONCE(local_tlb_gen
> mm_tlb_gen
);
573 WARN_ON_ONCE(f
->new_tlb_gen
> mm_tlb_gen
);
576 * If we get to this point, we know that our TLB is out of date.
577 * This does not strictly imply that we need to flush (it's
578 * possible that f->new_tlb_gen <= local_tlb_gen), but we're
579 * going to need to flush in the very near future, so we might
580 * as well get it over with.
582 * The only question is whether to do a full or partial flush.
584 * We do a partial flush if requested and two extra conditions
587 * 1. f->new_tlb_gen == local_tlb_gen + 1. We have an invariant that
588 * we've always done all needed flushes to catch up to
589 * local_tlb_gen. If, for example, local_tlb_gen == 2 and
590 * f->new_tlb_gen == 3, then we know that the flush needed to bring
591 * us up to date for tlb_gen 3 is the partial flush we're
594 * As an example of why this check is needed, suppose that there
595 * are two concurrent flushes. The first is a full flush that
596 * changes context.tlb_gen from 1 to 2. The second is a partial
597 * flush that changes context.tlb_gen from 2 to 3. If they get
598 * processed on this CPU in reverse order, we'll see
599 * local_tlb_gen == 1, mm_tlb_gen == 3, and end != TLB_FLUSH_ALL.
600 * If we were to use __flush_tlb_one_user() and set local_tlb_gen to
601 * 3, we'd be break the invariant: we'd update local_tlb_gen above
602 * 1 without the full flush that's needed for tlb_gen 2.
604 * 2. f->new_tlb_gen == mm_tlb_gen. This is purely an optimiation.
605 * Partial TLB flushes are not all that much cheaper than full TLB
606 * flushes, so it seems unlikely that it would be a performance win
607 * to do a partial flush if that won't bring our TLB fully up to
608 * date. By doing a full flush instead, we can increase
609 * local_tlb_gen all the way to mm_tlb_gen and we can probably
610 * avoid another flush in the very near future.
612 if (f
->end
!= TLB_FLUSH_ALL
&&
613 f
->new_tlb_gen
== local_tlb_gen
+ 1 &&
614 f
->new_tlb_gen
== mm_tlb_gen
) {
616 unsigned long nr_invalidate
= (f
->end
- f
->start
) >> f
->stride_shift
;
617 unsigned long addr
= f
->start
;
619 while (addr
< f
->end
) {
620 __flush_tlb_one_user(addr
);
621 addr
+= 1UL << f
->stride_shift
;
624 count_vm_tlb_events(NR_TLB_LOCAL_FLUSH_ONE
, nr_invalidate
);
625 trace_tlb_flush(reason
, nr_invalidate
);
630 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL
);
631 trace_tlb_flush(reason
, TLB_FLUSH_ALL
);
634 /* Both paths above update our state to mm_tlb_gen. */
635 this_cpu_write(cpu_tlbstate
.ctxs
[loaded_mm_asid
].tlb_gen
, mm_tlb_gen
);
638 static void flush_tlb_func_local(const void *info
, enum tlb_flush_reason reason
)
640 const struct flush_tlb_info
*f
= info
;
642 flush_tlb_func_common(f
, true, reason
);
645 static void flush_tlb_func_remote(void *info
)
647 const struct flush_tlb_info
*f
= info
;
649 inc_irq_stat(irq_tlb_count
);
651 if (f
->mm
&& f
->mm
!= this_cpu_read(cpu_tlbstate
.loaded_mm
))
654 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED
);
655 flush_tlb_func_common(f
, false, TLB_REMOTE_SHOOTDOWN
);
658 static bool tlb_is_not_lazy(int cpu
, void *data
)
660 return !per_cpu(cpu_tlbstate
.is_lazy
, cpu
);
663 void native_flush_tlb_others(const struct cpumask
*cpumask
,
664 const struct flush_tlb_info
*info
)
666 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH
);
667 if (info
->end
== TLB_FLUSH_ALL
)
668 trace_tlb_flush(TLB_REMOTE_SEND_IPI
, TLB_FLUSH_ALL
);
670 trace_tlb_flush(TLB_REMOTE_SEND_IPI
,
671 (info
->end
- info
->start
) >> PAGE_SHIFT
);
673 if (is_uv_system()) {
675 * This whole special case is confused. UV has a "Broadcast
676 * Assist Unit", which seems to be a fancy way to send IPIs.
677 * Back when x86 used an explicit TLB flush IPI, UV was
678 * optimized to use its own mechanism. These days, x86 uses
679 * smp_call_function_many(), but UV still uses a manual IPI,
680 * and that IPI's action is out of date -- it does a manual
681 * flush instead of calling flush_tlb_func_remote(). This
682 * means that the percpu tlb_gen variables won't be updated
683 * and we'll do pointless flushes on future context switches.
685 * Rather than hooking native_flush_tlb_others() here, I think
686 * that UV should be updated so that smp_call_function_many(),
687 * etc, are optimal on UV.
689 cpumask
= uv_flush_tlb_others(cpumask
, info
);
691 smp_call_function_many(cpumask
, flush_tlb_func_remote
,
697 * If no page tables were freed, we can skip sending IPIs to
698 * CPUs in lazy TLB mode. They will flush the CPU themselves
699 * at the next context switch.
701 * However, if page tables are getting freed, we need to send the
702 * IPI everywhere, to prevent CPUs in lazy TLB mode from tripping
703 * up on the new contents of what used to be page tables, while
704 * doing a speculative memory access.
706 if (info
->freed_tables
)
707 smp_call_function_many(cpumask
, flush_tlb_func_remote
,
710 on_each_cpu_cond_mask(tlb_is_not_lazy
, flush_tlb_func_remote
,
711 (void *)info
, 1, cpumask
);
715 * See Documentation/x86/tlb.rst for details. We choose 33
716 * because it is large enough to cover the vast majority (at
717 * least 95%) of allocations, and is small enough that we are
718 * confident it will not cause too much overhead. Each single
719 * flush is about 100 ns, so this caps the maximum overhead at
722 * This is in units of pages.
724 unsigned long tlb_single_page_flush_ceiling __read_mostly
= 33;
726 static DEFINE_PER_CPU_SHARED_ALIGNED(struct flush_tlb_info
, flush_tlb_info
);
728 #ifdef CONFIG_DEBUG_VM
729 static DEFINE_PER_CPU(unsigned int, flush_tlb_info_idx
);
732 static inline struct flush_tlb_info
*get_flush_tlb_info(struct mm_struct
*mm
,
733 unsigned long start
, unsigned long end
,
734 unsigned int stride_shift
, bool freed_tables
,
737 struct flush_tlb_info
*info
= this_cpu_ptr(&flush_tlb_info
);
739 #ifdef CONFIG_DEBUG_VM
741 * Ensure that the following code is non-reentrant and flush_tlb_info
742 * is not overwritten. This means no TLB flushing is initiated by
743 * interrupt handlers and machine-check exception handlers.
745 BUG_ON(this_cpu_inc_return(flush_tlb_info_idx
) != 1);
751 info
->stride_shift
= stride_shift
;
752 info
->freed_tables
= freed_tables
;
753 info
->new_tlb_gen
= new_tlb_gen
;
758 static inline void put_flush_tlb_info(void)
760 #ifdef CONFIG_DEBUG_VM
761 /* Complete reentrency prevention checks */
763 this_cpu_dec(flush_tlb_info_idx
);
767 void flush_tlb_mm_range(struct mm_struct
*mm
, unsigned long start
,
768 unsigned long end
, unsigned int stride_shift
,
771 struct flush_tlb_info
*info
;
777 /* Should we flush just the requested range? */
778 if ((end
== TLB_FLUSH_ALL
) ||
779 ((end
- start
) >> stride_shift
) > tlb_single_page_flush_ceiling
) {
784 /* This is also a barrier that synchronizes with switch_mm(). */
785 new_tlb_gen
= inc_mm_tlb_gen(mm
);
787 info
= get_flush_tlb_info(mm
, start
, end
, stride_shift
, freed_tables
,
790 if (mm
== this_cpu_read(cpu_tlbstate
.loaded_mm
)) {
791 lockdep_assert_irqs_enabled();
793 flush_tlb_func_local(info
, TLB_LOCAL_MM_SHOOTDOWN
);
797 if (cpumask_any_but(mm_cpumask(mm
), cpu
) < nr_cpu_ids
)
798 flush_tlb_others(mm_cpumask(mm
), info
);
800 put_flush_tlb_info();
805 static void do_flush_tlb_all(void *info
)
807 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED
);
811 void flush_tlb_all(void)
813 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH
);
814 on_each_cpu(do_flush_tlb_all
, NULL
, 1);
817 static void do_kernel_range_flush(void *info
)
819 struct flush_tlb_info
*f
= info
;
822 /* flush range by one by one 'invlpg' */
823 for (addr
= f
->start
; addr
< f
->end
; addr
+= PAGE_SIZE
)
824 __flush_tlb_one_kernel(addr
);
827 void flush_tlb_kernel_range(unsigned long start
, unsigned long end
)
829 /* Balance as user space task's flush, a bit conservative */
830 if (end
== TLB_FLUSH_ALL
||
831 (end
- start
) > tlb_single_page_flush_ceiling
<< PAGE_SHIFT
) {
832 on_each_cpu(do_flush_tlb_all
, NULL
, 1);
834 struct flush_tlb_info
*info
;
837 info
= get_flush_tlb_info(NULL
, start
, end
, 0, false, 0);
839 on_each_cpu(do_kernel_range_flush
, info
, 1);
841 put_flush_tlb_info();
847 * arch_tlbbatch_flush() performs a full TLB flush regardless of the active mm.
848 * This means that the 'struct flush_tlb_info' that describes which mappings to
849 * flush is actually fixed. We therefore set a single fixed struct and use it in
850 * arch_tlbbatch_flush().
852 static const struct flush_tlb_info full_flush_tlb_info
= {
855 .end
= TLB_FLUSH_ALL
,
858 void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch
*batch
)
862 if (cpumask_test_cpu(cpu
, &batch
->cpumask
)) {
863 lockdep_assert_irqs_enabled();
865 flush_tlb_func_local(&full_flush_tlb_info
, TLB_LOCAL_SHOOTDOWN
);
869 if (cpumask_any_but(&batch
->cpumask
, cpu
) < nr_cpu_ids
)
870 flush_tlb_others(&batch
->cpumask
, &full_flush_tlb_info
);
872 cpumask_clear(&batch
->cpumask
);
877 static ssize_t
tlbflush_read_file(struct file
*file
, char __user
*user_buf
,
878 size_t count
, loff_t
*ppos
)
883 len
= sprintf(buf
, "%ld\n", tlb_single_page_flush_ceiling
);
884 return simple_read_from_buffer(user_buf
, count
, ppos
, buf
, len
);
887 static ssize_t
tlbflush_write_file(struct file
*file
,
888 const char __user
*user_buf
, size_t count
, loff_t
*ppos
)
894 len
= min(count
, sizeof(buf
) - 1);
895 if (copy_from_user(buf
, user_buf
, len
))
899 if (kstrtoint(buf
, 0, &ceiling
))
905 tlb_single_page_flush_ceiling
= ceiling
;
909 static const struct file_operations fops_tlbflush
= {
910 .read
= tlbflush_read_file
,
911 .write
= tlbflush_write_file
,
912 .llseek
= default_llseek
,
915 static int __init
create_tlb_single_page_flush_ceiling(void)
917 debugfs_create_file("tlb_single_page_flush_ceiling", S_IRUSR
| S_IWUSR
,
918 arch_debugfs_dir
, NULL
, &fops_tlbflush
);
921 late_initcall(create_tlb_single_page_flush_ceiling
);