1 #ifndef _ASM_X86_MMU_CONTEXT_H
2 #define _ASM_X86_MMU_CONTEXT_H
5 #include <linux/atomic.h>
6 #include <linux/mm_types.h>
8 #include <trace/events/tlb.h>
10 #include <asm/pgalloc.h>
11 #include <asm/tlbflush.h>
12 #include <asm/paravirt.h>
14 #ifndef CONFIG_PARAVIRT
15 static inline void paravirt_activate_mm(struct mm_struct
*prev
,
16 struct mm_struct
*next
)
19 #endif /* !CONFIG_PARAVIRT */
21 #ifdef CONFIG_PERF_EVENTS
22 extern struct static_key rdpmc_always_available
;
24 static inline void load_mm_cr4(struct mm_struct
*mm
)
26 if (static_key_false(&rdpmc_always_available
) ||
27 atomic_read(&mm
->context
.perf_rdpmc_allowed
))
28 cr4_set_bits(X86_CR4_PCE
);
30 cr4_clear_bits(X86_CR4_PCE
);
33 static inline void load_mm_cr4(struct mm_struct
*mm
) {}
36 #ifdef CONFIG_MODIFY_LDT_SYSCALL
38 * ldt_structs can be allocated, used, and freed, but they are never
39 * modified while live.
43 * Xen requires page-aligned LDTs with special permissions. This is
44 * needed to prevent us from installing evil descriptors such as
45 * call gates. On native, we could merge the ldt_struct and LDT
46 * allocations, but it's not worth trying to optimize.
48 struct desc_struct
*entries
;
53 * Used for LDT copy/destruction.
55 int init_new_context_ldt(struct task_struct
*tsk
, struct mm_struct
*mm
);
56 void destroy_context_ldt(struct mm_struct
*mm
);
57 #else /* CONFIG_MODIFY_LDT_SYSCALL */
58 static inline int init_new_context_ldt(struct task_struct
*tsk
,
63 static inline void destroy_context_ldt(struct mm_struct
*mm
) {}
66 static inline void load_mm_ldt(struct mm_struct
*mm
)
68 #ifdef CONFIG_MODIFY_LDT_SYSCALL
69 struct ldt_struct
*ldt
;
71 /* lockless_dereference synchronizes with smp_store_release */
72 ldt
= lockless_dereference(mm
->context
.ldt
);
75 * Any change to mm->context.ldt is followed by an IPI to all
76 * CPUs with the mm active. The LDT will not be freed until
77 * after the IPI is handled by all such CPUs. This means that,
78 * if the ldt_struct changes before we return, the values we see
79 * will be safe, and the new values will be loaded before we run
82 * NB: don't try to convert this to use RCU without extreme care.
83 * We would still need IRQs off, because we don't want to change
84 * the local LDT after an IPI loaded a newer value than the one
89 set_ldt(ldt
->entries
, ldt
->size
);
96 DEBUG_LOCKS_WARN_ON(preemptible());
99 static inline void enter_lazy_tlb(struct mm_struct
*mm
, struct task_struct
*tsk
)
102 if (this_cpu_read(cpu_tlbstate
.state
) == TLBSTATE_OK
)
103 this_cpu_write(cpu_tlbstate
.state
, TLBSTATE_LAZY
);
107 static inline int init_new_context(struct task_struct
*tsk
,
108 struct mm_struct
*mm
)
110 init_new_context_ldt(tsk
, mm
);
113 static inline void destroy_context(struct mm_struct
*mm
)
115 destroy_context_ldt(mm
);
118 static inline void switch_mm(struct mm_struct
*prev
, struct mm_struct
*next
,
119 struct task_struct
*tsk
)
121 unsigned cpu
= smp_processor_id();
123 if (likely(prev
!= next
)) {
125 this_cpu_write(cpu_tlbstate
.state
, TLBSTATE_OK
);
126 this_cpu_write(cpu_tlbstate
.active_mm
, next
);
128 cpumask_set_cpu(cpu
, mm_cpumask(next
));
131 * Re-load page tables.
133 * This logic has an ordering constraint:
135 * CPU 0: Write to a PTE for 'next'
136 * CPU 0: load bit 1 in mm_cpumask. if nonzero, send IPI.
137 * CPU 1: set bit 1 in next's mm_cpumask
138 * CPU 1: load from the PTE that CPU 0 writes (implicit)
140 * We need to prevent an outcome in which CPU 1 observes
141 * the new PTE value and CPU 0 observes bit 1 clear in
142 * mm_cpumask. (If that occurs, then the IPI will never
143 * be sent, and CPU 0's TLB will contain a stale entry.)
145 * The bad outcome can occur if either CPU's load is
146 * reordered before that CPU's store, so both CPUs must
147 * execute full barriers to prevent this from happening.
149 * Thus, switch_mm needs a full barrier between the
150 * store to mm_cpumask and any operation that could load
151 * from next->pgd. TLB fills are special and can happen
152 * due to instruction fetches or for no reason at all,
153 * and neither LOCK nor MFENCE orders them.
154 * Fortunately, load_cr3() is serializing and gives the
155 * ordering guarantee we need.
160 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH
, TLB_FLUSH_ALL
);
162 /* Stop flush ipis for the previous mm */
163 cpumask_clear_cpu(cpu
, mm_cpumask(prev
));
165 /* Load per-mm CR4 state */
168 #ifdef CONFIG_MODIFY_LDT_SYSCALL
170 * Load the LDT, if the LDT is different.
172 * It's possible that prev->context.ldt doesn't match
173 * the LDT register. This can happen if leave_mm(prev)
174 * was called and then modify_ldt changed
175 * prev->context.ldt but suppressed an IPI to this CPU.
176 * In this case, prev->context.ldt != NULL, because we
177 * never set context.ldt to NULL while the mm still
178 * exists. That means that next->context.ldt !=
179 * prev->context.ldt, because mms never share an LDT.
181 if (unlikely(prev
->context
.ldt
!= next
->context
.ldt
))
187 this_cpu_write(cpu_tlbstate
.state
, TLBSTATE_OK
);
188 BUG_ON(this_cpu_read(cpu_tlbstate
.active_mm
) != next
);
190 if (!cpumask_test_cpu(cpu
, mm_cpumask(next
))) {
192 * On established mms, the mm_cpumask is only changed
193 * from irq context, from ptep_clear_flush() while in
194 * lazy tlb mode, and here. Irqs are blocked during
195 * schedule, protecting us from simultaneous changes.
197 cpumask_set_cpu(cpu
, mm_cpumask(next
));
200 * We were in lazy tlb mode and leave_mm disabled
201 * tlb flush IPI delivery. We must reload CR3
202 * to make sure to use no freed page tables.
204 * As above, load_cr3() is serializing and orders TLB
205 * fills with respect to the mm_cpumask write.
208 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH
, TLB_FLUSH_ALL
);
216 #define activate_mm(prev, next) \
218 paravirt_activate_mm((prev), (next)); \
219 switch_mm((prev), (next), NULL); \
223 #define deactivate_mm(tsk, mm) \
228 #define deactivate_mm(tsk, mm) \
231 loadsegment(fs, 0); \
235 static inline void arch_dup_mmap(struct mm_struct
*oldmm
,
236 struct mm_struct
*mm
)
238 paravirt_arch_dup_mmap(oldmm
, mm
);
241 static inline void arch_exit_mmap(struct mm_struct
*mm
)
243 paravirt_arch_exit_mmap(mm
);
247 static inline bool is_64bit_mm(struct mm_struct
*mm
)
249 return !config_enabled(CONFIG_IA32_EMULATION
) ||
250 !(mm
->context
.ia32_compat
== TIF_IA32
);
253 static inline bool is_64bit_mm(struct mm_struct
*mm
)
259 static inline void arch_bprm_mm_init(struct mm_struct
*mm
,
260 struct vm_area_struct
*vma
)
265 static inline void arch_unmap(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
266 unsigned long start
, unsigned long end
)
269 * mpx_notify_unmap() goes and reads a rarely-hot
270 * cacheline in the mm_struct. That can be expensive
271 * enough to be seen in profiles.
273 * The mpx_notify_unmap() call and its contents have been
274 * observed to affect munmap() performance on hardware
275 * where MPX is not present.
277 * The unlikely() optimizes for the fast case: no MPX
278 * in the CPU, or no MPX use in the process. Even if
279 * we get this wrong (in the unlikely event that MPX
280 * is widely enabled on some system) the overhead of
281 * MPX itself (reading bounds tables) is expected to
282 * overwhelm the overhead of getting this unlikely()
283 * consistently wrong.
285 if (unlikely(cpu_feature_enabled(X86_FEATURE_MPX
)))
286 mpx_notify_unmap(mm
, vma
, start
, end
);
289 static inline int vma_pkey(struct vm_area_struct
*vma
)
292 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
293 unsigned long vma_pkey_mask
= VM_PKEY_BIT0
| VM_PKEY_BIT1
|
294 VM_PKEY_BIT2
| VM_PKEY_BIT3
;
295 pkey
= (vma
->vm_flags
& vma_pkey_mask
) >> VM_PKEY_SHIFT
;
300 static inline bool __pkru_allows_pkey(u16 pkey
, bool write
)
302 u32 pkru
= read_pkru();
304 if (!__pkru_allows_read(pkru
, pkey
))
306 if (write
&& !__pkru_allows_write(pkru
, pkey
))
313 * We only want to enforce protection keys on the current process
314 * because we effectively have no access to PKRU for other
315 * processes or any way to tell *which * PKRU in a threaded
316 * process we could use.
318 * So do not enforce things if the VMA is not from the current
319 * mm, or if we are in a kernel thread.
321 static inline bool vma_is_foreign(struct vm_area_struct
*vma
)
326 * Should PKRU be enforced on the access to this VMA? If
327 * the VMA is from another process, then PKRU has no
328 * relevance and should not be enforced.
330 if (current
->mm
!= vma
->vm_mm
)
336 static inline bool arch_vma_access_permitted(struct vm_area_struct
*vma
,
337 bool write
, bool execute
, bool foreign
)
339 /* pkeys never affect instruction fetches */
342 /* allow access if the VMA is not one from this process */
343 if (foreign
|| vma_is_foreign(vma
))
345 return __pkru_allows_pkey(vma_pkey(vma
), write
);
348 static inline bool arch_pte_access_permitted(pte_t pte
, bool write
)
350 return __pkru_allows_pkey(pte_flags_pkey(pte_flags(pte
)), write
);
353 #endif /* _ASM_X86_MMU_CONTEXT_H */