1 // SPDX-License-Identifier: GPL-2.0
5 * (C) Copyright 1996 Linus Torvalds
7 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
8 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
12 #include <linux/hugetlb.h>
13 #include <linux/shm.h>
14 #include <linux/ksm.h>
15 #include <linux/mman.h>
16 #include <linux/swap.h>
17 #include <linux/capability.h>
19 #include <linux/swapops.h>
20 #include <linux/highmem.h>
21 #include <linux/security.h>
22 #include <linux/syscalls.h>
23 #include <linux/mmu_notifier.h>
24 #include <linux/uaccess.h>
25 #include <linux/mm-arch-hooks.h>
26 #include <linux/userfaultfd_k.h>
28 #include <asm/cacheflush.h>
29 #include <asm/tlbflush.h>
33 static pmd_t
*get_old_pmd(struct mm_struct
*mm
, unsigned long addr
)
40 pgd
= pgd_offset(mm
, addr
);
41 if (pgd_none_or_clear_bad(pgd
))
44 p4d
= p4d_offset(pgd
, addr
);
45 if (p4d_none_or_clear_bad(p4d
))
48 pud
= pud_offset(p4d
, addr
);
49 if (pud_none_or_clear_bad(pud
))
52 pmd
= pmd_offset(pud
, addr
);
59 static pmd_t
*alloc_new_pmd(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
67 pgd
= pgd_offset(mm
, addr
);
68 p4d
= p4d_alloc(mm
, pgd
, addr
);
71 pud
= pud_alloc(mm
, p4d
, addr
);
75 pmd
= pmd_alloc(mm
, pud
, addr
);
79 VM_BUG_ON(pmd_trans_huge(*pmd
));
84 static void take_rmap_locks(struct vm_area_struct
*vma
)
87 i_mmap_lock_write(vma
->vm_file
->f_mapping
);
89 anon_vma_lock_write(vma
->anon_vma
);
92 static void drop_rmap_locks(struct vm_area_struct
*vma
)
95 anon_vma_unlock_write(vma
->anon_vma
);
97 i_mmap_unlock_write(vma
->vm_file
->f_mapping
);
100 static pte_t
move_soft_dirty_pte(pte_t pte
)
103 * Set soft dirty bit so we can notice
104 * in userspace the ptes were moved.
106 #ifdef CONFIG_MEM_SOFT_DIRTY
107 if (pte_present(pte
))
108 pte
= pte_mksoft_dirty(pte
);
109 else if (is_swap_pte(pte
))
110 pte
= pte_swp_mksoft_dirty(pte
);
115 static void move_ptes(struct vm_area_struct
*vma
, pmd_t
*old_pmd
,
116 unsigned long old_addr
, unsigned long old_end
,
117 struct vm_area_struct
*new_vma
, pmd_t
*new_pmd
,
118 unsigned long new_addr
, bool need_rmap_locks
)
120 struct mm_struct
*mm
= vma
->vm_mm
;
121 pte_t
*old_pte
, *new_pte
, pte
;
122 spinlock_t
*old_ptl
, *new_ptl
;
123 bool force_flush
= false;
124 unsigned long len
= old_end
- old_addr
;
127 * When need_rmap_locks is true, we take the i_mmap_rwsem and anon_vma
128 * locks to ensure that rmap will always observe either the old or the
129 * new ptes. This is the easiest way to avoid races with
130 * truncate_pagecache(), page migration, etc...
132 * When need_rmap_locks is false, we use other ways to avoid
135 * - During exec() shift_arg_pages(), we use a specially tagged vma
136 * which rmap call sites look for using is_vma_temporary_stack().
138 * - During mremap(), new_vma is often known to be placed after vma
139 * in rmap traversal order. This ensures rmap will always observe
140 * either the old pte, or the new pte, or both (the page table locks
141 * serialize access to individual ptes, but only rmap traversal
142 * order guarantees that we won't miss both the old and new ptes).
145 take_rmap_locks(vma
);
148 * We don't have to worry about the ordering of src and dst
149 * pte locks because exclusive mmap_sem prevents deadlock.
151 old_pte
= pte_offset_map_lock(mm
, old_pmd
, old_addr
, &old_ptl
);
152 new_pte
= pte_offset_map(new_pmd
, new_addr
);
153 new_ptl
= pte_lockptr(mm
, new_pmd
);
154 if (new_ptl
!= old_ptl
)
155 spin_lock_nested(new_ptl
, SINGLE_DEPTH_NESTING
);
156 flush_tlb_batched_pending(vma
->vm_mm
);
157 arch_enter_lazy_mmu_mode();
159 for (; old_addr
< old_end
; old_pte
++, old_addr
+= PAGE_SIZE
,
160 new_pte
++, new_addr
+= PAGE_SIZE
) {
161 if (pte_none(*old_pte
))
164 pte
= ptep_get_and_clear(mm
, old_addr
, old_pte
);
166 * If we are remapping a valid PTE, make sure
167 * to flush TLB before we drop the PTL for the
170 * NOTE! Both old and new PTL matter: the old one
171 * for racing with page_mkclean(), the new one to
172 * make sure the physical page stays valid until
173 * the TLB entry for the old mapping has been
176 if (pte_present(pte
))
178 pte
= move_pte(pte
, new_vma
->vm_page_prot
, old_addr
, new_addr
);
179 pte
= move_soft_dirty_pte(pte
);
180 set_pte_at(mm
, new_addr
, new_pte
, pte
);
183 arch_leave_lazy_mmu_mode();
185 flush_tlb_range(vma
, old_end
- len
, old_end
);
186 if (new_ptl
!= old_ptl
)
187 spin_unlock(new_ptl
);
188 pte_unmap(new_pte
- 1);
189 pte_unmap_unlock(old_pte
- 1, old_ptl
);
191 drop_rmap_locks(vma
);
194 #define LATENCY_LIMIT (64 * PAGE_SIZE)
196 unsigned long move_page_tables(struct vm_area_struct
*vma
,
197 unsigned long old_addr
, struct vm_area_struct
*new_vma
,
198 unsigned long new_addr
, unsigned long len
,
199 bool need_rmap_locks
)
201 unsigned long extent
, next
, old_end
;
202 pmd_t
*old_pmd
, *new_pmd
;
203 unsigned long mmun_start
; /* For mmu_notifiers */
204 unsigned long mmun_end
; /* For mmu_notifiers */
206 old_end
= old_addr
+ len
;
207 flush_cache_range(vma
, old_addr
, old_end
);
209 mmun_start
= old_addr
;
211 mmu_notifier_invalidate_range_start(vma
->vm_mm
, mmun_start
, mmun_end
);
213 for (; old_addr
< old_end
; old_addr
+= extent
, new_addr
+= extent
) {
215 next
= (old_addr
+ PMD_SIZE
) & PMD_MASK
;
216 /* even if next overflowed, extent below will be ok */
217 extent
= next
- old_addr
;
218 if (extent
> old_end
- old_addr
)
219 extent
= old_end
- old_addr
;
220 old_pmd
= get_old_pmd(vma
->vm_mm
, old_addr
);
223 new_pmd
= alloc_new_pmd(vma
->vm_mm
, vma
, new_addr
);
226 if (is_swap_pmd(*old_pmd
) || pmd_trans_huge(*old_pmd
)) {
227 if (extent
== HPAGE_PMD_SIZE
) {
229 /* See comment in move_ptes() */
231 take_rmap_locks(vma
);
232 moved
= move_huge_pmd(vma
, old_addr
, new_addr
,
233 old_end
, old_pmd
, new_pmd
);
235 drop_rmap_locks(vma
);
239 split_huge_pmd(vma
, old_pmd
, old_addr
);
240 if (pmd_trans_unstable(old_pmd
))
243 if (pte_alloc(new_vma
->vm_mm
, new_pmd
, new_addr
))
245 next
= (new_addr
+ PMD_SIZE
) & PMD_MASK
;
246 if (extent
> next
- new_addr
)
247 extent
= next
- new_addr
;
248 if (extent
> LATENCY_LIMIT
)
249 extent
= LATENCY_LIMIT
;
250 move_ptes(vma
, old_pmd
, old_addr
, old_addr
+ extent
, new_vma
,
251 new_pmd
, new_addr
, need_rmap_locks
);
254 mmu_notifier_invalidate_range_end(vma
->vm_mm
, mmun_start
, mmun_end
);
256 return len
+ old_addr
- old_end
; /* how much done */
259 static unsigned long move_vma(struct vm_area_struct
*vma
,
260 unsigned long old_addr
, unsigned long old_len
,
261 unsigned long new_len
, unsigned long new_addr
,
262 bool *locked
, struct vm_userfaultfd_ctx
*uf
,
263 struct list_head
*uf_unmap
)
265 struct mm_struct
*mm
= vma
->vm_mm
;
266 struct vm_area_struct
*new_vma
;
267 unsigned long vm_flags
= vma
->vm_flags
;
268 unsigned long new_pgoff
;
269 unsigned long moved_len
;
270 unsigned long excess
= 0;
271 unsigned long hiwater_vm
;
274 bool need_rmap_locks
;
277 * We'd prefer to avoid failure later on in do_munmap:
278 * which may split one vma into three before unmapping.
280 if (mm
->map_count
>= sysctl_max_map_count
- 3)
284 * Advise KSM to break any KSM pages in the area to be moved:
285 * it would be confusing if they were to turn up at the new
286 * location, where they happen to coincide with different KSM
287 * pages recently unmapped. But leave vma->vm_flags as it was,
288 * so KSM can come around to merge on vma and new_vma afterwards.
290 err
= ksm_madvise(vma
, old_addr
, old_addr
+ old_len
,
291 MADV_UNMERGEABLE
, &vm_flags
);
295 new_pgoff
= vma
->vm_pgoff
+ ((old_addr
- vma
->vm_start
) >> PAGE_SHIFT
);
296 new_vma
= copy_vma(&vma
, new_addr
, new_len
, new_pgoff
,
301 moved_len
= move_page_tables(vma
, old_addr
, new_vma
, new_addr
, old_len
,
303 if (moved_len
< old_len
) {
305 } else if (vma
->vm_ops
&& vma
->vm_ops
->mremap
) {
306 err
= vma
->vm_ops
->mremap(new_vma
);
311 * On error, move entries back from new area to old,
312 * which will succeed since page tables still there,
313 * and then proceed to unmap new area instead of old.
315 move_page_tables(new_vma
, new_addr
, vma
, old_addr
, moved_len
,
322 mremap_userfaultfd_prep(new_vma
, uf
);
323 arch_remap(mm
, old_addr
, old_addr
+ old_len
,
324 new_addr
, new_addr
+ new_len
);
327 /* Conceal VM_ACCOUNT so old reservation is not undone */
328 if (vm_flags
& VM_ACCOUNT
) {
329 vma
->vm_flags
&= ~VM_ACCOUNT
;
330 excess
= vma
->vm_end
- vma
->vm_start
- old_len
;
331 if (old_addr
> vma
->vm_start
&&
332 old_addr
+ old_len
< vma
->vm_end
)
337 * If we failed to move page tables we still do total_vm increment
338 * since do_munmap() will decrement it by old_len == new_len.
340 * Since total_vm is about to be raised artificially high for a
341 * moment, we need to restore high watermark afterwards: if stats
342 * are taken meanwhile, total_vm and hiwater_vm appear too high.
343 * If this were a serious issue, we'd add a flag to do_munmap().
345 hiwater_vm
= mm
->hiwater_vm
;
346 vm_stat_account(mm
, vma
->vm_flags
, new_len
>> PAGE_SHIFT
);
348 /* Tell pfnmap has moved from this vma */
349 if (unlikely(vma
->vm_flags
& VM_PFNMAP
))
350 untrack_pfn_moved(vma
);
352 if (do_munmap(mm
, old_addr
, old_len
, uf_unmap
) < 0) {
353 /* OOM: unable to split vma, just get accounts right */
354 vm_unacct_memory(excess
>> PAGE_SHIFT
);
357 mm
->hiwater_vm
= hiwater_vm
;
359 /* Restore VM_ACCOUNT if one or two pieces of vma left */
361 vma
->vm_flags
|= VM_ACCOUNT
;
363 vma
->vm_next
->vm_flags
|= VM_ACCOUNT
;
366 if (vm_flags
& VM_LOCKED
) {
367 mm
->locked_vm
+= new_len
>> PAGE_SHIFT
;
374 static struct vm_area_struct
*vma_to_resize(unsigned long addr
,
375 unsigned long old_len
, unsigned long new_len
, unsigned long *p
)
377 struct mm_struct
*mm
= current
->mm
;
378 struct vm_area_struct
*vma
= find_vma(mm
, addr
);
381 if (!vma
|| vma
->vm_start
> addr
)
382 return ERR_PTR(-EFAULT
);
385 * !old_len is a special case where an attempt is made to 'duplicate'
386 * a mapping. This makes no sense for private mappings as it will
387 * instead create a fresh/new mapping unrelated to the original. This
388 * is contrary to the basic idea of mremap which creates new mappings
389 * based on the original. There are no known use cases for this
390 * behavior. As a result, fail such attempts.
392 if (!old_len
&& !(vma
->vm_flags
& (VM_SHARED
| VM_MAYSHARE
))) {
393 pr_warn_once("%s (%d): attempted to duplicate a private mapping with mremap. This is not supported.\n", current
->comm
, current
->pid
);
394 return ERR_PTR(-EINVAL
);
397 if (is_vm_hugetlb_page(vma
))
398 return ERR_PTR(-EINVAL
);
400 /* We can't remap across vm area boundaries */
401 if (old_len
> vma
->vm_end
- addr
)
402 return ERR_PTR(-EFAULT
);
404 if (new_len
== old_len
)
407 /* Need to be careful about a growing mapping */
408 pgoff
= (addr
- vma
->vm_start
) >> PAGE_SHIFT
;
409 pgoff
+= vma
->vm_pgoff
;
410 if (pgoff
+ (new_len
>> PAGE_SHIFT
) < pgoff
)
411 return ERR_PTR(-EINVAL
);
413 if (vma
->vm_flags
& (VM_DONTEXPAND
| VM_PFNMAP
))
414 return ERR_PTR(-EFAULT
);
416 if (vma
->vm_flags
& VM_LOCKED
) {
417 unsigned long locked
, lock_limit
;
418 locked
= mm
->locked_vm
<< PAGE_SHIFT
;
419 lock_limit
= rlimit(RLIMIT_MEMLOCK
);
420 locked
+= new_len
- old_len
;
421 if (locked
> lock_limit
&& !capable(CAP_IPC_LOCK
))
422 return ERR_PTR(-EAGAIN
);
425 if (!may_expand_vm(mm
, vma
->vm_flags
,
426 (new_len
- old_len
) >> PAGE_SHIFT
))
427 return ERR_PTR(-ENOMEM
);
429 if (vma
->vm_flags
& VM_ACCOUNT
) {
430 unsigned long charged
= (new_len
- old_len
) >> PAGE_SHIFT
;
431 if (security_vm_enough_memory_mm(mm
, charged
))
432 return ERR_PTR(-ENOMEM
);
439 static unsigned long mremap_to(unsigned long addr
, unsigned long old_len
,
440 unsigned long new_addr
, unsigned long new_len
, bool *locked
,
441 struct vm_userfaultfd_ctx
*uf
,
442 struct list_head
*uf_unmap_early
,
443 struct list_head
*uf_unmap
)
445 struct mm_struct
*mm
= current
->mm
;
446 struct vm_area_struct
*vma
;
447 unsigned long ret
= -EINVAL
;
448 unsigned long charged
= 0;
449 unsigned long map_flags
;
451 if (offset_in_page(new_addr
))
454 if (new_len
> TASK_SIZE
|| new_addr
> TASK_SIZE
- new_len
)
457 /* Ensure the old/new locations do not overlap */
458 if (addr
+ old_len
> new_addr
&& new_addr
+ new_len
> addr
)
461 ret
= do_munmap(mm
, new_addr
, new_len
, uf_unmap_early
);
465 if (old_len
>= new_len
) {
466 ret
= do_munmap(mm
, addr
+new_len
, old_len
- new_len
, uf_unmap
);
467 if (ret
&& old_len
!= new_len
)
472 vma
= vma_to_resize(addr
, old_len
, new_len
, &charged
);
478 map_flags
= MAP_FIXED
;
479 if (vma
->vm_flags
& VM_MAYSHARE
)
480 map_flags
|= MAP_SHARED
;
482 ret
= get_unmapped_area(vma
->vm_file
, new_addr
, new_len
, vma
->vm_pgoff
+
483 ((addr
- vma
->vm_start
) >> PAGE_SHIFT
),
485 if (offset_in_page(ret
))
488 ret
= move_vma(vma
, addr
, old_len
, new_len
, new_addr
, locked
, uf
,
490 if (!(offset_in_page(ret
)))
493 vm_unacct_memory(charged
);
499 static int vma_expandable(struct vm_area_struct
*vma
, unsigned long delta
)
501 unsigned long end
= vma
->vm_end
+ delta
;
502 if (end
< vma
->vm_end
) /* overflow */
504 if (vma
->vm_next
&& vma
->vm_next
->vm_start
< end
) /* intersection */
506 if (get_unmapped_area(NULL
, vma
->vm_start
, end
- vma
->vm_start
,
507 0, MAP_FIXED
) & ~PAGE_MASK
)
513 * Expand (or shrink) an existing mapping, potentially moving it at the
514 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
516 * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
517 * This option implies MREMAP_MAYMOVE.
519 SYSCALL_DEFINE5(mremap
, unsigned long, addr
, unsigned long, old_len
,
520 unsigned long, new_len
, unsigned long, flags
,
521 unsigned long, new_addr
)
523 struct mm_struct
*mm
= current
->mm
;
524 struct vm_area_struct
*vma
;
525 unsigned long ret
= -EINVAL
;
526 unsigned long charged
= 0;
528 struct vm_userfaultfd_ctx uf
= NULL_VM_UFFD_CTX
;
529 LIST_HEAD(uf_unmap_early
);
532 if (flags
& ~(MREMAP_FIXED
| MREMAP_MAYMOVE
))
535 if (flags
& MREMAP_FIXED
&& !(flags
& MREMAP_MAYMOVE
))
538 if (offset_in_page(addr
))
541 old_len
= PAGE_ALIGN(old_len
);
542 new_len
= PAGE_ALIGN(new_len
);
545 * We allow a zero old-len as a special case
546 * for DOS-emu "duplicate shm area" thing. But
547 * a zero new-len is nonsensical.
552 if (down_write_killable(¤t
->mm
->mmap_sem
))
555 if (flags
& MREMAP_FIXED
) {
556 ret
= mremap_to(addr
, old_len
, new_addr
, new_len
,
557 &locked
, &uf
, &uf_unmap_early
, &uf_unmap
);
562 * Always allow a shrinking remap: that just unmaps
563 * the unnecessary pages..
564 * do_munmap does all the needed commit accounting
566 if (old_len
>= new_len
) {
567 ret
= do_munmap(mm
, addr
+new_len
, old_len
- new_len
, &uf_unmap
);
568 if (ret
&& old_len
!= new_len
)
575 * Ok, we need to grow..
577 vma
= vma_to_resize(addr
, old_len
, new_len
, &charged
);
583 /* old_len exactly to the end of the area..
585 if (old_len
== vma
->vm_end
- addr
) {
586 /* can we just expand the current mapping? */
587 if (vma_expandable(vma
, new_len
- old_len
)) {
588 int pages
= (new_len
- old_len
) >> PAGE_SHIFT
;
590 if (vma_adjust(vma
, vma
->vm_start
, addr
+ new_len
,
591 vma
->vm_pgoff
, NULL
)) {
596 vm_stat_account(mm
, vma
->vm_flags
, pages
);
597 if (vma
->vm_flags
& VM_LOCKED
) {
598 mm
->locked_vm
+= pages
;
608 * We weren't able to just expand or shrink the area,
609 * we need to create a new one and move it..
612 if (flags
& MREMAP_MAYMOVE
) {
613 unsigned long map_flags
= 0;
614 if (vma
->vm_flags
& VM_MAYSHARE
)
615 map_flags
|= MAP_SHARED
;
617 new_addr
= get_unmapped_area(vma
->vm_file
, 0, new_len
,
619 ((addr
- vma
->vm_start
) >> PAGE_SHIFT
),
621 if (offset_in_page(new_addr
)) {
626 ret
= move_vma(vma
, addr
, old_len
, new_len
, new_addr
,
627 &locked
, &uf
, &uf_unmap
);
630 if (offset_in_page(ret
)) {
631 vm_unacct_memory(charged
);
634 up_write(¤t
->mm
->mmap_sem
);
635 if (locked
&& new_len
> old_len
)
636 mm_populate(new_addr
+ old_len
, new_len
- old_len
);
637 userfaultfd_unmap_complete(mm
, &uf_unmap_early
);
638 mremap_userfaultfd_complete(&uf
, addr
, new_addr
, old_len
);
639 userfaultfd_unmap_complete(mm
, &uf_unmap
);