| blob | f557e90fc719bd91e9eed81bda1b51900eb9a269 |
1 /*
2 * mm/mmap.c
3 *
4 * Written by obz.
5 *
6 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
7 */
11 #include <linux/kernel.h>
12 #include <linux/slab.h>
13 #include <linux/backing-dev.h>
14 #include <linux/mm.h>
15 #include <linux/vmacache.h>
16 #include <linux/shm.h>
17 #include <linux/mman.h>
18 #include <linux/pagemap.h>
19 #include <linux/swap.h>
20 #include <linux/syscalls.h>
21 #include <linux/capability.h>
22 #include <linux/init.h>
23 #include <linux/file.h>
24 #include <linux/fs.h>
25 #include <linux/personality.h>
26 #include <linux/security.h>
27 #include <linux/hugetlb.h>
28 #include <linux/profile.h>
29 #include <linux/export.h>
30 #include <linux/mount.h>
31 #include <linux/mempolicy.h>
32 #include <linux/rmap.h>
33 #include <linux/mmu_notifier.h>
34 #include <linux/perf_event.h>
35 #include <linux/audit.h>
36 #include <linux/khugepaged.h>
37 #include <linux/uprobes.h>
38 #include <linux/rbtree_augmented.h>
39 #include <linux/sched/sysctl.h>
40 #include <linux/notifier.h>
41 #include <linux/memory.h>
42 #include <linux/printk.h>
44 #include <asm/uaccess.h>
45 #include <asm/cacheflush.h>
46 #include <asm/tlb.h>
47 #include <asm/mmu_context.h>
51 #ifndef arch_mmap_check
52 #define arch_mmap_check(addr, len, flags) (0)
53 #endif
55 #ifndef arch_rebalance_pgtables
56 #define arch_rebalance_pgtables(addr, len) (addr)
57 #endif
63 /* description of effects of mapping type and prot in current implementation.
64 * this is due to the limited x86 page protection hardware. The expected
65 * behavior is in parens:
66 *
67 * map_type prot
68 * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC
69 * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes
70 * w: (no) no w: (no) no w: (yes) yes w: (no) no
71 * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
72 *
73 * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes
74 * w: (no) no w: (no) no w: (copy) copy w: (no) no
75 * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
76 *
77 */
81 };
84 {
88 }
97 /*
98 * Make sure vm_committed_as in one cacheline and not cacheline shared with
99 * other variables. It can be updated by several CPUs frequently.
100 */
103 /*
104 * The global memory commitment made in the system can be a metric
105 * that can be used to drive ballooning decisions when Linux is hosted
106 * as a guest. On Hyper-V, the host implements a policy engine for dynamically
107 * balancing memory across competing virtual machines that are hosted.
108 * Several metrics drive this policy engine including the guest reported
109 * memory commitment.
110 */
112 {
114 }
117 /*
118 * Check that a process has enough memory to allocate a new virtual
119 * mapping. 0 means there is enough memory for the allocation to
120 * succeed and -ENOMEM implies there is not.
121 *
122 * We currently support three overcommit policies, which are set via the
123 * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
124 *
125 * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
126 * Additional code 2002 Jul 20 by Robert Love.
127 *
128 * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
129 *
130 * Note this is a helper function intended to be used by LSMs which
131 * wish to use this logic.
132 */
134 {
139 /*
140 * Sometimes we want to use more memory than we have
141 */
149 /*
150 * shmem pages shouldn't be counted as free in this
151 * case, they can't be purged, only swapped out, and
152 * that won't affect the overall amount of available
153 * memory in the system.
154 */
159 /*
160 * Any slabs which are created with the
161 * SLAB_RECLAIM_ACCOUNT flag claim to have contents
162 * which are reclaimable, under pressure. The dentry
163 * cache and most inode caches should fall into this
164 */
167 /*
168 * Leave reserved pages. The pages are not for anonymous pages.
169 */
172 else
175 /*
176 * Reserve some for root
177 */
185 }
188 /*
189 * Reserve some for root
190 */
194 /*
195 * Don't let a single process grow so big a user can't recover
196 */
200 }
204 error:
208 }
210 /*
211 * Requires inode->i_mapping->i_mmap_mutex
212 */
215 {
224 }
226 /*
227 * Unlink a file-based vm structure from its interval tree, to hide
228 * vma from rmap and vmtruncate before freeing its page tables.
229 */
231 {
239 }
240 }
242 /*
243 * Close a vm structure and free it, returning the next.
244 */
246 {
257 }
262 {
272 #ifdef CONFIG_COMPAT_BRK
273 /*
274 * CONFIG_COMPAT_BRK can still be overridden by setting
275 * randomize_va_space to 2, which will still cause mm->start_brk
276 * to be arbitrarily shifted
277 */
280 else
282 #else
284 #endif
288 /*
289 * Check against rlimit here. If this check is done later after the test
290 * of oldbrk with newbrk then it can escape the test and let the data
291 * segment grow beyond its set limit the in case where the limit is
292 * not page aligned -Ram Gupta
293 */
304 /* Always allow shrinking brk. */
309 }
311 /* Check against existing mmap mappings. */
316 /* Ok, looks good - let it rip. */
320 set_brk:
328 out:
332 }
335 {
338 /*
339 * Note: in the rare case of a VM_GROWSDOWN above a VM_GROWSUP, we
340 * allow two stack_guard_gaps between them here, and when choosing
341 * an unmapped area; whereas when expanding we only require one.
342 * That's a little inconsistent, but keeps the code here simpler.
343 */
349 else
351 }
357 }
363 }
365 }
367 #ifdef CONFIG_DEBUG_VM_RB
369 {
380 }
384 }
389 }
395 }
396 i++;
400 }
403 j++;
407 }
409 }
412 {
420 }
421 }
424 {
438 }
442 i++;
443 }
447 }
452 }
457 }
459 }
460 #else
461 #define validate_mm_rb(root, ignore) do { } while (0)
462 #define validate_mm(mm) do { } while (0)
463 #endif
468 /*
469 * Update augmented rbtree rb_subtree_gap values after vma->vm_start or
470 * vma->vm_prev->vm_end values changed, without modifying the vma's position
471 * in the rbtree.
472 */
474 {
475 /*
476 * As it turns out, RB_DECLARE_CALLBACKS() already created a callback
477 * function that does exacltly what we want.
478 */
480 }
484 {
485 /* All rb_subtree_gap values must be consistent prior to insertion */
489 }
492 {
493 /*
494 * All rb_subtree_gap values must be consistent prior to erase,
495 * with the possible exception of the vma being erased.
496 */
499 /*
500 * Note rb_erase_augmented is a fairly large inline function,
501 * so make sure we instantiate it only once with our desired
502 * augmented rbtree callbacks.
503 */
505 }
507 /*
508 * vma has some anon_vma assigned, and is already inserted on that
509 * anon_vma's interval trees.
510 *
511 * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
512 * vma must be removed from the anon_vma's interval trees using
513 * anon_vma_interval_tree_pre_update_vma().
514 *
515 * After the update, the vma will be reinserted using
516 * anon_vma_interval_tree_post_update_vma().
517 *
518 * The entire update must be protected by exclusive mmap_sem and by
519 * the root anon_vma's mutex.
520 */
523 {
528 }
532 {
537 }
542 {
555 /* Fail if an existing vma overlaps the area */
562 }
563 }
571 }
575 {
579 /* Find first overlaping mapping */
587 /* Iterate over the rest of the overlaps */
596 }
599 }
603 {
604 /* Update tracking information for the gap following the new vma. */
607 else
610 /*
611 * vma->vm_prev wasn't known when we followed the rbtree to find the
612 * correct insertion point for that vma. As a result, we could not
613 * update the vma vm_rb parents rb_subtree_gap values on the way down.
614 * So, we first insert the vma with a zero rb_subtree_gap value
615 * (to be consistent with what we did on the way down), and then
616 * immediately update the gap to the correct value. Finally we
617 * rebalance the rbtree after all augmented values have been set.
618 */
623 }
626 {
641 }
642 }
644 static void
648 {
651 }
656 {
662 }
672 }
674 /*
675 * Helper for vma_adjust() in the split_vma insert case: insert a vma into the
676 * mm's list and rbtree. It has already been inserted into the interval tree.
677 */
679 {
688 }
693 {
701 /* Kill the cache */
703 }
705 /*
706 * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
707 * is already present in an i_mmap tree without adjusting the tree.
708 * The following helper function should be used when such adjustments
709 * are necessary. The "insert" vma (if any) is to be inserted
710 * before we drop the necessary locks.
711 */
714 {
730 /*
731 * vma expands, overlapping all the next, and
732 * perhaps the one after too (mprotect case 6).
733 */
739 /*
740 * vma expands, overlapping part of the next:
741 * mprotect case 5 shifting the boundary up.
742 */
747 /*
748 * vma shrinks, and !insert tells it's not
749 * split_vma inserting another: so it must be
750 * mprotect case 4 shifting the boundary down.
751 */
755 }
757 /*
758 * Easily overlooked: when mprotect shifts the boundary,
759 * make sure the expanding vma has anon_vma set if the
760 * shrinking vma had, to cover any anon pages imported.
761 */
769 }
770 }
782 /*
783 * Put into interval tree now, so instantiated pages
784 * are visible to arm/parisc __flush_dcache_page
785 * throughout; but we cannot insert into address
786 * space until vma start or end is updated.
787 */
789 }
790 }
804 }
811 }
816 }
820 }
825 }
832 }
835 /*
836 * vma_merge has merged next into vma, and needs
837 * us to remove next before dropping the locks.
838 */
843 /*
844 * split_vma has split insert from vma, and needs
845 * us to insert it before dropping the locks
846 * (it may either follow vma or precede it).
847 */
857 }
858 }
865 }
874 }
880 }
886 /*
887 * In mprotect's case 6 (see comments on vma_merge),
888 * we must remove another next too. It would clutter
889 * up the code too much to do both in one go.
890 */
896 else
898 }
905 }
907 /*
908 * If the vma has a ->close operation then the driver probably needs to release
909 * per-vma resources, so we don't attempt to merge those.
910 */
913 {
914 /*
915 * VM_SOFTDIRTY should not prevent from VMA merging, if we
916 * match the flags but dirty bit -- the caller should mark
917 * merged VMA as dirty. If dirty bit won't be excluded from
918 * comparison, we increase pressue on the memory system forcing
919 * the kernel to generate new VMAs when old one could be
920 * extended instead.
921 */
929 }
934 {
935 /*
936 * The list_is_singular() test is to avoid merging VMA cloned from
937 * parents. This can improve scalability caused by anon_vma lock.
938 */
943 }
945 /*
946 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
947 * in front of (at a lower virtual address and file offset than) the vma.
948 *
949 * We cannot merge two vmas if they have differently assigned (non-NULL)
950 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
951 *
952 * We don't check here for the merged mmap wrapping around the end of pagecache
953 * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
954 * wrap, nor mmaps which cover the final page at index -1UL.
955 */
956 static int
959 {
964 }
966 }
968 /*
969 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
970 * beyond (at a higher virtual address and file offset than) the vma.
971 *
972 * We cannot merge two vmas if they have differently assigned (non-NULL)
973 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
974 */
975 static int
978 {
981 pgoff_t vm_pglen;
985 }
987 }
989 /*
990 * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
991 * whether that can be merged with its predecessor or its successor.
992 * Or both (it neatly fills a hole).
993 *
994 * In most cases - when called for mmap, brk or mremap - [addr,end) is
995 * certain not to be mapped by the time vma_merge is called; but when
996 * called for mprotect, it is certain to be already mapped (either at
997 * an offset within prev, or at the start of next), and the flags of
998 * this area are about to be changed to vm_flags - and the no-change
999 * case has already been eliminated.
1000 *
1001 * The following mprotect cases have to be considered, where AAAA is
1002 * the area passed down from mprotect_fixup, never extending beyond one
1003 * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
1004 *
1005 * AAAA AAAA AAAA AAAA
1006 * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN PPPPNNNNXXXX
1007 * cannot merge might become might become might become
1008 * PPNNNNNNNNNN PPPPPPPPPPNN PPPPPPPPPPPP 6 or
1009 * mmap, brk or case 4 below case 5 below PPPPPPPPXXXX 7 or
1010 * mremap move: PPPPNNNNNNNN 8
1011 * AAAA
1012 * PPPP NNNN PPPPPPPPPPPP PPPPPPPPNNNN PPPPNNNNNNNN
1013 * might become case 1 below case 2 below case 3 below
1014 *
1015 * Odd one out? Case 8, because it extends NNNN but needs flags of XXXX:
1016 * mprotect_fixup updates vm_flags & vm_page_prot on successful return.
1017 */
1023 {
1028 /*
1029 * We later require that vma->vm_flags == vm_flags,
1030 * so this tests vma->vm_flags & VM_SPECIAL, too.
1031 */
1037 else
1043 /*
1044 * Can it merge with the predecessor?
1045 */
1050 /*
1051 * OK, it can. Can we now merge in the successor as well?
1052 */
1059 /* cases 1, 6 */
1069 }
1071 /*
1072 * Can this new request be merged in front of next?
1073 */
1088 }
1091 }
1093 /*
1094 * Rough compatbility check to quickly see if it's even worth looking
1095 * at sharing an anon_vma.
1096 *
1097 * They need to have the same vm_file, and the flags can only differ
1098 * in things that mprotect may change.
1099 *
1100 * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
1101 * we can merge the two vma's. For example, we refuse to merge a vma if
1102 * there is a vm_ops->close() function, because that indicates that the
1103 * driver is doing some kind of reference counting. But that doesn't
1104 * really matter for the anon_vma sharing case.
1105 */
1107 {
1113 }
1115 /*
1116 * Do some basic sanity checking to see if we can re-use the anon_vma
1117 * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
1118 * the same as 'old', the other will be the new one that is trying
1119 * to share the anon_vma.
1120 *
1121 * NOTE! This runs with mm_sem held for reading, so it is possible that
1122 * the anon_vma of 'old' is concurrently in the process of being set up
1123 * by another page fault trying to merge _that_. But that's ok: if it
1124 * is being set up, that automatically means that it will be a singleton
1125 * acceptable for merging, so we can do all of this optimistically. But
1126 * we do that ACCESS_ONCE() to make sure that we never re-load the pointer.
1127 *
1128 * IOW: that the "list_is_singular()" test on the anon_vma_chain only
1129 * matters for the 'stable anon_vma' case (ie the thing we want to avoid
1130 * is to return an anon_vma that is "complex" due to having gone through
1131 * a fork).
1132 *
1133 * We also make sure that the two vma's are compatible (adjacent,
1134 * and with the same memory policies). That's all stable, even with just
1135 * a read lock on the mm_sem.
1136 */
1137 static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
1138 {
1144 }
1146 }
1148 /*
1149 * find_mergeable_anon_vma is used by anon_vma_prepare, to check
1150 * neighbouring vmas for a suitable anon_vma, before it goes off
1151 * to allocate a new anon_vma. It checks because a repetitive
1152 * sequence of mprotects and faults may otherwise lead to distinct
1153 * anon_vmas being allocated, preventing vma merge in subsequent
1154 * mprotect.
1155 */
1157 {
1168 try_prev:
1176 none:
1177 /*
1178 * There's no absolute need to look only at touching neighbours:
1179 * we could search further afield for "compatible" anon_vmas.
1180 * But it would probably just be a waste of time searching,
1181 * or lead to too many vmas hanging off the same anon_vma.
1182 * We're trying to allow mprotect remerging later on,
1183 * not trying to minimize memory used for anon_vmas.
1184 */
1186 }
1188 #ifdef CONFIG_PROC_FS
1191 {
1192 const unsigned long stack_flags
1203 }
1206 /*
1207 * If a hint addr is less than mmap_min_addr change hint to be as
1208 * low as possible but still greater than mmap_min_addr
1209 */
1211 {
1217 }
1222 {
1225 /* mlock MCL_FUTURE? */
1233 }
1235 }
1238 {
1245 /* Special "we do even unsigned file positions" case */
1249 /* Yes, random drivers might want more. But I'm tired of buggy drivers */
1251 }
1255 {
1264 }
1266 /*
1267 * The caller must hold down_write(¤t->mm->mmap_sem).
1268 */
1274 {
1276 vm_flags_t vm_flags;
1280 /*
1281 * Does the application expect PROT_READ to imply PROT_EXEC?
1282 *
1283 * (the exception is when the underlying filesystem is noexec
1284 * mounted, in which case we dont add PROT_EXEC.)
1285 */
1296 /* Careful about overflows.. */
1301 /* offset overflow? */
1305 /* Too many mappings? */
1309 /* Obtain the address to map to. we verify (or select) it and ensure
1310 * that it represents a valid section of the address space.
1311 */
1316 /* Do simple checking here so the lower-level routines won't have
1317 * to. we assume access permissions have been handled by the open
1318 * of the memory object, so we don't do any here.
1319 */
1341 /*
1342 * Make sure we don't allow writing to an append-only
1343 * file..
1344 */
1348 /*
1349 * Make sure there are no mandatory locks on the file.
1350 */
1358 /* fall through */
1366 }
1376 }
1382 /*
1383 * Ignore pgoff.
1384 */
1389 /*
1390 * Set pgoff according to addr for anon_vma.
1391 */
1396 }
1397 }
1399 /*
1400 * Set 'VM_NORESERVE' if we should not account for the
1401 * memory use of this mapping.
1402 */
1404 /* We honor MAP_NORESERVE if allowed to overcommit */
1408 /* hugetlb applies strict overcommit unless MAP_NORESERVE */
1411 }
1419 }
1424 {
1447 /*
1448 * VM_NORESERVE is used because the reservations will be
1449 * taken when vm_ops->mmap() is called
1450 * A dummy user value is used because we are not locking
1451 * memory so no accounting is necessary
1452 */
1454 VM_NORESERVE,
1459 }
1464 out_fput:
1467 out:
1469 }
1471 #ifdef __ARCH_WANT_SYS_OLD_MMAP
1479 };
1482 {
1492 }
1495 /*
1496 * Some shared mappigns will want the pages marked read-only
1497 * to track write events. If so, we'll downgrade vm_page_prot
1498 * to the private version (using protection_map[] without the
1499 * VM_SHARED bit).
1500 */
1502 {
1505 /* If it was private or non-writable, the write bit is already clear */
1509 /* The backer wishes to know when pages are first written to? */
1513 /* The open routine did something to the protections already? */
1518 /* Specialty mapping? */
1522 /* Can the mapping track the dirty pages? */
1525 }
1527 /*
1528 * We account for memory if it's a private writeable mapping,
1529 * not hugepages and VM_NORESERVE wasn't set.
1530 */
1532 {
1533 /*
1534 * hugetlb has its own accounting separate from the core VM
1535 * VM_HUGETLB may not be set yet so we cannot check for that flag.
1536 */
1541 }
1545 {
1552 /* Check against address space limit. */
1556 /*
1557 * MAP_FIXED may remove pages of mappings that intersects with
1558 * requested mapping. Account for the pages it would unmap.
1559 */
1567 }
1569 /* Clear old maps */
1571 munmap_back:
1576 }
1578 /*
1579 * Private writable mapping: check memory availability
1580 */
1586 }
1588 /*
1589 * Can we just expand an old mapping?
1590 */
1595 /*
1596 * Determine the object being mapped and call the appropriate
1597 * specific mapper. the address has already been validated, but
1598 * not unmapped, but the maps are removed from the list.
1599 */
1604 }
1619 }
1625 /* Can addr have changed??
1626 *
1627 * Answer: Yes, several device drivers can do it in their
1628 * f_op->mmap method. -DaveM
1629 * Bug: If addr is changed, prev, rb_link, rb_parent should
1630 * be updated for vma_link()
1631 */
1640 }
1645 /* Can vma->vm_page_prot have changed??
1646 *
1647 * Answer: Yes, drivers may have changed it in their
1648 * f_op->mmap method.
1649 *
1650 * Ensures that vmas marked as uncached stay that way.
1651 */
1655 }
1658 /* Once vma denies write, undo our temporary denial count */
1662 out:
1670 else
1672 }
1677 /*
1678 * New (or expanded) vma always get soft dirty status.
1679 * Otherwise user-space soft-dirty page tracker won't
1680 * be able to distinguish situation when vma area unmapped,
1681 * then new mapped in-place (which must be aimed as
1682 * a completely new data area).
1683 */
1688 unmap_and_free_vma:
1694 /* Undo any partial mapping done by a device driver. */
1697 free_vma:
1699 unacct_error:
1703 }
1706 {
1707 /*
1708 * We implement the search by looking for an rbtree node that
1709 * immediately follows a suitable gap. That is,
1710 * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length;
1711 * - gap_end = vma->vm_start >= info->low_limit + length;
1712 * - gap_end - gap_start >= length
1713 */
1719 /* Adjust search length to account for worst case alignment overhead */
1724 /* Adjust search limits by the desired length */
1733 /* Check if rbtree root looks promising */
1741 /* Visit left subtree if it looks promising */
1750 }
1751 }
1754 check_current:
1755 /* Check if current node has a suitable gap */
1762 /* Visit right subtree if it looks promising */
1770 }
1771 }
1773 /* Go back up the rbtree to find next candidate node */
1784 }
1785 }
1786 }
1788 check_highest:
1789 /* Check highest gap, which does not precede any rbtree node */
1795 found:
1796 /* We found a suitable gap. Clip it with the original low_limit. */
1800 /* Adjust gap address to the desired alignment */
1806 }
1809 {
1814 /* Adjust search length to account for worst case alignment overhead */
1819 /*
1820 * Adjust search limits by the desired length.
1821 * See implementation comment at top of unmapped_area().
1822 */
1832 /* Check highest gap, which does not precede any rbtree node */
1837 /* Check if rbtree root looks promising */
1845 /* Visit right subtree if it looks promising */
1854 }
1855 }
1857 check_current:
1858 /* Check if current node has a suitable gap */
1866 /* Visit left subtree if it looks promising */
1874 }
1875 }
1877 /* Go back up the rbtree to find next candidate node */
1888 }
1889 }
1890 }
1892 found:
1893 /* We found a suitable gap. Clip it with the original high_limit. */
1897 found_highest:
1898 /* Compute highest gap address at the desired alignment */
1905 }
1907 /* Get an address range which is currently unmapped.
1908 * For shmat() with addr=0.
1909 *
1910 * Ugly calling convention alert:
1911 * Return value with the low bits set means error value,
1912 * ie
1913 * if (ret & ~PAGE_MASK)
1914 * error = ret;
1915 *
1916 * This function "knows" that -ENOMEM has the bits set.
1917 */
1918 #ifndef HAVE_ARCH_UNMAPPED_AREA
1919 unsigned long
1922 {
1940 }
1948 }
1949 #endif
1951 /*
1952 * This mmap-allocator allocates new areas top-down from below the
1953 * stack's low limit (the base):
1954 */
1955 #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
1956 unsigned long
1960 {
1966 /* requested length too big for entire address space */
1973 /* requesting a specific address */
1981 }
1990 /*
1991 * A failed mmap() very likely causes application failure,
1992 * so fall back to the bottom-up function here. This scenario
1993 * can happen with large stack limits and large mmap()
1994 * allocations.
1995 */
2002 }
2005 }
2006 #endif
2008 unsigned long
2011 {
2019 /* Careful about overflows.. */
2038 }
2042 /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
2044 {
2048 /* Check the cache first. */
2068 }
2073 }
2077 /*
2078 * Same as find_vma, but also return a pointer to the previous VMA in *pprev.
2079 */
2083 {
2095 }
2096 }
2098 }
2100 /*
2101 * Verify that the stack growth is acceptable and
2102 * update accounting. This is shared with both the
2103 * grow-up and grow-down cases.
2104 */
2107 {
2112 /* address space limit tests */
2116 /* Stack limit test */
2120 /* mlock limit tests */
2129 }
2131 /* Check to ensure the stack will not grow into a hugetlb-only region */
2137 /*
2138 * Overcommit.. This must be the final test, as it will
2139 * update security statistics.
2140 */
2144 /* Ok, everything looks good - let it rip */
2149 }
2151 #if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
2152 /*
2153 * PA-RISC uses this for its stack; IA64 for its Register Backing Store.
2154 * vma is the last one with address > vma->vm_end. Have to extend vma.
2155 */
2157 {
2165 /* Guard against exceeding limits of the address space. */
2171 /* Enforce stack_guard_gap */
2174 /* Guard against overflow */
2183 /* Check that both stack segments have the same anon_vma? */
2184 }
2186 /* We must make sure the anon_vma is allocated. */
2190 /*
2191 * vma->vm_start/vm_end cannot change under us because the caller
2192 * is required to hold the mmap_sem in read mode. We need the
2193 * anon_vma lock to serialize against concurrent expand_stacks.
2194 */
2197 /* Somebody else might have raced and expanded it already */
2208 /*
2209 * vma_gap_update() doesn't support concurrent
2210 * updates, but we only hold a shared mmap_sem
2211 * lock here, so we need to protect against
2212 * concurrent vma expansions.
2213 * anon_vma_lock_write() doesn't help here, as
2214 * we don't guarantee that all growable vmas
2215 * in a mm share the same root anon vma.
2216 * So, we reuse mm->page_table_lock to guard
2217 * against concurrent vma expansions.
2218 */
2225 else
2230 }
2231 }
2232 }
2237 }
2240 /*
2241 * vma is the first one with address < vma->vm_start. Have to extend vma.
2242 */
2245 {
2255 /* Enforce stack_guard_gap */
2264 /* Check that both stack segments have the same anon_vma? */
2265 }
2267 /* We must make sure the anon_vma is allocated. */
2271 /*
2272 * vma->vm_start/vm_end cannot change under us because the caller
2273 * is required to hold the mmap_sem in read mode. We need the
2274 * anon_vma lock to serialize against concurrent expand_stacks.
2275 */
2278 /* Somebody else might have raced and expanded it already */
2289 /*
2290 * vma_gap_update() doesn't support concurrent
2291 * updates, but we only hold a shared mmap_sem
2292 * lock here, so we need to protect against
2293 * concurrent vma expansions.
2294 * anon_vma_lock_write() doesn't help here, as
2295 * we don't guarantee that all growable vmas
2296 * in a mm share the same root anon vma.
2297 * So, we reuse mm->page_table_lock to guard
2298 * against concurrent vma expansions.
2299 */
2309 }
2310 }
2311 }
2316 }
2318 /* enforced gap between the expanding stack and other mappings. */
2322 {
2331 }
2334 #ifdef CONFIG_STACK_GROWSUP
2336 {
2338 }
2342 {
2354 }
2355 #else
2357 {
2359 }
2363 {
2381 }
2382 #endif
2384 /*
2385 * Ok - we have the memory areas we should free on the vma list,
2386 * so release them, and do the vma updates.
2387 *
2388 * Called with the mm semaphore held.
2389 */
2391 {
2394 /* Update high watermark before we lower total_vm */
2406 }
2408 /*
2409 * Get rid of page table information in the indicated region.
2410 *
2411 * Called with the mm semaphore held.
2412 */
2416 {
2427 }
2429 /*
2430 * Create a list of vma's touched by the unmap, removing them from the mm's
2431 * vma list as we go..
2432 */
2433 static void
2436 {
2456 /* Kill the cache */
2458 }
2460 /*
2461 * __split_vma() bypasses sysctl_max_map_count checking. We use this on the
2462 * munmap path where it doesn't make sense to fail.
2463 */
2466 {
2478 /* most fields are the same, copy all, and then fixup */
2488 }
2507 else
2510 /* Success. */
2514 /* Clean everything up if vma_adjust failed. */
2520 out_free_mpol:
2522 out_free_vma:
2524 out_err:
2526 }
2528 /*
2529 * Split a vma into two pieces at address 'addr', a new vma is allocated
2530 * either for the first part or the tail.
2531 */
2534 {
2539 }
2541 /* Munmap is split into 2 main parts -- this part which finds
2542 * what needs doing, and the areas themselves, which do the
2543 * work. This now handles partial unmappings.
2544 * Jeremy Fitzhardinge <jeremy@goop.org>
2545 */
2547 {
2557 /* Find the first overlapping VMA */
2562 /* we have start < vma->vm_end */
2564 /* if it doesn't overlap, we have nothing.. */
2569 /*
2570 * If we need to split any vma, do it now to save pain later.
2571 *
2572 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
2573 * unmapped vm_area_struct will remain in use: so lower split_vma
2574 * places tmp vma above, and higher split_vma places tmp vma below.
2575 */
2579 /*
2580 * Make sure that map_count on return from munmap() will
2581 * not exceed its limit; but let map_count go just above
2582 * its limit temporarily, to help free resources as expected.
2583 */
2591 }
2593 /* Does it split the last one? */
2599 }
2602 /*
2603 * unlock any mlock()ed ranges before detaching vmas
2604 */
2611 }
2613 }
2614 }
2616 /*
2617 * Remove the vma's, and unmap the actual pages
2618 */
2622 /* Fix up all other VM information */
2626 }
2629 {
2637 }
2641 {
2644 }
2647 /*
2648 * Emulation of deprecated remap_file_pages() syscall.
2649 */
2652 {
2672 /* Does pgoff wrap? */
2689 /* hole between vmas ? */
2701 }
2705 }
2717 /* drop PG_Mlocked flag for over-mapped range */
2723 }
2724 }
2730 out:
2737 }
2740 {
2741 #ifdef CONFIG_DEBUG_VM
2745 }
2746 #endif
2747 }
2749 /*
2750 * this is really a simplified "do_mmap". it only handles
2751 * anonymous maps. eventually we may be able to do some
2752 * brk-specific accounting here.
2753 */
2755 {
2773 /*
2774 * mm->mmap_sem is required to protect against another thread
2775 * changing the mappings in case we sleep.
2776 */
2779 /*
2780 * Clear old maps. this also does some error checking for us
2781 */
2782 munmap_back:
2787 }
2789 /* Check against address space limits *after* clearing old maps... */
2799 /* Can we just expand an old private anonymous mapping? */
2805 /*
2806 * create a vma struct for an anonymous mapping
2807 */
2812 }
2822 out:
2829 }
2832 {
2851 }
2854 /* Release all mmaps. */
2856 {
2861 /* mm's last user has gone, and its about to be pulled down */
2870 }
2871 }
2882 /* update_hiwater_rss(mm) here? but nobody should be looking */
2883 /* Use -1 here to ensure all VMAs in the mm are unmapped */
2889 /*
2890 * Walk the list again, actually closing and freeing it,
2891 * with preemption enabled, without holding any MM locks.
2892 */
2897 }
2902 }
2904 /* Insert vm structure into process list sorted by address
2905 * and into the inode's i_mmap tree. If vm_file is non-NULL
2906 * then i_mmap_mutex is taken here.
2907 */
2909 {
2913 /*
2914 * The vm_pgoff of a purely anonymous vma should be irrelevant
2915 * until its first write fault, when page's anon_vma and index
2916 * are set. But now set the vm_pgoff it will almost certainly
2917 * end up with (unless mremap moves it elsewhere before that
2918 * first wfault), so /proc/pid/maps tells a consistent story.
2919 *
2920 * By setting it to reflect the virtual start address of the
2921 * vma, merges and splits can happen in a seamless way, just
2922 * using the existing file pgoff checks and manipulations.
2923 * Similarly in do_mmap_pgoff and in do_brk.
2924 */
2928 }
2938 }
2940 /*
2941 * Copy the vma structure to a new location in the same mm,
2942 * prior to moving page table entries, to effect an mremap move.
2943 */
2947 {
2955 /*
2956 * If anonymous vma has not yet been faulted, update new pgoff
2957 * to match new location, to increase its chance of merging.
2958 */
2962 }
2969 /*
2970 * Source vma may have been merged into new_vma
2971 */
2974 /*
2975 * The only way we can get a vma_merge with
2976 * self during an mremap is if the vma hasn't
2977 * been faulted in yet and we were allowed to
2978 * reset the dst vma->vm_pgoff to the
2979 * destination address of the mremap to allow
2980 * the merge to happen. mremap must change the
2981 * vm_pgoff linearity between src and dst vmas
2982 * (in turn preventing a vma_merge) to be
2983 * safe. It is only safe to keep the vm_pgoff
2984 * linear if there are no pages mapped yet.
2985 */
2988 }
3008 }
3009 }
3012 out_free_mempol:
3014 out_free_vma:
3017 }
3019 /*
3020 * Return true if the calling process may expand its vm space by the passed
3021 * number of pages
3022 */
3024 {
3033 }
3038 /*
3039 * Having a close hook prevents vma merging regardless of flags.
3040 */
3042 {
3043 }
3046 {
3048 }
3054 };
3059 };
3063 {
3064 pgoff_t pgoff;
3067 /*
3068 * special mappings have no vm_file, and in that case, the mm
3069 * uses vm_pgoff internally. So we have to subtract it from here.
3070 * We are allowed to do this because we are the mm; do not copy
3071 * this code into drivers!
3072 */
3077 else
3079 pages;
3082 pgoff--;
3089 }
3092 }
3099 {
3128 out:
3131 }
3133 /*
3134 * Called with mm->mmap_sem held for writing.
3135 * Insert a new vma covering the given region, with the given flags.
3136 * Its pages are supplied by the given array of struct page *.
3137 * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
3138 * The region past the last page supplied will always produce SIGBUS.
3139 * The array pointer and the pages it points to are assumed to stay alive
3140 * for as long as this mapping might exist.
3141 */
3146 {
3149 }
3154 {
3160 }
3165 {
3167 /*
3168 * The LSB of head.next can't change from under us
3169 * because we hold the mm_all_locks_mutex.
3170 */
3172 /*
3173 * We can safely modify head.next after taking the
3174 * anon_vma->root->rwsem. If some other vma in this mm shares
3175 * the same anon_vma we won't take it again.
3176 *
3177 * No need of atomic instructions here, head.next
3178 * can't change from under us thanks to the
3179 * anon_vma->root->rwsem.
3180 */
3184 }
3185 }
3188 {
3190 /*
3191 * AS_MM_ALL_LOCKS can't change from under us because
3192 * we hold the mm_all_locks_mutex.
3193 *
3194 * Operations on ->flags have to be atomic because
3195 * even if AS_MM_ALL_LOCKS is stable thanks to the
3196 * mm_all_locks_mutex, there may be other cpus
3197 * changing other bitflags in parallel to us.
3198 */
3202 }
3203 }
3205 /*
3206 * This operation locks against the VM for all pte/vma/mm related
3207 * operations that could ever happen on a certain mm. This includes
3208 * vmtruncate, try_to_unmap, and all page faults.
3209 *
3210 * The caller must take the mmap_sem in write mode before calling
3211 * mm_take_all_locks(). The caller isn't allowed to release the
3212 * mmap_sem until mm_drop_all_locks() returns.
3213 *
3214 * mmap_sem in write mode is required in order to block all operations
3215 * that could modify pagetables and free pages without need of
3216 * altering the vma layout. It's also needed in write mode to avoid new
3217 * anon_vmas to be associated with existing vmas.
3218 *
3219 * A single task can't take more than one mm_take_all_locks() in a row
3220 * or it would deadlock.
3221 *
3222 * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
3223 * mapping->flags avoid to take the same lock twice, if more than one
3224 * vma in this mm is backed by the same anon_vma or address_space.
3225 *
3226 * We can take all the locks in random order because the VM code
3227 * taking i_mmap_mutex or anon_vma->rwsem outside the mmap_sem never
3228 * takes more than one of them in a row. Secondly we're protected
3229 * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex.
3230 *
3231 * mm_take_all_locks() and mm_drop_all_locks are expensive operations
3232 * that may have to take thousand of locks.
3233 *
3234 * mm_take_all_locks() can fail if it's interrupted by signals.
3235 */
3237 {
3250 }
3258 }
3262 out_unlock:
3265 }
3268 {
3270 /*
3271 * The LSB of head.next can't change to 0 from under
3272 * us because we hold the mm_all_locks_mutex.
3273 *
3274 * We must however clear the bitflag before unlocking
3275 * the vma so the users using the anon_vma->rb_root will
3276 * never see our bitflag.
3277 *
3278 * No need of atomic instructions here, head.next
3279 * can't change from under us until we release the
3280 * anon_vma->root->rwsem.
3281 */
3286 }
3287 }
3290 {
3292 /*
3293 * AS_MM_ALL_LOCKS can't change to 0 from under us
3294 * because we hold the mm_all_locks_mutex.
3295 */
3300 }
3301 }
3303 /*
3304 * The mmap_sem cannot be released by the caller until
3305 * mm_drop_all_locks() returns.
3306 */
3308 {
3321 }
3324 }
3326 /*
3327 * initialise the VMA slab
3328 */
3330 {
3335 }
3337 /*
3338 * Initialise sysctl_user_reserve_kbytes.
3339 *
3340 * This is intended to prevent a user from starting a single memory hogging
3341 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
3342 * mode.
3343 *
3344 * The default value is min(3% of free memory, 128MB)
3345 * 128MB is enough to recover with sshd/login, bash, and top/kill.
3346 */
3348 {
3355 }
3358 /*
3359 * Initialise sysctl_admin_reserve_kbytes.
3360 *
3361 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
3362 * to log in and kill a memory hogging process.
3363 *
3364 * Systems with more than 256MB will reserve 8MB, enough to recover
3365 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
3366 * only reserve 3% of free pages by default.
3367 */
3369 {
3376 }
3379 /*
3380 * Reinititalise user and admin reserves if memory is added or removed.
3381 *
3382 * The default user reserve max is 128MB, and the default max for the
3383 * admin reserve is 8MB. These are usually, but not always, enough to
3384 * enable recovery from a memory hogging process using login/sshd, a shell,
3385 * and tools like top. It may make sense to increase or even disable the
3386 * reserve depending on the existence of swap or variations in the recovery
3387 * tools. So, the admin may have changed them.
3388 *
3389 * If memory is added and the reserves have been eliminated or increased above
3390 * the default max, then we'll trust the admin.
3391 *
3392 * If memory is removed and there isn't enough free memory, then we
3393 * need to reset the reserves.
3394 *
3395 * Otherwise keep the reserve set by the admin.
3396 */
3399 {
3404 /* Default max is 128MB. Leave alone if modified by operator. */
3409 /* Default max is 8MB. Leave alone if modified by operator. */
3421 sysctl_user_reserve_kbytes);
3422 }
3427 sysctl_admin_reserve_kbytes);
3428 }
3432 }
3434 }
3438 };
3441 {
3446 }