ACPI: video: Use acpi_device's handle instead of driver's
[linux-2.6/verdex.git] / mm / mempolicy.c
blob6b9740bbf4c0199cee2096ad6e6544d4c6b8a255
1 /*
2 * Simple NUMA memory policy for the Linux kernel.
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
5 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
6 * Subject to the GNU Public License, version 2.
8 * NUMA policy allows the user to give hints in which node(s) memory should
9 * be allocated.
11 * Support four policies per VMA and per process:
13 * The VMA policy has priority over the process policy for a page fault.
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
20 * is used.
22 * bind Only allocate memory on a specific set of nodes,
23 * no fallback.
24 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
28 * preferred Try a specific node first before normal fallback.
29 * As a special case node -1 here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
32 * process policy.
34 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
56 /* Notebook:
57 fix mmap readahead to honour policy and enable policy for any page cache
58 object
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
61 first item above.
62 handle mremap for shared memory (currently ignored for the policy)
63 grows down?
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
66 could replace all the switch()es with a mempolicy_ops structure.
69 #include <linux/mempolicy.h>
70 #include <linux/mm.h>
71 #include <linux/highmem.h>
72 #include <linux/hugetlb.h>
73 #include <linux/kernel.h>
74 #include <linux/sched.h>
75 #include <linux/mm.h>
76 #include <linux/nodemask.h>
77 #include <linux/cpuset.h>
78 #include <linux/gfp.h>
79 #include <linux/slab.h>
80 #include <linux/string.h>
81 #include <linux/module.h>
82 #include <linux/interrupt.h>
83 #include <linux/init.h>
84 #include <linux/compat.h>
85 #include <linux/mempolicy.h>
86 #include <linux/swap.h>
87 #include <linux/seq_file.h>
88 #include <linux/proc_fs.h>
89 #include <linux/migrate.h>
90 #include <linux/rmap.h>
91 #include <linux/security.h>
93 #include <asm/tlbflush.h>
94 #include <asm/uaccess.h>
96 /* Internal flags */
97 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
98 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
99 #define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2) /* Gather statistics */
101 static struct kmem_cache *policy_cache;
102 static struct kmem_cache *sn_cache;
104 #define PDprintk(fmt...)
106 /* Highest zone. An specific allocation for a zone below that is not
107 policied. */
108 int policy_zone = ZONE_DMA;
110 struct mempolicy default_policy = {
111 .refcnt = ATOMIC_INIT(1), /* never free it */
112 .policy = MPOL_DEFAULT,
115 /* Do sanity checking on a policy */
116 static int mpol_check_policy(int mode, nodemask_t *nodes)
118 int empty = nodes_empty(*nodes);
120 switch (mode) {
121 case MPOL_DEFAULT:
122 if (!empty)
123 return -EINVAL;
124 break;
125 case MPOL_BIND:
126 case MPOL_INTERLEAVE:
127 /* Preferred will only use the first bit, but allow
128 more for now. */
129 if (empty)
130 return -EINVAL;
131 break;
133 return nodes_subset(*nodes, node_online_map) ? 0 : -EINVAL;
136 /* Generate a custom zonelist for the BIND policy. */
137 static struct zonelist *bind_zonelist(nodemask_t *nodes)
139 struct zonelist *zl;
140 int num, max, nd, k;
142 max = 1 + MAX_NR_ZONES * nodes_weight(*nodes);
143 zl = kmalloc(sizeof(struct zone *) * max, GFP_KERNEL);
144 if (!zl)
145 return NULL;
146 num = 0;
147 /* First put in the highest zones from all nodes, then all the next
148 lower zones etc. Avoid empty zones because the memory allocator
149 doesn't like them. If you implement node hot removal you
150 have to fix that. */
151 for (k = policy_zone; k >= 0; k--) {
152 for_each_node_mask(nd, *nodes) {
153 struct zone *z = &NODE_DATA(nd)->node_zones[k];
154 if (z->present_pages > 0)
155 zl->zones[num++] = z;
158 zl->zones[num] = NULL;
159 return zl;
162 /* Create a new policy */
163 static struct mempolicy *mpol_new(int mode, nodemask_t *nodes)
165 struct mempolicy *policy;
167 PDprintk("setting mode %d nodes[0] %lx\n", mode, nodes_addr(*nodes)[0]);
168 if (mode == MPOL_DEFAULT)
169 return NULL;
170 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
171 if (!policy)
172 return ERR_PTR(-ENOMEM);
173 atomic_set(&policy->refcnt, 1);
174 switch (mode) {
175 case MPOL_INTERLEAVE:
176 policy->v.nodes = *nodes;
177 if (nodes_weight(*nodes) == 0) {
178 kmem_cache_free(policy_cache, policy);
179 return ERR_PTR(-EINVAL);
181 break;
182 case MPOL_PREFERRED:
183 policy->v.preferred_node = first_node(*nodes);
184 if (policy->v.preferred_node >= MAX_NUMNODES)
185 policy->v.preferred_node = -1;
186 break;
187 case MPOL_BIND:
188 policy->v.zonelist = bind_zonelist(nodes);
189 if (policy->v.zonelist == NULL) {
190 kmem_cache_free(policy_cache, policy);
191 return ERR_PTR(-ENOMEM);
193 break;
195 policy->policy = mode;
196 policy->cpuset_mems_allowed = cpuset_mems_allowed(current);
197 return policy;
200 static void gather_stats(struct page *, void *, int pte_dirty);
201 static void migrate_page_add(struct page *page, struct list_head *pagelist,
202 unsigned long flags);
204 /* Scan through pages checking if pages follow certain conditions. */
205 static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
206 unsigned long addr, unsigned long end,
207 const nodemask_t *nodes, unsigned long flags,
208 void *private)
210 pte_t *orig_pte;
211 pte_t *pte;
212 spinlock_t *ptl;
214 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
215 do {
216 struct page *page;
217 unsigned int nid;
219 if (!pte_present(*pte))
220 continue;
221 page = vm_normal_page(vma, addr, *pte);
222 if (!page)
223 continue;
225 * The check for PageReserved here is important to avoid
226 * handling zero pages and other pages that may have been
227 * marked special by the system.
229 * If the PageReserved would not be checked here then f.e.
230 * the location of the zero page could have an influence
231 * on MPOL_MF_STRICT, zero pages would be counted for
232 * the per node stats, and there would be useless attempts
233 * to put zero pages on the migration list.
235 if (PageReserved(page))
236 continue;
237 nid = page_to_nid(page);
238 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
239 continue;
241 if (flags & MPOL_MF_STATS)
242 gather_stats(page, private, pte_dirty(*pte));
243 else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
244 migrate_page_add(page, private, flags);
245 else
246 break;
247 } while (pte++, addr += PAGE_SIZE, addr != end);
248 pte_unmap_unlock(orig_pte, ptl);
249 return addr != end;
252 static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
253 unsigned long addr, unsigned long end,
254 const nodemask_t *nodes, unsigned long flags,
255 void *private)
257 pmd_t *pmd;
258 unsigned long next;
260 pmd = pmd_offset(pud, addr);
261 do {
262 next = pmd_addr_end(addr, end);
263 if (pmd_none_or_clear_bad(pmd))
264 continue;
265 if (check_pte_range(vma, pmd, addr, next, nodes,
266 flags, private))
267 return -EIO;
268 } while (pmd++, addr = next, addr != end);
269 return 0;
272 static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
273 unsigned long addr, unsigned long end,
274 const nodemask_t *nodes, unsigned long flags,
275 void *private)
277 pud_t *pud;
278 unsigned long next;
280 pud = pud_offset(pgd, addr);
281 do {
282 next = pud_addr_end(addr, end);
283 if (pud_none_or_clear_bad(pud))
284 continue;
285 if (check_pmd_range(vma, pud, addr, next, nodes,
286 flags, private))
287 return -EIO;
288 } while (pud++, addr = next, addr != end);
289 return 0;
292 static inline int check_pgd_range(struct vm_area_struct *vma,
293 unsigned long addr, unsigned long end,
294 const nodemask_t *nodes, unsigned long flags,
295 void *private)
297 pgd_t *pgd;
298 unsigned long next;
300 pgd = pgd_offset(vma->vm_mm, addr);
301 do {
302 next = pgd_addr_end(addr, end);
303 if (pgd_none_or_clear_bad(pgd))
304 continue;
305 if (check_pud_range(vma, pgd, addr, next, nodes,
306 flags, private))
307 return -EIO;
308 } while (pgd++, addr = next, addr != end);
309 return 0;
312 /* Check if a vma is migratable */
313 static inline int vma_migratable(struct vm_area_struct *vma)
315 if (vma->vm_flags & (
316 VM_LOCKED|VM_IO|VM_HUGETLB|VM_PFNMAP|VM_RESERVED))
317 return 0;
318 return 1;
322 * Check if all pages in a range are on a set of nodes.
323 * If pagelist != NULL then isolate pages from the LRU and
324 * put them on the pagelist.
326 static struct vm_area_struct *
327 check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
328 const nodemask_t *nodes, unsigned long flags, void *private)
330 int err;
331 struct vm_area_struct *first, *vma, *prev;
333 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
335 err = migrate_prep();
336 if (err)
337 return ERR_PTR(err);
340 first = find_vma(mm, start);
341 if (!first)
342 return ERR_PTR(-EFAULT);
343 prev = NULL;
344 for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) {
345 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
346 if (!vma->vm_next && vma->vm_end < end)
347 return ERR_PTR(-EFAULT);
348 if (prev && prev->vm_end < vma->vm_start)
349 return ERR_PTR(-EFAULT);
351 if (!is_vm_hugetlb_page(vma) &&
352 ((flags & MPOL_MF_STRICT) ||
353 ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
354 vma_migratable(vma)))) {
355 unsigned long endvma = vma->vm_end;
357 if (endvma > end)
358 endvma = end;
359 if (vma->vm_start > start)
360 start = vma->vm_start;
361 err = check_pgd_range(vma, start, endvma, nodes,
362 flags, private);
363 if (err) {
364 first = ERR_PTR(err);
365 break;
368 prev = vma;
370 return first;
373 /* Apply policy to a single VMA */
374 static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
376 int err = 0;
377 struct mempolicy *old = vma->vm_policy;
379 PDprintk("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
380 vma->vm_start, vma->vm_end, vma->vm_pgoff,
381 vma->vm_ops, vma->vm_file,
382 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
384 if (vma->vm_ops && vma->vm_ops->set_policy)
385 err = vma->vm_ops->set_policy(vma, new);
386 if (!err) {
387 mpol_get(new);
388 vma->vm_policy = new;
389 mpol_free(old);
391 return err;
394 /* Step 2: apply policy to a range and do splits. */
395 static int mbind_range(struct vm_area_struct *vma, unsigned long start,
396 unsigned long end, struct mempolicy *new)
398 struct vm_area_struct *next;
399 int err;
401 err = 0;
402 for (; vma && vma->vm_start < end; vma = next) {
403 next = vma->vm_next;
404 if (vma->vm_start < start)
405 err = split_vma(vma->vm_mm, vma, start, 1);
406 if (!err && vma->vm_end > end)
407 err = split_vma(vma->vm_mm, vma, end, 0);
408 if (!err)
409 err = policy_vma(vma, new);
410 if (err)
411 break;
413 return err;
416 static int contextualize_policy(int mode, nodemask_t *nodes)
418 if (!nodes)
419 return 0;
421 cpuset_update_task_memory_state();
422 if (!cpuset_nodes_subset_current_mems_allowed(*nodes))
423 return -EINVAL;
424 return mpol_check_policy(mode, nodes);
429 * Update task->flags PF_MEMPOLICY bit: set iff non-default
430 * mempolicy. Allows more rapid checking of this (combined perhaps
431 * with other PF_* flag bits) on memory allocation hot code paths.
433 * If called from outside this file, the task 'p' should -only- be
434 * a newly forked child not yet visible on the task list, because
435 * manipulating the task flags of a visible task is not safe.
437 * The above limitation is why this routine has the funny name
438 * mpol_fix_fork_child_flag().
440 * It is also safe to call this with a task pointer of current,
441 * which the static wrapper mpol_set_task_struct_flag() does,
442 * for use within this file.
445 void mpol_fix_fork_child_flag(struct task_struct *p)
447 if (p->mempolicy)
448 p->flags |= PF_MEMPOLICY;
449 else
450 p->flags &= ~PF_MEMPOLICY;
453 static void mpol_set_task_struct_flag(void)
455 mpol_fix_fork_child_flag(current);
458 /* Set the process memory policy */
459 long do_set_mempolicy(int mode, nodemask_t *nodes)
461 struct mempolicy *new;
463 if (contextualize_policy(mode, nodes))
464 return -EINVAL;
465 new = mpol_new(mode, nodes);
466 if (IS_ERR(new))
467 return PTR_ERR(new);
468 mpol_free(current->mempolicy);
469 current->mempolicy = new;
470 mpol_set_task_struct_flag();
471 if (new && new->policy == MPOL_INTERLEAVE)
472 current->il_next = first_node(new->v.nodes);
473 return 0;
476 /* Fill a zone bitmap for a policy */
477 static void get_zonemask(struct mempolicy *p, nodemask_t *nodes)
479 int i;
481 nodes_clear(*nodes);
482 switch (p->policy) {
483 case MPOL_BIND:
484 for (i = 0; p->v.zonelist->zones[i]; i++)
485 node_set(p->v.zonelist->zones[i]->zone_pgdat->node_id,
486 *nodes);
487 break;
488 case MPOL_DEFAULT:
489 break;
490 case MPOL_INTERLEAVE:
491 *nodes = p->v.nodes;
492 break;
493 case MPOL_PREFERRED:
494 /* or use current node instead of online map? */
495 if (p->v.preferred_node < 0)
496 *nodes = node_online_map;
497 else
498 node_set(p->v.preferred_node, *nodes);
499 break;
500 default:
501 BUG();
505 static int lookup_node(struct mm_struct *mm, unsigned long addr)
507 struct page *p;
508 int err;
510 err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
511 if (err >= 0) {
512 err = page_to_nid(p);
513 put_page(p);
515 return err;
518 /* Retrieve NUMA policy */
519 long do_get_mempolicy(int *policy, nodemask_t *nmask,
520 unsigned long addr, unsigned long flags)
522 int err;
523 struct mm_struct *mm = current->mm;
524 struct vm_area_struct *vma = NULL;
525 struct mempolicy *pol = current->mempolicy;
527 cpuset_update_task_memory_state();
528 if (flags & ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR))
529 return -EINVAL;
530 if (flags & MPOL_F_ADDR) {
531 down_read(&mm->mmap_sem);
532 vma = find_vma_intersection(mm, addr, addr+1);
533 if (!vma) {
534 up_read(&mm->mmap_sem);
535 return -EFAULT;
537 if (vma->vm_ops && vma->vm_ops->get_policy)
538 pol = vma->vm_ops->get_policy(vma, addr);
539 else
540 pol = vma->vm_policy;
541 } else if (addr)
542 return -EINVAL;
544 if (!pol)
545 pol = &default_policy;
547 if (flags & MPOL_F_NODE) {
548 if (flags & MPOL_F_ADDR) {
549 err = lookup_node(mm, addr);
550 if (err < 0)
551 goto out;
552 *policy = err;
553 } else if (pol == current->mempolicy &&
554 pol->policy == MPOL_INTERLEAVE) {
555 *policy = current->il_next;
556 } else {
557 err = -EINVAL;
558 goto out;
560 } else
561 *policy = pol->policy;
563 if (vma) {
564 up_read(&current->mm->mmap_sem);
565 vma = NULL;
568 err = 0;
569 if (nmask)
570 get_zonemask(pol, nmask);
572 out:
573 if (vma)
574 up_read(&current->mm->mmap_sem);
575 return err;
578 #ifdef CONFIG_MIGRATION
580 * page migration
582 static void migrate_page_add(struct page *page, struct list_head *pagelist,
583 unsigned long flags)
586 * Avoid migrating a page that is shared with others.
588 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1)
589 isolate_lru_page(page, pagelist);
592 static struct page *new_node_page(struct page *page, unsigned long node, int **x)
594 return alloc_pages_node(node, GFP_HIGHUSER, 0);
598 * Migrate pages from one node to a target node.
599 * Returns error or the number of pages not migrated.
601 int migrate_to_node(struct mm_struct *mm, int source, int dest, int flags)
603 nodemask_t nmask;
604 LIST_HEAD(pagelist);
605 int err = 0;
607 nodes_clear(nmask);
608 node_set(source, nmask);
610 check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nmask,
611 flags | MPOL_MF_DISCONTIG_OK, &pagelist);
613 if (!list_empty(&pagelist))
614 err = migrate_pages(&pagelist, new_node_page, dest);
616 return err;
620 * Move pages between the two nodesets so as to preserve the physical
621 * layout as much as possible.
623 * Returns the number of page that could not be moved.
625 int do_migrate_pages(struct mm_struct *mm,
626 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
628 LIST_HEAD(pagelist);
629 int busy = 0;
630 int err = 0;
631 nodemask_t tmp;
633 down_read(&mm->mmap_sem);
635 err = migrate_vmas(mm, from_nodes, to_nodes, flags);
636 if (err)
637 goto out;
640 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
641 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
642 * bit in 'tmp', and return that <source, dest> pair for migration.
643 * The pair of nodemasks 'to' and 'from' define the map.
645 * If no pair of bits is found that way, fallback to picking some
646 * pair of 'source' and 'dest' bits that are not the same. If the
647 * 'source' and 'dest' bits are the same, this represents a node
648 * that will be migrating to itself, so no pages need move.
650 * If no bits are left in 'tmp', or if all remaining bits left
651 * in 'tmp' correspond to the same bit in 'to', return false
652 * (nothing left to migrate).
654 * This lets us pick a pair of nodes to migrate between, such that
655 * if possible the dest node is not already occupied by some other
656 * source node, minimizing the risk of overloading the memory on a
657 * node that would happen if we migrated incoming memory to a node
658 * before migrating outgoing memory source that same node.
660 * A single scan of tmp is sufficient. As we go, we remember the
661 * most recent <s, d> pair that moved (s != d). If we find a pair
662 * that not only moved, but what's better, moved to an empty slot
663 * (d is not set in tmp), then we break out then, with that pair.
664 * Otherwise when we finish scannng from_tmp, we at least have the
665 * most recent <s, d> pair that moved. If we get all the way through
666 * the scan of tmp without finding any node that moved, much less
667 * moved to an empty node, then there is nothing left worth migrating.
670 tmp = *from_nodes;
671 while (!nodes_empty(tmp)) {
672 int s,d;
673 int source = -1;
674 int dest = 0;
676 for_each_node_mask(s, tmp) {
677 d = node_remap(s, *from_nodes, *to_nodes);
678 if (s == d)
679 continue;
681 source = s; /* Node moved. Memorize */
682 dest = d;
684 /* dest not in remaining from nodes? */
685 if (!node_isset(dest, tmp))
686 break;
688 if (source == -1)
689 break;
691 node_clear(source, tmp);
692 err = migrate_to_node(mm, source, dest, flags);
693 if (err > 0)
694 busy += err;
695 if (err < 0)
696 break;
698 out:
699 up_read(&mm->mmap_sem);
700 if (err < 0)
701 return err;
702 return busy;
706 static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
708 struct vm_area_struct *vma = (struct vm_area_struct *)private;
710 return alloc_page_vma(GFP_HIGHUSER, vma, page_address_in_vma(page, vma));
712 #else
714 static void migrate_page_add(struct page *page, struct list_head *pagelist,
715 unsigned long flags)
719 int do_migrate_pages(struct mm_struct *mm,
720 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
722 return -ENOSYS;
725 static struct page *new_vma_page(struct page *page, unsigned long private)
727 return NULL;
729 #endif
731 long do_mbind(unsigned long start, unsigned long len,
732 unsigned long mode, nodemask_t *nmask, unsigned long flags)
734 struct vm_area_struct *vma;
735 struct mm_struct *mm = current->mm;
736 struct mempolicy *new;
737 unsigned long end;
738 int err;
739 LIST_HEAD(pagelist);
741 if ((flags & ~(unsigned long)(MPOL_MF_STRICT |
742 MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
743 || mode > MPOL_MAX)
744 return -EINVAL;
745 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
746 return -EPERM;
748 if (start & ~PAGE_MASK)
749 return -EINVAL;
751 if (mode == MPOL_DEFAULT)
752 flags &= ~MPOL_MF_STRICT;
754 len = (len + PAGE_SIZE - 1) & PAGE_MASK;
755 end = start + len;
757 if (end < start)
758 return -EINVAL;
759 if (end == start)
760 return 0;
762 if (mpol_check_policy(mode, nmask))
763 return -EINVAL;
765 new = mpol_new(mode, nmask);
766 if (IS_ERR(new))
767 return PTR_ERR(new);
770 * If we are using the default policy then operation
771 * on discontinuous address spaces is okay after all
773 if (!new)
774 flags |= MPOL_MF_DISCONTIG_OK;
776 PDprintk("mbind %lx-%lx mode:%ld nodes:%lx\n",start,start+len,
777 mode,nodes_addr(nodes)[0]);
779 down_write(&mm->mmap_sem);
780 vma = check_range(mm, start, end, nmask,
781 flags | MPOL_MF_INVERT, &pagelist);
783 err = PTR_ERR(vma);
784 if (!IS_ERR(vma)) {
785 int nr_failed = 0;
787 err = mbind_range(vma, start, end, new);
789 if (!list_empty(&pagelist))
790 nr_failed = migrate_pages(&pagelist, new_vma_page,
791 (unsigned long)vma);
793 if (!err && nr_failed && (flags & MPOL_MF_STRICT))
794 err = -EIO;
797 up_write(&mm->mmap_sem);
798 mpol_free(new);
799 return err;
803 * User space interface with variable sized bitmaps for nodelists.
806 /* Copy a node mask from user space. */
807 static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
808 unsigned long maxnode)
810 unsigned long k;
811 unsigned long nlongs;
812 unsigned long endmask;
814 --maxnode;
815 nodes_clear(*nodes);
816 if (maxnode == 0 || !nmask)
817 return 0;
818 if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
819 return -EINVAL;
821 nlongs = BITS_TO_LONGS(maxnode);
822 if ((maxnode % BITS_PER_LONG) == 0)
823 endmask = ~0UL;
824 else
825 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
827 /* When the user specified more nodes than supported just check
828 if the non supported part is all zero. */
829 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
830 if (nlongs > PAGE_SIZE/sizeof(long))
831 return -EINVAL;
832 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
833 unsigned long t;
834 if (get_user(t, nmask + k))
835 return -EFAULT;
836 if (k == nlongs - 1) {
837 if (t & endmask)
838 return -EINVAL;
839 } else if (t)
840 return -EINVAL;
842 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
843 endmask = ~0UL;
846 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
847 return -EFAULT;
848 nodes_addr(*nodes)[nlongs-1] &= endmask;
849 return 0;
852 /* Copy a kernel node mask to user space */
853 static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
854 nodemask_t *nodes)
856 unsigned long copy = ALIGN(maxnode-1, 64) / 8;
857 const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
859 if (copy > nbytes) {
860 if (copy > PAGE_SIZE)
861 return -EINVAL;
862 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
863 return -EFAULT;
864 copy = nbytes;
866 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
869 asmlinkage long sys_mbind(unsigned long start, unsigned long len,
870 unsigned long mode,
871 unsigned long __user *nmask, unsigned long maxnode,
872 unsigned flags)
874 nodemask_t nodes;
875 int err;
877 err = get_nodes(&nodes, nmask, maxnode);
878 if (err)
879 return err;
880 return do_mbind(start, len, mode, &nodes, flags);
883 /* Set the process memory policy */
884 asmlinkage long sys_set_mempolicy(int mode, unsigned long __user *nmask,
885 unsigned long maxnode)
887 int err;
888 nodemask_t nodes;
890 if (mode < 0 || mode > MPOL_MAX)
891 return -EINVAL;
892 err = get_nodes(&nodes, nmask, maxnode);
893 if (err)
894 return err;
895 return do_set_mempolicy(mode, &nodes);
898 asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
899 const unsigned long __user *old_nodes,
900 const unsigned long __user *new_nodes)
902 struct mm_struct *mm;
903 struct task_struct *task;
904 nodemask_t old;
905 nodemask_t new;
906 nodemask_t task_nodes;
907 int err;
909 err = get_nodes(&old, old_nodes, maxnode);
910 if (err)
911 return err;
913 err = get_nodes(&new, new_nodes, maxnode);
914 if (err)
915 return err;
917 /* Find the mm_struct */
918 read_lock(&tasklist_lock);
919 task = pid ? find_task_by_pid(pid) : current;
920 if (!task) {
921 read_unlock(&tasklist_lock);
922 return -ESRCH;
924 mm = get_task_mm(task);
925 read_unlock(&tasklist_lock);
927 if (!mm)
928 return -EINVAL;
931 * Check if this process has the right to modify the specified
932 * process. The right exists if the process has administrative
933 * capabilities, superuser privileges or the same
934 * userid as the target process.
936 if ((current->euid != task->suid) && (current->euid != task->uid) &&
937 (current->uid != task->suid) && (current->uid != task->uid) &&
938 !capable(CAP_SYS_NICE)) {
939 err = -EPERM;
940 goto out;
943 task_nodes = cpuset_mems_allowed(task);
944 /* Is the user allowed to access the target nodes? */
945 if (!nodes_subset(new, task_nodes) && !capable(CAP_SYS_NICE)) {
946 err = -EPERM;
947 goto out;
950 err = security_task_movememory(task);
951 if (err)
952 goto out;
954 err = do_migrate_pages(mm, &old, &new,
955 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
956 out:
957 mmput(mm);
958 return err;
962 /* Retrieve NUMA policy */
963 asmlinkage long sys_get_mempolicy(int __user *policy,
964 unsigned long __user *nmask,
965 unsigned long maxnode,
966 unsigned long addr, unsigned long flags)
968 int err, pval;
969 nodemask_t nodes;
971 if (nmask != NULL && maxnode < MAX_NUMNODES)
972 return -EINVAL;
974 err = do_get_mempolicy(&pval, &nodes, addr, flags);
976 if (err)
977 return err;
979 if (policy && put_user(pval, policy))
980 return -EFAULT;
982 if (nmask)
983 err = copy_nodes_to_user(nmask, maxnode, &nodes);
985 return err;
988 #ifdef CONFIG_COMPAT
990 asmlinkage long compat_sys_get_mempolicy(int __user *policy,
991 compat_ulong_t __user *nmask,
992 compat_ulong_t maxnode,
993 compat_ulong_t addr, compat_ulong_t flags)
995 long err;
996 unsigned long __user *nm = NULL;
997 unsigned long nr_bits, alloc_size;
998 DECLARE_BITMAP(bm, MAX_NUMNODES);
1000 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1001 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1003 if (nmask)
1004 nm = compat_alloc_user_space(alloc_size);
1006 err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
1008 if (!err && nmask) {
1009 err = copy_from_user(bm, nm, alloc_size);
1010 /* ensure entire bitmap is zeroed */
1011 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1012 err |= compat_put_bitmap(nmask, bm, nr_bits);
1015 return err;
1018 asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask,
1019 compat_ulong_t maxnode)
1021 long err = 0;
1022 unsigned long __user *nm = NULL;
1023 unsigned long nr_bits, alloc_size;
1024 DECLARE_BITMAP(bm, MAX_NUMNODES);
1026 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1027 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1029 if (nmask) {
1030 err = compat_get_bitmap(bm, nmask, nr_bits);
1031 nm = compat_alloc_user_space(alloc_size);
1032 err |= copy_to_user(nm, bm, alloc_size);
1035 if (err)
1036 return -EFAULT;
1038 return sys_set_mempolicy(mode, nm, nr_bits+1);
1041 asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
1042 compat_ulong_t mode, compat_ulong_t __user *nmask,
1043 compat_ulong_t maxnode, compat_ulong_t flags)
1045 long err = 0;
1046 unsigned long __user *nm = NULL;
1047 unsigned long nr_bits, alloc_size;
1048 nodemask_t bm;
1050 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1051 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1053 if (nmask) {
1054 err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
1055 nm = compat_alloc_user_space(alloc_size);
1056 err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
1059 if (err)
1060 return -EFAULT;
1062 return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
1065 #endif
1067 /* Return effective policy for a VMA */
1068 static struct mempolicy * get_vma_policy(struct task_struct *task,
1069 struct vm_area_struct *vma, unsigned long addr)
1071 struct mempolicy *pol = task->mempolicy;
1073 if (vma) {
1074 if (vma->vm_ops && vma->vm_ops->get_policy)
1075 pol = vma->vm_ops->get_policy(vma, addr);
1076 else if (vma->vm_policy &&
1077 vma->vm_policy->policy != MPOL_DEFAULT)
1078 pol = vma->vm_policy;
1080 if (!pol)
1081 pol = &default_policy;
1082 return pol;
1085 /* Return a zonelist representing a mempolicy */
1086 static struct zonelist *zonelist_policy(gfp_t gfp, struct mempolicy *policy)
1088 int nd;
1090 switch (policy->policy) {
1091 case MPOL_PREFERRED:
1092 nd = policy->v.preferred_node;
1093 if (nd < 0)
1094 nd = numa_node_id();
1095 break;
1096 case MPOL_BIND:
1097 /* Lower zones don't get a policy applied */
1098 /* Careful: current->mems_allowed might have moved */
1099 if (gfp_zone(gfp) >= policy_zone)
1100 if (cpuset_zonelist_valid_mems_allowed(policy->v.zonelist))
1101 return policy->v.zonelist;
1102 /*FALL THROUGH*/
1103 case MPOL_INTERLEAVE: /* should not happen */
1104 case MPOL_DEFAULT:
1105 nd = numa_node_id();
1106 break;
1107 default:
1108 nd = 0;
1109 BUG();
1111 return NODE_DATA(nd)->node_zonelists + gfp_zone(gfp);
1114 /* Do dynamic interleaving for a process */
1115 static unsigned interleave_nodes(struct mempolicy *policy)
1117 unsigned nid, next;
1118 struct task_struct *me = current;
1120 nid = me->il_next;
1121 next = next_node(nid, policy->v.nodes);
1122 if (next >= MAX_NUMNODES)
1123 next = first_node(policy->v.nodes);
1124 me->il_next = next;
1125 return nid;
1129 * Depending on the memory policy provide a node from which to allocate the
1130 * next slab entry.
1132 unsigned slab_node(struct mempolicy *policy)
1134 switch (policy->policy) {
1135 case MPOL_INTERLEAVE:
1136 return interleave_nodes(policy);
1138 case MPOL_BIND:
1140 * Follow bind policy behavior and start allocation at the
1141 * first node.
1143 return policy->v.zonelist->zones[0]->zone_pgdat->node_id;
1145 case MPOL_PREFERRED:
1146 if (policy->v.preferred_node >= 0)
1147 return policy->v.preferred_node;
1148 /* Fall through */
1150 default:
1151 return numa_node_id();
1155 /* Do static interleaving for a VMA with known offset. */
1156 static unsigned offset_il_node(struct mempolicy *pol,
1157 struct vm_area_struct *vma, unsigned long off)
1159 unsigned nnodes = nodes_weight(pol->v.nodes);
1160 unsigned target = (unsigned)off % nnodes;
1161 int c;
1162 int nid = -1;
1164 c = 0;
1165 do {
1166 nid = next_node(nid, pol->v.nodes);
1167 c++;
1168 } while (c <= target);
1169 return nid;
1172 /* Determine a node number for interleave */
1173 static inline unsigned interleave_nid(struct mempolicy *pol,
1174 struct vm_area_struct *vma, unsigned long addr, int shift)
1176 if (vma) {
1177 unsigned long off;
1179 off = vma->vm_pgoff;
1180 off += (addr - vma->vm_start) >> shift;
1181 return offset_il_node(pol, vma, off);
1182 } else
1183 return interleave_nodes(pol);
1186 #ifdef CONFIG_HUGETLBFS
1187 /* Return a zonelist suitable for a huge page allocation. */
1188 struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr)
1190 struct mempolicy *pol = get_vma_policy(current, vma, addr);
1192 if (pol->policy == MPOL_INTERLEAVE) {
1193 unsigned nid;
1195 nid = interleave_nid(pol, vma, addr, HPAGE_SHIFT);
1196 return NODE_DATA(nid)->node_zonelists + gfp_zone(GFP_HIGHUSER);
1198 return zonelist_policy(GFP_HIGHUSER, pol);
1200 #endif
1202 /* Allocate a page in interleaved policy.
1203 Own path because it needs to do special accounting. */
1204 static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1205 unsigned nid)
1207 struct zonelist *zl;
1208 struct page *page;
1210 zl = NODE_DATA(nid)->node_zonelists + gfp_zone(gfp);
1211 page = __alloc_pages(gfp, order, zl);
1212 if (page && page_zone(page) == zl->zones[0]) {
1213 zone_pcp(zl->zones[0],get_cpu())->interleave_hit++;
1214 put_cpu();
1216 return page;
1220 * alloc_page_vma - Allocate a page for a VMA.
1222 * @gfp:
1223 * %GFP_USER user allocation.
1224 * %GFP_KERNEL kernel allocations,
1225 * %GFP_HIGHMEM highmem/user allocations,
1226 * %GFP_FS allocation should not call back into a file system.
1227 * %GFP_ATOMIC don't sleep.
1229 * @vma: Pointer to VMA or NULL if not available.
1230 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1232 * This function allocates a page from the kernel page pool and applies
1233 * a NUMA policy associated with the VMA or the current process.
1234 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1235 * mm_struct of the VMA to prevent it from going away. Should be used for
1236 * all allocations for pages that will be mapped into
1237 * user space. Returns NULL when no page can be allocated.
1239 * Should be called with the mm_sem of the vma hold.
1241 struct page *
1242 alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
1244 struct mempolicy *pol = get_vma_policy(current, vma, addr);
1246 cpuset_update_task_memory_state();
1248 if (unlikely(pol->policy == MPOL_INTERLEAVE)) {
1249 unsigned nid;
1251 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);
1252 return alloc_page_interleave(gfp, 0, nid);
1254 return __alloc_pages(gfp, 0, zonelist_policy(gfp, pol));
1258 * alloc_pages_current - Allocate pages.
1260 * @gfp:
1261 * %GFP_USER user allocation,
1262 * %GFP_KERNEL kernel allocation,
1263 * %GFP_HIGHMEM highmem allocation,
1264 * %GFP_FS don't call back into a file system.
1265 * %GFP_ATOMIC don't sleep.
1266 * @order: Power of two of allocation size in pages. 0 is a single page.
1268 * Allocate a page from the kernel page pool. When not in
1269 * interrupt context and apply the current process NUMA policy.
1270 * Returns NULL when no page can be allocated.
1272 * Don't call cpuset_update_task_memory_state() unless
1273 * 1) it's ok to take cpuset_sem (can WAIT), and
1274 * 2) allocating for current task (not interrupt).
1276 struct page *alloc_pages_current(gfp_t gfp, unsigned order)
1278 struct mempolicy *pol = current->mempolicy;
1280 if ((gfp & __GFP_WAIT) && !in_interrupt())
1281 cpuset_update_task_memory_state();
1282 if (!pol || in_interrupt())
1283 pol = &default_policy;
1284 if (pol->policy == MPOL_INTERLEAVE)
1285 return alloc_page_interleave(gfp, order, interleave_nodes(pol));
1286 return __alloc_pages(gfp, order, zonelist_policy(gfp, pol));
1288 EXPORT_SYMBOL(alloc_pages_current);
1291 * If mpol_copy() sees current->cpuset == cpuset_being_rebound, then it
1292 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
1293 * with the mems_allowed returned by cpuset_mems_allowed(). This
1294 * keeps mempolicies cpuset relative after its cpuset moves. See
1295 * further kernel/cpuset.c update_nodemask().
1297 void *cpuset_being_rebound;
1299 /* Slow path of a mempolicy copy */
1300 struct mempolicy *__mpol_copy(struct mempolicy *old)
1302 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
1304 if (!new)
1305 return ERR_PTR(-ENOMEM);
1306 if (current_cpuset_is_being_rebound()) {
1307 nodemask_t mems = cpuset_mems_allowed(current);
1308 mpol_rebind_policy(old, &mems);
1310 *new = *old;
1311 atomic_set(&new->refcnt, 1);
1312 if (new->policy == MPOL_BIND) {
1313 int sz = ksize(old->v.zonelist);
1314 new->v.zonelist = kmalloc(sz, SLAB_KERNEL);
1315 if (!new->v.zonelist) {
1316 kmem_cache_free(policy_cache, new);
1317 return ERR_PTR(-ENOMEM);
1319 memcpy(new->v.zonelist, old->v.zonelist, sz);
1321 return new;
1324 /* Slow path of a mempolicy comparison */
1325 int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
1327 if (!a || !b)
1328 return 0;
1329 if (a->policy != b->policy)
1330 return 0;
1331 switch (a->policy) {
1332 case MPOL_DEFAULT:
1333 return 1;
1334 case MPOL_INTERLEAVE:
1335 return nodes_equal(a->v.nodes, b->v.nodes);
1336 case MPOL_PREFERRED:
1337 return a->v.preferred_node == b->v.preferred_node;
1338 case MPOL_BIND: {
1339 int i;
1340 for (i = 0; a->v.zonelist->zones[i]; i++)
1341 if (a->v.zonelist->zones[i] != b->v.zonelist->zones[i])
1342 return 0;
1343 return b->v.zonelist->zones[i] == NULL;
1345 default:
1346 BUG();
1347 return 0;
1351 /* Slow path of a mpol destructor. */
1352 void __mpol_free(struct mempolicy *p)
1354 if (!atomic_dec_and_test(&p->refcnt))
1355 return;
1356 if (p->policy == MPOL_BIND)
1357 kfree(p->v.zonelist);
1358 p->policy = MPOL_DEFAULT;
1359 kmem_cache_free(policy_cache, p);
1363 * Shared memory backing store policy support.
1365 * Remember policies even when nobody has shared memory mapped.
1366 * The policies are kept in Red-Black tree linked from the inode.
1367 * They are protected by the sp->lock spinlock, which should be held
1368 * for any accesses to the tree.
1371 /* lookup first element intersecting start-end */
1372 /* Caller holds sp->lock */
1373 static struct sp_node *
1374 sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
1376 struct rb_node *n = sp->root.rb_node;
1378 while (n) {
1379 struct sp_node *p = rb_entry(n, struct sp_node, nd);
1381 if (start >= p->end)
1382 n = n->rb_right;
1383 else if (end <= p->start)
1384 n = n->rb_left;
1385 else
1386 break;
1388 if (!n)
1389 return NULL;
1390 for (;;) {
1391 struct sp_node *w = NULL;
1392 struct rb_node *prev = rb_prev(n);
1393 if (!prev)
1394 break;
1395 w = rb_entry(prev, struct sp_node, nd);
1396 if (w->end <= start)
1397 break;
1398 n = prev;
1400 return rb_entry(n, struct sp_node, nd);
1403 /* Insert a new shared policy into the list. */
1404 /* Caller holds sp->lock */
1405 static void sp_insert(struct shared_policy *sp, struct sp_node *new)
1407 struct rb_node **p = &sp->root.rb_node;
1408 struct rb_node *parent = NULL;
1409 struct sp_node *nd;
1411 while (*p) {
1412 parent = *p;
1413 nd = rb_entry(parent, struct sp_node, nd);
1414 if (new->start < nd->start)
1415 p = &(*p)->rb_left;
1416 else if (new->end > nd->end)
1417 p = &(*p)->rb_right;
1418 else
1419 BUG();
1421 rb_link_node(&new->nd, parent, p);
1422 rb_insert_color(&new->nd, &sp->root);
1423 PDprintk("inserting %lx-%lx: %d\n", new->start, new->end,
1424 new->policy ? new->policy->policy : 0);
1427 /* Find shared policy intersecting idx */
1428 struct mempolicy *
1429 mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
1431 struct mempolicy *pol = NULL;
1432 struct sp_node *sn;
1434 if (!sp->root.rb_node)
1435 return NULL;
1436 spin_lock(&sp->lock);
1437 sn = sp_lookup(sp, idx, idx+1);
1438 if (sn) {
1439 mpol_get(sn->policy);
1440 pol = sn->policy;
1442 spin_unlock(&sp->lock);
1443 return pol;
1446 static void sp_delete(struct shared_policy *sp, struct sp_node *n)
1448 PDprintk("deleting %lx-l%x\n", n->start, n->end);
1449 rb_erase(&n->nd, &sp->root);
1450 mpol_free(n->policy);
1451 kmem_cache_free(sn_cache, n);
1454 struct sp_node *
1455 sp_alloc(unsigned long start, unsigned long end, struct mempolicy *pol)
1457 struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
1459 if (!n)
1460 return NULL;
1461 n->start = start;
1462 n->end = end;
1463 mpol_get(pol);
1464 n->policy = pol;
1465 return n;
1468 /* Replace a policy range. */
1469 static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
1470 unsigned long end, struct sp_node *new)
1472 struct sp_node *n, *new2 = NULL;
1474 restart:
1475 spin_lock(&sp->lock);
1476 n = sp_lookup(sp, start, end);
1477 /* Take care of old policies in the same range. */
1478 while (n && n->start < end) {
1479 struct rb_node *next = rb_next(&n->nd);
1480 if (n->start >= start) {
1481 if (n->end <= end)
1482 sp_delete(sp, n);
1483 else
1484 n->start = end;
1485 } else {
1486 /* Old policy spanning whole new range. */
1487 if (n->end > end) {
1488 if (!new2) {
1489 spin_unlock(&sp->lock);
1490 new2 = sp_alloc(end, n->end, n->policy);
1491 if (!new2)
1492 return -ENOMEM;
1493 goto restart;
1495 n->end = start;
1496 sp_insert(sp, new2);
1497 new2 = NULL;
1498 break;
1499 } else
1500 n->end = start;
1502 if (!next)
1503 break;
1504 n = rb_entry(next, struct sp_node, nd);
1506 if (new)
1507 sp_insert(sp, new);
1508 spin_unlock(&sp->lock);
1509 if (new2) {
1510 mpol_free(new2->policy);
1511 kmem_cache_free(sn_cache, new2);
1513 return 0;
1516 void mpol_shared_policy_init(struct shared_policy *info, int policy,
1517 nodemask_t *policy_nodes)
1519 info->root = RB_ROOT;
1520 spin_lock_init(&info->lock);
1522 if (policy != MPOL_DEFAULT) {
1523 struct mempolicy *newpol;
1525 /* Falls back to MPOL_DEFAULT on any error */
1526 newpol = mpol_new(policy, policy_nodes);
1527 if (!IS_ERR(newpol)) {
1528 /* Create pseudo-vma that contains just the policy */
1529 struct vm_area_struct pvma;
1531 memset(&pvma, 0, sizeof(struct vm_area_struct));
1532 /* Policy covers entire file */
1533 pvma.vm_end = TASK_SIZE;
1534 mpol_set_shared_policy(info, &pvma, newpol);
1535 mpol_free(newpol);
1540 int mpol_set_shared_policy(struct shared_policy *info,
1541 struct vm_area_struct *vma, struct mempolicy *npol)
1543 int err;
1544 struct sp_node *new = NULL;
1545 unsigned long sz = vma_pages(vma);
1547 PDprintk("set_shared_policy %lx sz %lu %d %lx\n",
1548 vma->vm_pgoff,
1549 sz, npol? npol->policy : -1,
1550 npol ? nodes_addr(npol->v.nodes)[0] : -1);
1552 if (npol) {
1553 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
1554 if (!new)
1555 return -ENOMEM;
1557 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
1558 if (err && new)
1559 kmem_cache_free(sn_cache, new);
1560 return err;
1563 /* Free a backing policy store on inode delete. */
1564 void mpol_free_shared_policy(struct shared_policy *p)
1566 struct sp_node *n;
1567 struct rb_node *next;
1569 if (!p->root.rb_node)
1570 return;
1571 spin_lock(&p->lock);
1572 next = rb_first(&p->root);
1573 while (next) {
1574 n = rb_entry(next, struct sp_node, nd);
1575 next = rb_next(&n->nd);
1576 rb_erase(&n->nd, &p->root);
1577 mpol_free(n->policy);
1578 kmem_cache_free(sn_cache, n);
1580 spin_unlock(&p->lock);
1583 /* assumes fs == KERNEL_DS */
1584 void __init numa_policy_init(void)
1586 policy_cache = kmem_cache_create("numa_policy",
1587 sizeof(struct mempolicy),
1588 0, SLAB_PANIC, NULL, NULL);
1590 sn_cache = kmem_cache_create("shared_policy_node",
1591 sizeof(struct sp_node),
1592 0, SLAB_PANIC, NULL, NULL);
1594 /* Set interleaving policy for system init. This way not all
1595 the data structures allocated at system boot end up in node zero. */
1597 if (do_set_mempolicy(MPOL_INTERLEAVE, &node_online_map))
1598 printk("numa_policy_init: interleaving failed\n");
1601 /* Reset policy of current process to default */
1602 void numa_default_policy(void)
1604 do_set_mempolicy(MPOL_DEFAULT, NULL);
1607 /* Migrate a policy to a different set of nodes */
1608 void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
1610 nodemask_t *mpolmask;
1611 nodemask_t tmp;
1613 if (!pol)
1614 return;
1615 mpolmask = &pol->cpuset_mems_allowed;
1616 if (nodes_equal(*mpolmask, *newmask))
1617 return;
1619 switch (pol->policy) {
1620 case MPOL_DEFAULT:
1621 break;
1622 case MPOL_INTERLEAVE:
1623 nodes_remap(tmp, pol->v.nodes, *mpolmask, *newmask);
1624 pol->v.nodes = tmp;
1625 *mpolmask = *newmask;
1626 current->il_next = node_remap(current->il_next,
1627 *mpolmask, *newmask);
1628 break;
1629 case MPOL_PREFERRED:
1630 pol->v.preferred_node = node_remap(pol->v.preferred_node,
1631 *mpolmask, *newmask);
1632 *mpolmask = *newmask;
1633 break;
1634 case MPOL_BIND: {
1635 nodemask_t nodes;
1636 struct zone **z;
1637 struct zonelist *zonelist;
1639 nodes_clear(nodes);
1640 for (z = pol->v.zonelist->zones; *z; z++)
1641 node_set((*z)->zone_pgdat->node_id, nodes);
1642 nodes_remap(tmp, nodes, *mpolmask, *newmask);
1643 nodes = tmp;
1645 zonelist = bind_zonelist(&nodes);
1647 /* If no mem, then zonelist is NULL and we keep old zonelist.
1648 * If that old zonelist has no remaining mems_allowed nodes,
1649 * then zonelist_policy() will "FALL THROUGH" to MPOL_DEFAULT.
1652 if (zonelist) {
1653 /* Good - got mem - substitute new zonelist */
1654 kfree(pol->v.zonelist);
1655 pol->v.zonelist = zonelist;
1657 *mpolmask = *newmask;
1658 break;
1660 default:
1661 BUG();
1662 break;
1667 * Wrapper for mpol_rebind_policy() that just requires task
1668 * pointer, and updates task mempolicy.
1671 void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
1673 mpol_rebind_policy(tsk->mempolicy, new);
1677 * Rebind each vma in mm to new nodemask.
1679 * Call holding a reference to mm. Takes mm->mmap_sem during call.
1682 void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
1684 struct vm_area_struct *vma;
1686 down_write(&mm->mmap_sem);
1687 for (vma = mm->mmap; vma; vma = vma->vm_next)
1688 mpol_rebind_policy(vma->vm_policy, new);
1689 up_write(&mm->mmap_sem);
1693 * Display pages allocated per node and memory policy via /proc.
1696 static const char *policy_types[] = { "default", "prefer", "bind",
1697 "interleave" };
1700 * Convert a mempolicy into a string.
1701 * Returns the number of characters in buffer (if positive)
1702 * or an error (negative)
1704 static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
1706 char *p = buffer;
1707 int l;
1708 nodemask_t nodes;
1709 int mode = pol ? pol->policy : MPOL_DEFAULT;
1711 switch (mode) {
1712 case MPOL_DEFAULT:
1713 nodes_clear(nodes);
1714 break;
1716 case MPOL_PREFERRED:
1717 nodes_clear(nodes);
1718 node_set(pol->v.preferred_node, nodes);
1719 break;
1721 case MPOL_BIND:
1722 get_zonemask(pol, &nodes);
1723 break;
1725 case MPOL_INTERLEAVE:
1726 nodes = pol->v.nodes;
1727 break;
1729 default:
1730 BUG();
1731 return -EFAULT;
1734 l = strlen(policy_types[mode]);
1735 if (buffer + maxlen < p + l + 1)
1736 return -ENOSPC;
1738 strcpy(p, policy_types[mode]);
1739 p += l;
1741 if (!nodes_empty(nodes)) {
1742 if (buffer + maxlen < p + 2)
1743 return -ENOSPC;
1744 *p++ = '=';
1745 p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
1747 return p - buffer;
1750 struct numa_maps {
1751 unsigned long pages;
1752 unsigned long anon;
1753 unsigned long active;
1754 unsigned long writeback;
1755 unsigned long mapcount_max;
1756 unsigned long dirty;
1757 unsigned long swapcache;
1758 unsigned long node[MAX_NUMNODES];
1761 static void gather_stats(struct page *page, void *private, int pte_dirty)
1763 struct numa_maps *md = private;
1764 int count = page_mapcount(page);
1766 md->pages++;
1767 if (pte_dirty || PageDirty(page))
1768 md->dirty++;
1770 if (PageSwapCache(page))
1771 md->swapcache++;
1773 if (PageActive(page))
1774 md->active++;
1776 if (PageWriteback(page))
1777 md->writeback++;
1779 if (PageAnon(page))
1780 md->anon++;
1782 if (count > md->mapcount_max)
1783 md->mapcount_max = count;
1785 md->node[page_to_nid(page)]++;
1788 #ifdef CONFIG_HUGETLB_PAGE
1789 static void check_huge_range(struct vm_area_struct *vma,
1790 unsigned long start, unsigned long end,
1791 struct numa_maps *md)
1793 unsigned long addr;
1794 struct page *page;
1796 for (addr = start; addr < end; addr += HPAGE_SIZE) {
1797 pte_t *ptep = huge_pte_offset(vma->vm_mm, addr & HPAGE_MASK);
1798 pte_t pte;
1800 if (!ptep)
1801 continue;
1803 pte = *ptep;
1804 if (pte_none(pte))
1805 continue;
1807 page = pte_page(pte);
1808 if (!page)
1809 continue;
1811 gather_stats(page, md, pte_dirty(*ptep));
1814 #else
1815 static inline void check_huge_range(struct vm_area_struct *vma,
1816 unsigned long start, unsigned long end,
1817 struct numa_maps *md)
1820 #endif
1822 int show_numa_map(struct seq_file *m, void *v)
1824 struct proc_maps_private *priv = m->private;
1825 struct vm_area_struct *vma = v;
1826 struct numa_maps *md;
1827 struct file *file = vma->vm_file;
1828 struct mm_struct *mm = vma->vm_mm;
1829 int n;
1830 char buffer[50];
1832 if (!mm)
1833 return 0;
1835 md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL);
1836 if (!md)
1837 return 0;
1839 mpol_to_str(buffer, sizeof(buffer),
1840 get_vma_policy(priv->task, vma, vma->vm_start));
1842 seq_printf(m, "%08lx %s", vma->vm_start, buffer);
1844 if (file) {
1845 seq_printf(m, " file=");
1846 seq_path(m, file->f_vfsmnt, file->f_dentry, "\n\t= ");
1847 } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
1848 seq_printf(m, " heap");
1849 } else if (vma->vm_start <= mm->start_stack &&
1850 vma->vm_end >= mm->start_stack) {
1851 seq_printf(m, " stack");
1854 if (is_vm_hugetlb_page(vma)) {
1855 check_huge_range(vma, vma->vm_start, vma->vm_end, md);
1856 seq_printf(m, " huge");
1857 } else {
1858 check_pgd_range(vma, vma->vm_start, vma->vm_end,
1859 &node_online_map, MPOL_MF_STATS, md);
1862 if (!md->pages)
1863 goto out;
1865 if (md->anon)
1866 seq_printf(m," anon=%lu",md->anon);
1868 if (md->dirty)
1869 seq_printf(m," dirty=%lu",md->dirty);
1871 if (md->pages != md->anon && md->pages != md->dirty)
1872 seq_printf(m, " mapped=%lu", md->pages);
1874 if (md->mapcount_max > 1)
1875 seq_printf(m, " mapmax=%lu", md->mapcount_max);
1877 if (md->swapcache)
1878 seq_printf(m," swapcache=%lu", md->swapcache);
1880 if (md->active < md->pages && !is_vm_hugetlb_page(vma))
1881 seq_printf(m," active=%lu", md->active);
1883 if (md->writeback)
1884 seq_printf(m," writeback=%lu", md->writeback);
1886 for_each_online_node(n)
1887 if (md->node[n])
1888 seq_printf(m, " N%d=%lu", n, md->node[n]);
1889 out:
1890 seq_putc(m, '\n');
1891 kfree(md);
1893 if (m->count < m->size)
1894 m->version = (vma != priv->tail_vma) ? vma->vm_start : 0;
1895 return 0;