mm/sparse-vmemmap.c

   1 /*
   2  * Virtual Memory Map support
   3  *
   4  * (C) 2007 sgi. Christoph Lameter <clameter@sgi.com>.
   5  *
   6  * Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn,
   7  * virt_to_page, page_address() to be implemented as a base offset
   8  * calculation without memory access.
   9  *
  10  * However, virtual mappings need a page table and TLBs. Many Linux
  11  * architectures already map their physical space using 1-1 mappings
  12  * via TLBs. For those arches the virtual memmory map is essentially
  13  * for free if we use the same page size as the 1-1 mappings. In that
  14  * case the overhead consists of a few additional pages that are
  15  * allocated to create a view of memory for vmemmap.
  16  *
  17  * The architecture is expected to provide a vmemmap_populate() function
  18  * to instantiate the mapping.
  19  */
  20 #include <linux/mm.h>
  21 #include <linux/mmzone.h>
  22 #include <linux/bootmem.h>
  23 #include <linux/highmem.h>
  24 #include <linux/module.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/vmalloc.h>
  27 #include <asm/dma.h>
  28 #include <asm/pgalloc.h>
  29 #include <asm/pgtable.h>
  30
  31 /*
  32  * Allocate a block of memory to be used to back the virtual memory map
  33  * or to back the page tables that are used to create the mapping.
  34  * Uses the main allocators if they are available, else bootmem.
  35  */
  36 void * __meminit vmemmap_alloc_block(unsigned long size, int node)
  37 {
  38         /* If the main allocator is up use that, fallback to bootmem. */
  39         if (slab_is_available()) {
  40                 struct page *page = alloc_pages_node(node,
  41                                 GFP_KERNEL | __GFP_ZERO, get_order(size));
  42                 if (page)
  43                         return page_address(page);
  44                 return NULL;
  45         } else
  46                 return __alloc_bootmem_node(NODE_DATA(node), size, size,
  47                                 __pa(MAX_DMA_ADDRESS));
  48 }
  49
  50 void __meminit vmemmap_verify(pte_t *pte, int node,
  51                                 unsigned long start, unsigned long end)
  52 {
  53         unsigned long pfn = pte_pfn(*pte);
  54         int actual_node = early_pfn_to_nid(pfn);
  55
  56         if (actual_node != node)
  57                 printk(KERN_WARNING "[%lx-%lx] potential offnode "
  58                         "page_structs\n", start, end - 1);
  59 }
  60
  61 pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node)
  62 {
  63         pte_t *pte = pte_offset_kernel(pmd, addr);
  64         if (pte_none(*pte)) {
  65                 pte_t entry;
  66                 void *p = vmemmap_alloc_block(PAGE_SIZE, node);
  67                 if (!p)
  68                         return 0;
  69                 entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL);
  70                 set_pte_at(&init_mm, addr, pte, entry);
  71         }
  72         return pte;
  73 }
  74
  75 pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
  76 {
  77         pmd_t *pmd = pmd_offset(pud, addr);
  78         if (pmd_none(*pmd)) {
  79                 void *p = vmemmap_alloc_block(PAGE_SIZE, node);
  80                 if (!p)
  81                         return 0;
  82                 pmd_populate_kernel(&init_mm, pmd, p);
  83         }
  84         return pmd;
  85 }
  86
  87 pud_t * __meminit vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node)
  88 {
  89         pud_t *pud = pud_offset(pgd, addr);
  90         if (pud_none(*pud)) {
  91                 void *p = vmemmap_alloc_block(PAGE_SIZE, node);
  92                 if (!p)
  93                         return 0;
  94                 pud_populate(&init_mm, pud, p);
  95         }
  96         return pud;
  97 }
  98
  99 pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
 100 {
 101         pgd_t *pgd = pgd_offset_k(addr);
 102         if (pgd_none(*pgd)) {
 103                 void *p = vmemmap_alloc_block(PAGE_SIZE, node);
 104                 if (!p)
 105                         return 0;
 106                 pgd_populate(&init_mm, pgd, p);
 107         }
 108         return pgd;
 109 }
 110
 111 int __meminit vmemmap_populate_basepages(struct page *start_page,
 112                                                 unsigned long size, int node)
 113 {
 114         unsigned long addr = (unsigned long)start_page;
 115         unsigned long end = (unsigned long)(start_page + size);
 116         pgd_t *pgd;
 117         pud_t *pud;
 118         pmd_t *pmd;
 119         pte_t *pte;
 120
 121         for (; addr < end; addr += PAGE_SIZE) {
 122                 pgd = vmemmap_pgd_populate(addr, node);
 123                 if (!pgd)
 124                         return -ENOMEM;
 125                 pud = vmemmap_pud_populate(pgd, addr, node);
 126                 if (!pud)
 127                         return -ENOMEM;
 128                 pmd = vmemmap_pmd_populate(pud, addr, node);
 129                 if (!pmd)
 130                         return -ENOMEM;
 131                 pte = vmemmap_pte_populate(pmd, addr, node);
 132                 if (!pte)
 133                         return -ENOMEM;
 134                 vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
 135         }
 136
 137         return 0;
 138 }
 139
 140 struct page __init *sparse_early_mem_map_populate(unsigned long pnum, int nid)
 141 {
 142         struct page *map = pfn_to_page(pnum * PAGES_PER_SECTION);
 143         int error = vmemmap_populate(map, PAGES_PER_SECTION, nid);
 144         if (error)
 145                 return NULL;
 146
 147         return map;
 148 }