2 * sparse memory mappings.
5 #include <linux/mmzone.h>
6 #include <linux/bootmem.h>
7 #include <linux/highmem.h>
8 #include <linux/module.h>
9 #include <linux/spinlock.h>
10 #include <linux/vmalloc.h>
12 #include <asm/pgalloc.h>
13 #include <asm/pgtable.h>
16 * Permanent SPARSEMEM data:
18 * 1) mem_section - memory sections, mem_map's for valid memory
20 #ifdef CONFIG_SPARSEMEM_EXTREME
21 struct mem_section
*mem_section
[NR_SECTION_ROOTS
]
22 ____cacheline_internodealigned_in_smp
;
24 struct mem_section mem_section
[NR_SECTION_ROOTS
][SECTIONS_PER_ROOT
]
25 ____cacheline_internodealigned_in_smp
;
27 EXPORT_SYMBOL(mem_section
);
29 #ifdef NODE_NOT_IN_PAGE_FLAGS
31 * If we did not store the node number in the page then we have to
32 * do a lookup in the section_to_node_table in order to find which
33 * node the page belongs to.
35 #if MAX_NUMNODES <= 256
36 static u8 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
38 static u16 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
41 int page_to_nid(struct page
*page
)
43 return section_to_node_table
[page_to_section(page
)];
45 EXPORT_SYMBOL(page_to_nid
);
47 static void set_section_nid(unsigned long section_nr
, int nid
)
49 section_to_node_table
[section_nr
] = nid
;
51 #else /* !NODE_NOT_IN_PAGE_FLAGS */
52 static inline void set_section_nid(unsigned long section_nr
, int nid
)
57 #ifdef CONFIG_SPARSEMEM_EXTREME
58 static struct mem_section noinline __init_refok
*sparse_index_alloc(int nid
)
60 struct mem_section
*section
= NULL
;
61 unsigned long array_size
= SECTIONS_PER_ROOT
*
62 sizeof(struct mem_section
);
64 if (slab_is_available())
65 section
= kmalloc_node(array_size
, GFP_KERNEL
, nid
);
67 section
= alloc_bootmem_node(NODE_DATA(nid
), array_size
);
70 memset(section
, 0, array_size
);
75 static int __meminit
sparse_index_init(unsigned long section_nr
, int nid
)
77 static DEFINE_SPINLOCK(index_init_lock
);
78 unsigned long root
= SECTION_NR_TO_ROOT(section_nr
);
79 struct mem_section
*section
;
82 if (mem_section
[root
])
85 section
= sparse_index_alloc(nid
);
87 * This lock keeps two different sections from
88 * reallocating for the same index
90 spin_lock(&index_init_lock
);
92 if (mem_section
[root
]) {
97 mem_section
[root
] = section
;
99 spin_unlock(&index_init_lock
);
102 #else /* !SPARSEMEM_EXTREME */
103 static inline int sparse_index_init(unsigned long section_nr
, int nid
)
110 * Although written for the SPARSEMEM_EXTREME case, this happens
111 * to also work for the flat array case because
112 * NR_SECTION_ROOTS==NR_MEM_SECTIONS.
114 int __section_nr(struct mem_section
* ms
)
116 unsigned long root_nr
;
117 struct mem_section
* root
;
119 for (root_nr
= 0; root_nr
< NR_SECTION_ROOTS
; root_nr
++) {
120 root
= __nr_to_section(root_nr
* SECTIONS_PER_ROOT
);
124 if ((ms
>= root
) && (ms
< (root
+ SECTIONS_PER_ROOT
)))
128 return (root_nr
* SECTIONS_PER_ROOT
) + (ms
- root
);
132 * During early boot, before section_mem_map is used for an actual
133 * mem_map, we use section_mem_map to store the section's NUMA
134 * node. This keeps us from having to use another data structure. The
135 * node information is cleared just before we store the real mem_map.
137 static inline unsigned long sparse_encode_early_nid(int nid
)
139 return (nid
<< SECTION_NID_SHIFT
);
142 static inline int sparse_early_nid(struct mem_section
*section
)
144 return (section
->section_mem_map
>> SECTION_NID_SHIFT
);
147 /* Record a memory area against a node. */
148 void __init
memory_present(int nid
, unsigned long start
, unsigned long end
)
152 start
&= PAGE_SECTION_MASK
;
153 for (pfn
= start
; pfn
< end
; pfn
+= PAGES_PER_SECTION
) {
154 unsigned long section
= pfn_to_section_nr(pfn
);
155 struct mem_section
*ms
;
157 sparse_index_init(section
, nid
);
158 set_section_nid(section
, nid
);
160 ms
= __nr_to_section(section
);
161 if (!ms
->section_mem_map
)
162 ms
->section_mem_map
= sparse_encode_early_nid(nid
) |
163 SECTION_MARKED_PRESENT
;
168 * Only used by the i386 NUMA architecures, but relatively
171 unsigned long __init
node_memmap_size_bytes(int nid
, unsigned long start_pfn
,
172 unsigned long end_pfn
)
175 unsigned long nr_pages
= 0;
177 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
178 if (nid
!= early_pfn_to_nid(pfn
))
181 if (pfn_present(pfn
))
182 nr_pages
+= PAGES_PER_SECTION
;
185 return nr_pages
* sizeof(struct page
);
189 * Subtle, we encode the real pfn into the mem_map such that
190 * the identity pfn - section_mem_map will return the actual
191 * physical page frame number.
193 static unsigned long sparse_encode_mem_map(struct page
*mem_map
, unsigned long pnum
)
195 return (unsigned long)(mem_map
- (section_nr_to_pfn(pnum
)));
199 * We need this if we ever free the mem_maps. While not implemented yet,
200 * this function is included for parity with its sibling.
202 static __attribute((unused
))
203 struct page
*sparse_decode_mem_map(unsigned long coded_mem_map
, unsigned long pnum
)
205 return ((struct page
*)coded_mem_map
) + section_nr_to_pfn(pnum
);
208 static int __meminit
sparse_init_one_section(struct mem_section
*ms
,
209 unsigned long pnum
, struct page
*mem_map
)
211 if (!present_section(ms
))
214 ms
->section_mem_map
&= ~SECTION_MAP_MASK
;
215 ms
->section_mem_map
|= sparse_encode_mem_map(mem_map
, pnum
) |
221 __attribute__((weak
)) __init
222 void *alloc_bootmem_high_node(pg_data_t
*pgdat
, unsigned long size
)
227 #ifndef CONFIG_SPARSEMEM_VMEMMAP
228 struct page __init
*sparse_early_mem_map_populate(unsigned long pnum
, int nid
)
232 map
= alloc_remap(nid
, sizeof(struct page
) * PAGES_PER_SECTION
);
236 map
= alloc_bootmem_high_node(NODE_DATA(nid
),
237 sizeof(struct page
) * PAGES_PER_SECTION
);
241 map
= alloc_bootmem_node(NODE_DATA(nid
),
242 sizeof(struct page
) * PAGES_PER_SECTION
);
245 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
247 struct page __init
*sparse_early_mem_map_alloc(unsigned long pnum
)
250 struct mem_section
*ms
= __nr_to_section(pnum
);
251 int nid
= sparse_early_nid(ms
);
253 map
= sparse_early_mem_map_populate(pnum
, nid
);
257 printk(KERN_ERR
"%s: sparsemem memory map backing failed "
258 "some memory will not be available.\n", __FUNCTION__
);
259 ms
->section_mem_map
= 0;
264 * Allocate the accumulated non-linear sections, allocate a mem_map
265 * for each and record the physical to section mapping.
267 void __init
sparse_init(void)
272 for (pnum
= 0; pnum
< NR_MEM_SECTIONS
; pnum
++) {
273 if (!present_section_nr(pnum
))
276 map
= sparse_early_mem_map_alloc(pnum
);
279 sparse_init_one_section(__nr_to_section(pnum
), pnum
, map
);
283 #ifdef CONFIG_MEMORY_HOTPLUG
284 static struct page
*__kmalloc_section_memmap(unsigned long nr_pages
)
286 struct page
*page
, *ret
;
287 unsigned long memmap_size
= sizeof(struct page
) * nr_pages
;
289 page
= alloc_pages(GFP_KERNEL
|__GFP_NOWARN
, get_order(memmap_size
));
293 ret
= vmalloc(memmap_size
);
299 ret
= (struct page
*)pfn_to_kaddr(page_to_pfn(page
));
301 memset(ret
, 0, memmap_size
);
306 static int vaddr_in_vmalloc_area(void *addr
)
308 if (addr
>= (void *)VMALLOC_START
&&
309 addr
< (void *)VMALLOC_END
)
314 static void __kfree_section_memmap(struct page
*memmap
, unsigned long nr_pages
)
316 if (vaddr_in_vmalloc_area(memmap
))
319 free_pages((unsigned long)memmap
,
320 get_order(sizeof(struct page
) * nr_pages
));
324 * returns the number of sections whose mem_maps were properly
325 * set. If this is <=0, then that means that the passed-in
326 * map was not consumed and must be freed.
328 int sparse_add_one_section(struct zone
*zone
, unsigned long start_pfn
,
331 unsigned long section_nr
= pfn_to_section_nr(start_pfn
);
332 struct pglist_data
*pgdat
= zone
->zone_pgdat
;
333 struct mem_section
*ms
;
339 * no locking for this, because it does its own
340 * plus, it does a kmalloc
342 sparse_index_init(section_nr
, pgdat
->node_id
);
343 memmap
= __kmalloc_section_memmap(nr_pages
);
345 pgdat_resize_lock(pgdat
, &flags
);
347 ms
= __pfn_to_section(start_pfn
);
348 if (ms
->section_mem_map
& SECTION_MARKED_PRESENT
) {
352 ms
->section_mem_map
|= SECTION_MARKED_PRESENT
;
354 ret
= sparse_init_one_section(ms
, section_nr
, memmap
);
357 pgdat_resize_unlock(pgdat
, &flags
);
359 __kfree_section_memmap(memmap
, nr_pages
);