x86: entry_32.S - use flags from processor-flags.h
[linux-2.6/openmoko-kernel/knife-kernel.git] / arch / x86 / mm / srat_64.c
blob04e06c8226e3f29ce03f02950509a29f3e44663f
1 /*
2 * ACPI 3.0 based NUMA setup
3 * Copyright 2004 Andi Kleen, SuSE Labs.
5 * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
7 * Called from acpi_numa_init while reading the SRAT and SLIT tables.
8 * Assumes all memory regions belonging to a single proximity domain
9 * are in one chunk. Holes between them will be included in the node.
12 #include <linux/kernel.h>
13 #include <linux/acpi.h>
14 #include <linux/mmzone.h>
15 #include <linux/bitmap.h>
16 #include <linux/module.h>
17 #include <linux/topology.h>
18 #include <linux/bootmem.h>
19 #include <linux/mm.h>
20 #include <asm/proto.h>
21 #include <asm/numa.h>
22 #include <asm/e820.h>
24 int acpi_numa __initdata;
26 static struct acpi_table_slit *acpi_slit;
28 static nodemask_t nodes_parsed __initdata;
29 static struct bootnode nodes[MAX_NUMNODES] __initdata;
30 static struct bootnode nodes_add[MAX_NUMNODES];
31 static int found_add_area __initdata;
32 int hotadd_percent __initdata = 0;
34 /* Too small nodes confuse the VM badly. Usually they result
35 from BIOS bugs. */
36 #define NODE_MIN_SIZE (4*1024*1024)
38 static __init int setup_node(int pxm)
40 return acpi_map_pxm_to_node(pxm);
43 static __init int conflicting_nodes(unsigned long start, unsigned long end)
45 int i;
46 for_each_node_mask(i, nodes_parsed) {
47 struct bootnode *nd = &nodes[i];
48 if (nd->start == nd->end)
49 continue;
50 if (nd->end > start && nd->start < end)
51 return i;
52 if (nd->end == end && nd->start == start)
53 return i;
55 return -1;
58 static __init void cutoff_node(int i, unsigned long start, unsigned long end)
60 struct bootnode *nd = &nodes[i];
62 if (found_add_area)
63 return;
65 if (nd->start < start) {
66 nd->start = start;
67 if (nd->end < nd->start)
68 nd->start = nd->end;
70 if (nd->end > end) {
71 nd->end = end;
72 if (nd->start > nd->end)
73 nd->start = nd->end;
77 static __init void bad_srat(void)
79 int i;
80 printk(KERN_ERR "SRAT: SRAT not used.\n");
81 acpi_numa = -1;
82 found_add_area = 0;
83 for (i = 0; i < MAX_LOCAL_APIC; i++)
84 apicid_to_node[i] = NUMA_NO_NODE;
85 for (i = 0; i < MAX_NUMNODES; i++)
86 nodes_add[i].start = nodes[i].end = 0;
87 remove_all_active_ranges();
90 static __init inline int srat_disabled(void)
92 return numa_off || acpi_numa < 0;
96 * A lot of BIOS fill in 10 (= no distance) everywhere. This messes
97 * up the NUMA heuristics which wants the local node to have a smaller
98 * distance than the others.
99 * Do some quick checks here and only use the SLIT if it passes.
101 static __init int slit_valid(struct acpi_table_slit *slit)
103 int i, j;
104 int d = slit->locality_count;
105 for (i = 0; i < d; i++) {
106 for (j = 0; j < d; j++) {
107 u8 val = slit->entry[d*i + j];
108 if (i == j) {
109 if (val != LOCAL_DISTANCE)
110 return 0;
111 } else if (val <= LOCAL_DISTANCE)
112 return 0;
115 return 1;
118 /* Callback for SLIT parsing */
119 void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
121 if (!slit_valid(slit)) {
122 printk(KERN_INFO "ACPI: SLIT table looks invalid. Not used.\n");
123 return;
125 acpi_slit = slit;
128 /* Callback for Proximity Domain -> LAPIC mapping */
129 void __init
130 acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
132 int pxm, node;
133 int apic_id;
135 if (srat_disabled())
136 return;
137 if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
138 bad_srat();
139 return;
141 if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
142 return;
143 pxm = pa->proximity_domain_lo;
144 node = setup_node(pxm);
145 if (node < 0) {
146 printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
147 bad_srat();
148 return;
151 apic_id = pa->apic_id;
152 apicid_to_node[apic_id] = node;
153 acpi_numa = 1;
154 printk(KERN_INFO "SRAT: PXM %u -> APIC %u -> Node %u\n",
155 pxm, apic_id, node);
158 int update_end_of_memory(unsigned long end) {return -1;}
159 static int hotadd_enough_memory(struct bootnode *nd) {return 1;}
160 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
161 static inline int save_add_info(void) {return 1;}
162 #else
163 static inline int save_add_info(void) {return 0;}
164 #endif
166 * Update nodes_add and decide if to include add are in the zone.
167 * Both SPARSE and RESERVE need nodes_add information.
168 * This code supports one contiguous hot add area per node.
170 static int __init
171 reserve_hotadd(int node, unsigned long start, unsigned long end)
173 unsigned long s_pfn = start >> PAGE_SHIFT;
174 unsigned long e_pfn = end >> PAGE_SHIFT;
175 int ret = 0, changed = 0;
176 struct bootnode *nd = &nodes_add[node];
178 /* I had some trouble with strange memory hotadd regions breaking
179 the boot. Be very strict here and reject anything unexpected.
180 If you want working memory hotadd write correct SRATs.
182 The node size check is a basic sanity check to guard against
183 mistakes */
184 if ((signed long)(end - start) < NODE_MIN_SIZE) {
185 printk(KERN_ERR "SRAT: Hotplug area too small\n");
186 return -1;
189 /* This check might be a bit too strict, but I'm keeping it for now. */
190 if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
191 printk(KERN_ERR
192 "SRAT: Hotplug area %lu -> %lu has existing memory\n",
193 s_pfn, e_pfn);
194 return -1;
197 if (!hotadd_enough_memory(&nodes_add[node])) {
198 printk(KERN_ERR "SRAT: Hotplug area too large\n");
199 return -1;
202 /* Looks good */
204 if (nd->start == nd->end) {
205 nd->start = start;
206 nd->end = end;
207 changed = 1;
208 } else {
209 if (nd->start == end) {
210 nd->start = start;
211 changed = 1;
213 if (nd->end == start) {
214 nd->end = end;
215 changed = 1;
217 if (!changed)
218 printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
221 ret = update_end_of_memory(nd->end);
223 if (changed)
224 printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n", nd->start, nd->end);
225 return ret;
228 /* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
229 void __init
230 acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
232 struct bootnode *nd, oldnode;
233 unsigned long start, end;
234 int node, pxm;
235 int i;
237 if (srat_disabled())
238 return;
239 if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
240 bad_srat();
241 return;
243 if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
244 return;
246 if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
247 return;
248 start = ma->base_address;
249 end = start + ma->length;
250 pxm = ma->proximity_domain;
251 node = setup_node(pxm);
252 if (node < 0) {
253 printk(KERN_ERR "SRAT: Too many proximity domains.\n");
254 bad_srat();
255 return;
257 i = conflicting_nodes(start, end);
258 if (i == node) {
259 printk(KERN_WARNING
260 "SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n",
261 pxm, start, end, nodes[i].start, nodes[i].end);
262 } else if (i >= 0) {
263 printk(KERN_ERR
264 "SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n",
265 pxm, start, end, node_to_pxm(i),
266 nodes[i].start, nodes[i].end);
267 bad_srat();
268 return;
270 nd = &nodes[node];
271 oldnode = *nd;
272 if (!node_test_and_set(node, nodes_parsed)) {
273 nd->start = start;
274 nd->end = end;
275 } else {
276 if (start < nd->start)
277 nd->start = start;
278 if (nd->end < end)
279 nd->end = end;
282 printk(KERN_INFO "SRAT: Node %u PXM %u %Lx-%Lx\n", node, pxm,
283 nd->start, nd->end);
284 e820_register_active_regions(node, nd->start >> PAGE_SHIFT,
285 nd->end >> PAGE_SHIFT);
286 push_node_boundaries(node, nd->start >> PAGE_SHIFT,
287 nd->end >> PAGE_SHIFT);
289 if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) &&
290 (reserve_hotadd(node, start, end) < 0)) {
291 /* Ignore hotadd region. Undo damage */
292 printk(KERN_NOTICE "SRAT: Hotplug region ignored\n");
293 *nd = oldnode;
294 if ((nd->start | nd->end) == 0)
295 node_clear(node, nodes_parsed);
299 /* Sanity check to catch more bad SRATs (they are amazingly common).
300 Make sure the PXMs cover all memory. */
301 static int __init nodes_cover_memory(const struct bootnode *nodes)
303 int i;
304 unsigned long pxmram, e820ram;
306 pxmram = 0;
307 for_each_node_mask(i, nodes_parsed) {
308 unsigned long s = nodes[i].start >> PAGE_SHIFT;
309 unsigned long e = nodes[i].end >> PAGE_SHIFT;
310 pxmram += e - s;
311 pxmram -= absent_pages_in_range(s, e);
312 if ((long)pxmram < 0)
313 pxmram = 0;
316 e820ram = end_pfn - absent_pages_in_range(0, end_pfn);
317 /* We seem to lose 3 pages somewhere. Allow a bit of slack. */
318 if ((long)(e820ram - pxmram) >= 1*1024*1024) {
319 printk(KERN_ERR
320 "SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n",
321 (pxmram << PAGE_SHIFT) >> 20,
322 (e820ram << PAGE_SHIFT) >> 20);
323 return 0;
325 return 1;
328 static void __init unparse_node(int node)
330 int i;
331 node_clear(node, nodes_parsed);
332 for (i = 0; i < MAX_LOCAL_APIC; i++) {
333 if (apicid_to_node[i] == node)
334 apicid_to_node[i] = NUMA_NO_NODE;
338 void __init acpi_numa_arch_fixup(void) {}
340 /* Use the information discovered above to actually set up the nodes. */
341 int __init acpi_scan_nodes(unsigned long start, unsigned long end)
343 int i;
345 if (acpi_numa <= 0)
346 return -1;
348 /* First clean up the node list */
349 for (i = 0; i < MAX_NUMNODES; i++) {
350 cutoff_node(i, start, end);
352 * don't confuse VM with a node that doesn't have the
353 * minimum memory.
355 if (nodes[i].end &&
356 (nodes[i].end - nodes[i].start) < NODE_MIN_SIZE) {
357 unparse_node(i);
358 node_set_offline(i);
362 if (!nodes_cover_memory(nodes)) {
363 bad_srat();
364 return -1;
367 memnode_shift = compute_hash_shift(nodes, MAX_NUMNODES);
368 if (memnode_shift < 0) {
369 printk(KERN_ERR
370 "SRAT: No NUMA node hash function found. Contact maintainer\n");
371 bad_srat();
372 return -1;
375 node_possible_map = nodes_parsed;
377 /* Finally register nodes */
378 for_each_node_mask(i, node_possible_map)
379 setup_node_bootmem(i, nodes[i].start, nodes[i].end);
380 /* Try again in case setup_node_bootmem missed one due
381 to missing bootmem */
382 for_each_node_mask(i, node_possible_map)
383 if (!node_online(i))
384 setup_node_bootmem(i, nodes[i].start, nodes[i].end);
386 for (i = 0; i < NR_CPUS; i++) {
387 int node = early_cpu_to_node(i);
389 if (node == NUMA_NO_NODE)
390 continue;
391 if (!node_isset(node, node_possible_map))
392 numa_set_node(i, NUMA_NO_NODE);
394 numa_init_array();
395 return 0;
398 #ifdef CONFIG_NUMA_EMU
399 static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
400 [0 ... MAX_NUMNODES-1] = PXM_INVAL
402 static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
403 [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
405 static int __init find_node_by_addr(unsigned long addr)
407 int ret = NUMA_NO_NODE;
408 int i;
410 for_each_node_mask(i, nodes_parsed) {
412 * Find the real node that this emulated node appears on. For
413 * the sake of simplicity, we only use a real node's starting
414 * address to determine which emulated node it appears on.
416 if (addr >= nodes[i].start && addr < nodes[i].end) {
417 ret = i;
418 break;
421 return ret;
425 * In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
426 * mappings that respect the real ACPI topology but reflect our emulated
427 * environment. For each emulated node, we find which real node it appears on
428 * and create PXM to NID mappings for those fake nodes which mirror that
429 * locality. SLIT will now represent the correct distances between emulated
430 * nodes as a result of the real topology.
432 void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes)
434 int i, j;
436 printk(KERN_INFO "Faking PXM affinity for fake nodes on real "
437 "topology.\n");
438 for (i = 0; i < num_nodes; i++) {
439 int nid, pxm;
441 nid = find_node_by_addr(fake_nodes[i].start);
442 if (nid == NUMA_NO_NODE)
443 continue;
444 pxm = node_to_pxm(nid);
445 if (pxm == PXM_INVAL)
446 continue;
447 fake_node_to_pxm_map[i] = pxm;
449 * For each apicid_to_node mapping that exists for this real
450 * node, it must now point to the fake node ID.
452 for (j = 0; j < MAX_LOCAL_APIC; j++)
453 if (apicid_to_node[j] == nid)
454 fake_apicid_to_node[j] = i;
456 for (i = 0; i < num_nodes; i++)
457 __acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i);
458 memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node));
460 nodes_clear(nodes_parsed);
461 for (i = 0; i < num_nodes; i++)
462 if (fake_nodes[i].start != fake_nodes[i].end)
463 node_set(i, nodes_parsed);
464 WARN_ON(!nodes_cover_memory(fake_nodes));
467 static int null_slit_node_compare(int a, int b)
469 return node_to_pxm(a) == node_to_pxm(b);
471 #else
472 static int null_slit_node_compare(int a, int b)
474 return a == b;
476 #endif /* CONFIG_NUMA_EMU */
478 void __init srat_reserve_add_area(int nodeid)
480 if (found_add_area && nodes_add[nodeid].end) {
481 u64 total_mb;
483 printk(KERN_INFO "SRAT: Reserving hot-add memory space "
484 "for node %d at %Lx-%Lx\n",
485 nodeid, nodes_add[nodeid].start, nodes_add[nodeid].end);
486 total_mb = (nodes_add[nodeid].end - nodes_add[nodeid].start)
487 >> PAGE_SHIFT;
488 total_mb *= sizeof(struct page);
489 total_mb >>= 20;
490 printk(KERN_INFO "SRAT: This will cost you %Lu MB of "
491 "pre-allocated memory.\n", (unsigned long long)total_mb);
492 reserve_bootmem_node(NODE_DATA(nodeid), nodes_add[nodeid].start,
493 nodes_add[nodeid].end - nodes_add[nodeid].start,
494 BOOTMEM_DEFAULT);
498 int __node_distance(int a, int b)
500 int index;
502 if (!acpi_slit)
503 return null_slit_node_compare(a, b) ? LOCAL_DISTANCE :
504 REMOTE_DISTANCE;
505 index = acpi_slit->locality_count * node_to_pxm(a);
506 return acpi_slit->entry[index + node_to_pxm(b)];
509 EXPORT_SYMBOL(__node_distance);
511 int memory_add_physaddr_to_nid(u64 start)
513 int i, ret = 0;
515 for_each_node(i)
516 if (nodes_add[i].start <= start && nodes_add[i].end > start)
517 ret = i;
519 return ret;
521 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);