2 * Copyright (C) 2001-2005 Silicon Graphics, Inc. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
8 * A simple uncached page allocator using the generic allocator. This
9 * allocator first utilizes the spare (spill) pages found in the EFI
10 * memmap and will then start converting cached pages to uncached ones
11 * at a granule at a time. Node awareness is implemented by having a
12 * pool of pages per node.
15 #include <linux/types.h>
16 #include <linux/kernel.h>
17 #include <linux/module.h>
18 #include <linux/init.h>
19 #include <linux/errno.h>
20 #include <linux/string.h>
21 #include <linux/slab.h>
22 #include <linux/efi.h>
23 #include <linux/genalloc.h>
26 #include <asm/system.h>
27 #include <asm/pgtable.h>
28 #include <asm/atomic.h>
29 #include <asm/tlbflush.h>
30 #include <asm/sn/arch.h>
35 #define dprintk printk
37 #define dprintk(x...) do { } while (0)
40 void __init
efi_memmap_walk_uc (efi_freemem_callback_t callback
);
42 #define MAX_UNCACHED_GRANULES 5
43 static int allocated_granules
;
45 struct gen_pool
*uncached_pool
[MAX_NUMNODES
];
48 static void uncached_ipi_visibility(void *data
)
52 status
= ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL
);
53 if ((status
!= PAL_VISIBILITY_OK
) &&
54 (status
!= PAL_VISIBILITY_OK_REMOTE_NEEDED
))
55 printk(KERN_DEBUG
"pal_prefetch_visibility() returns %i on "
56 "CPU %i\n", status
, raw_smp_processor_id());
60 static void uncached_ipi_mc_drain(void *data
)
63 status
= ia64_pal_mc_drain();
65 printk(KERN_WARNING
"ia64_pal_mc_drain() failed with %i on "
66 "CPU %i\n", status
, raw_smp_processor_id());
71 uncached_get_new_chunk(struct gen_pool
*poolp
)
76 unsigned long addr
, node
;
78 if (allocated_granules
>= MAX_UNCACHED_GRANULES
)
81 node
= poolp
->private;
82 page
= alloc_pages_node(node
, GFP_KERNEL
| __GFP_ZERO
,
83 IA64_GRANULE_SHIFT
-PAGE_SHIFT
);
85 dprintk(KERN_INFO
"get_new_chunk page %p, addr %lx\n",
86 page
, (unsigned long)(page
-vmem_map
) << PAGE_SHIFT
);
89 * Do magic if no mem on local node! XXX
93 tmp
= page_address(page
);
96 * There's a small race here where it's possible for someone to
97 * access the page through /dev/mem halfway through the conversion
98 * to uncached - not sure it's really worth bothering about
100 for (i
= 0; i
< (IA64_GRANULE_SIZE
/ PAGE_SIZE
); i
++)
101 SetPageUncached(&page
[i
]);
103 flush_tlb_kernel_range(tmp
, tmp
+ IA64_GRANULE_SIZE
);
105 status
= ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL
);
107 dprintk(KERN_INFO
"pal_prefetch_visibility() returns %i on cpu %i\n",
108 status
, raw_smp_processor_id());
111 status
= smp_call_function(uncached_ipi_visibility
, NULL
, 0, 1);
113 printk(KERN_WARNING
"smp_call_function failed for "
114 "uncached_ipi_visibility! (%i)\n", status
);
117 if (ia64_platform_is("sn2"))
118 sn_flush_all_caches((unsigned long)tmp
, IA64_GRANULE_SIZE
);
120 flush_icache_range((unsigned long)tmp
,
121 (unsigned long)tmp
+IA64_GRANULE_SIZE
);
124 status
= smp_call_function(uncached_ipi_mc_drain
, NULL
, 0, 1);
126 printk(KERN_WARNING
"smp_call_function failed for "
127 "uncached_ipi_mc_drain! (%i)\n", status
);
129 addr
= (unsigned long)tmp
- PAGE_OFFSET
+ __IA64_UNCACHED_OFFSET
;
131 allocated_granules
++;
137 * uncached_alloc_page
139 * Allocate 1 uncached page. Allocates on the requested node. If no
140 * uncached pages are available on the requested node, roundrobin starting
144 uncached_alloc_page(int nid
)
148 maddr
= gen_pool_alloc(uncached_pool
[nid
], PAGE_SIZE
);
150 dprintk(KERN_DEBUG
"uncached_alloc_page returns %lx on node %i\n",
154 * If no memory is availble on our local node, try the
155 * remaining nodes in the system.
160 for (i
= MAX_NUMNODES
- 1; i
>= 0; i
--) {
161 if (i
== nid
|| !node_online(i
))
163 maddr
= gen_pool_alloc(uncached_pool
[i
], PAGE_SIZE
);
164 dprintk(KERN_DEBUG
"uncached_alloc_page alternate search "
165 "returns %lx on node %i\n", maddr
, i
);
174 EXPORT_SYMBOL(uncached_alloc_page
);
180 * Free a single uncached page.
183 uncached_free_page(unsigned long maddr
)
187 node
= paddr_to_nid(maddr
- __IA64_UNCACHED_OFFSET
);
189 dprintk(KERN_DEBUG
"uncached_free_page(%lx) on node %i\n", maddr
, node
);
191 if ((maddr
& (0XFUL
<< 60)) != __IA64_UNCACHED_OFFSET
)
192 panic("uncached_free_page invalid address %lx\n", maddr
);
194 gen_pool_free(uncached_pool
[node
], maddr
, PAGE_SIZE
);
196 EXPORT_SYMBOL(uncached_free_page
);
200 * uncached_build_memmap,
202 * Called at boot time to build a map of pages that can be used for
203 * memory special operations.
206 uncached_build_memmap(unsigned long start
, unsigned long end
, void *arg
)
208 long length
= end
- start
;
211 dprintk(KERN_ERR
"uncached_build_memmap(%lx %lx)\n", start
, end
);
213 touch_softlockup_watchdog();
214 memset((char *)start
, 0, length
);
216 node
= paddr_to_nid(start
- __IA64_UNCACHED_OFFSET
);
218 for (; start
< end
; start
+= PAGE_SIZE
) {
219 dprintk(KERN_INFO
"sticking %lx into the pool!\n", start
);
220 gen_pool_free(uncached_pool
[node
], start
, PAGE_SIZE
);
227 static int __init
uncached_init(void) {
230 for (i
= 0; i
< MAX_NUMNODES
; i
++) {
233 uncached_pool
[i
] = gen_pool_create(0, IA64_GRANULE_SHIFT
,
234 &uncached_get_new_chunk
, i
);
237 efi_memmap_walk_uc(uncached_build_memmap
);
242 __initcall(uncached_init
);