arch/s390/mm/vmem.c

   1 /*
   2  *  arch/s390/mm/vmem.c
   3  *
   4  *    Copyright IBM Corp. 2006
   5  *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
   6  */
   7
   8 #include <linux/bootmem.h>
   9 #include <linux/pfn.h>
  10 #include <linux/mm.h>
  11 #include <linux/module.h>
  12 #include <linux/list.h>
  13 #include <asm/pgalloc.h>
  14 #include <asm/pgtable.h>
  15 #include <asm/setup.h>
  16 #include <asm/tlbflush.h>
  17
  18 unsigned long vmalloc_end;
  19 EXPORT_SYMBOL(vmalloc_end);
  20
  21 static struct page *vmem_map;
  22 static DEFINE_MUTEX(vmem_mutex);
  23
  24 struct memory_segment {
  25         struct list_head list;
  26         unsigned long start;
  27         unsigned long size;
  28 };
  29
  30 static LIST_HEAD(mem_segs);
  31
  32 void memmap_init(unsigned long size, int nid, unsigned long zone,
  33                  unsigned long start_pfn)
  34 {
  35         struct page *start, *end;
  36         struct page *map_start, *map_end;
  37         int i;
  38
  39         start = pfn_to_page(start_pfn);
  40         end = start + size;
  41
  42         for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) {
  43                 unsigned long cstart, cend;
  44
  45                 cstart = PFN_DOWN(memory_chunk[i].addr);
  46                 cend = cstart + PFN_DOWN(memory_chunk[i].size);
  47
  48                 map_start = mem_map + cstart;
  49                 map_end = mem_map + cend;
  50
  51                 if (map_start < start)
  52                         map_start = start;
  53                 if (map_end > end)
  54                         map_end = end;
  55
  56                 map_start -= ((unsigned long) map_start & (PAGE_SIZE - 1))
  57                         / sizeof(struct page);
  58                 map_end += ((PFN_ALIGN((unsigned long) map_end)
  59                              - (unsigned long) map_end)
  60                             / sizeof(struct page));
  61
  62                 if (map_start < map_end)
  63                         memmap_init_zone((unsigned long)(map_end - map_start),
  64                                          nid, zone, page_to_pfn(map_start),
  65                                          MEMMAP_EARLY);
  66         }
  67 }
  68
  69 static inline void *vmem_alloc_pages(unsigned int order)
  70 {
  71         if (slab_is_available())
  72                 return (void *)__get_free_pages(GFP_KERNEL, order);
  73         return alloc_bootmem_pages((1 << order) * PAGE_SIZE);
  74 }
  75
  76 static inline pmd_t *vmem_pmd_alloc(void)
  77 {
  78         pmd_t *pmd;
  79         int i;
  80
  81         pmd = vmem_alloc_pages(PMD_ALLOC_ORDER);
  82         if (!pmd)
  83                 return NULL;
  84         for (i = 0; i < PTRS_PER_PMD; i++)
  85                 pmd_clear_kernel(pmd + i);
  86         return pmd;
  87 }
  88
  89 static inline pte_t *vmem_pte_alloc(void)
  90 {
  91         pte_t *pte;
  92         pte_t empty_pte;
  93         int i;
  94
  95         pte = vmem_alloc_pages(PTE_ALLOC_ORDER);
  96         if (!pte)
  97                 return NULL;
  98         pte_val(empty_pte) = _PAGE_TYPE_EMPTY;
  99         for (i = 0; i < PTRS_PER_PTE; i++)
 100                 pte[i] = empty_pte;
 101         return pte;
 102 }
 103
 104 /*
 105  * Add a physical memory range to the 1:1 mapping.
 106  */
 107 static int vmem_add_range(unsigned long start, unsigned long size)
 108 {
 109         unsigned long address;
 110         pgd_t *pg_dir;
 111         pmd_t *pm_dir;
 112         pte_t *pt_dir;
 113         pte_t  pte;
 114         int ret = -ENOMEM;
 115
 116         for (address = start; address < start + size; address += PAGE_SIZE) {
 117                 pg_dir = pgd_offset_k(address);
 118                 if (pgd_none(*pg_dir)) {
 119                         pm_dir = vmem_pmd_alloc();
 120                         if (!pm_dir)
 121                                 goto out;
 122                         pgd_populate_kernel(&init_mm, pg_dir, pm_dir);
 123                 }
 124
 125                 pm_dir = pmd_offset(pg_dir, address);
 126                 if (pmd_none(*pm_dir)) {
 127                         pt_dir = vmem_pte_alloc();
 128                         if (!pt_dir)
 129                                 goto out;
 130                         pmd_populate_kernel(&init_mm, pm_dir, pt_dir);
 131                 }
 132
 133                 pt_dir = pte_offset_kernel(pm_dir, address);
 134                 pte = pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL);
 135                 *pt_dir = pte;
 136         }
 137         ret = 0;
 138 out:
 139         flush_tlb_kernel_range(start, start + size);
 140         return ret;
 141 }
 142
 143 /*
 144  * Remove a physical memory range from the 1:1 mapping.
 145  * Currently only invalidates page table entries.
 146  */
 147 static void vmem_remove_range(unsigned long start, unsigned long size)
 148 {
 149         unsigned long address;
 150         pgd_t *pg_dir;
 151         pmd_t *pm_dir;
 152         pte_t *pt_dir;
 153         pte_t  pte;
 154
 155         pte_val(pte) = _PAGE_TYPE_EMPTY;
 156         for (address = start; address < start + size; address += PAGE_SIZE) {
 157                 pg_dir = pgd_offset_k(address);
 158                 if (pgd_none(*pg_dir))
 159                         continue;
 160                 pm_dir = pmd_offset(pg_dir, address);
 161                 if (pmd_none(*pm_dir))
 162                         continue;
 163                 pt_dir = pte_offset_kernel(pm_dir, address);
 164                 *pt_dir = pte;
 165         }
 166         flush_tlb_kernel_range(start, start + size);
 167 }
 168
 169 /*
 170  * Add a backed mem_map array to the virtual mem_map array.
 171  */
 172 static int vmem_add_mem_map(unsigned long start, unsigned long size)
 173 {
 174         unsigned long address, start_addr, end_addr;
 175         struct page *map_start, *map_end;
 176         pgd_t *pg_dir;
 177         pmd_t *pm_dir;
 178         pte_t *pt_dir;
 179         pte_t  pte;
 180         int ret = -ENOMEM;
 181
 182         map_start = vmem_map + PFN_DOWN(start);
 183         map_end = vmem_map + PFN_DOWN(start + size);
 184
 185         start_addr = (unsigned long) map_start & PAGE_MASK;
 186         end_addr = PFN_ALIGN((unsigned long) map_end);
 187
 188         for (address = start_addr; address < end_addr; address += PAGE_SIZE) {
 189                 pg_dir = pgd_offset_k(address);
 190                 if (pgd_none(*pg_dir)) {
 191                         pm_dir = vmem_pmd_alloc();
 192                         if (!pm_dir)
 193                                 goto out;
 194                         pgd_populate_kernel(&init_mm, pg_dir, pm_dir);
 195                 }
 196
 197                 pm_dir = pmd_offset(pg_dir, address);
 198                 if (pmd_none(*pm_dir)) {
 199                         pt_dir = vmem_pte_alloc();
 200                         if (!pt_dir)
 201                                 goto out;
 202                         pmd_populate_kernel(&init_mm, pm_dir, pt_dir);
 203                 }
 204
 205                 pt_dir = pte_offset_kernel(pm_dir, address);
 206                 if (pte_none(*pt_dir)) {
 207                         unsigned long new_page;
 208
 209                         new_page =__pa(vmem_alloc_pages(0));
 210                         if (!new_page)
 211                                 goto out;
 212                         pte = pfn_pte(new_page >> PAGE_SHIFT, PAGE_KERNEL);
 213                         *pt_dir = pte;
 214                 }
 215         }
 216         ret = 0;
 217 out:
 218         flush_tlb_kernel_range(start_addr, end_addr);
 219         return ret;
 220 }
 221
 222 static int vmem_add_mem(unsigned long start, unsigned long size)
 223 {
 224         int ret;
 225
 226         ret = vmem_add_range(start, size);
 227         if (ret)
 228                 return ret;
 229         return vmem_add_mem_map(start, size);
 230 }
 231
 232 /*
 233  * Add memory segment to the segment list if it doesn't overlap with
 234  * an already present segment.
 235  */
 236 static int insert_memory_segment(struct memory_segment *seg)
 237 {
 238         struct memory_segment *tmp;
 239
 240         if (PFN_DOWN(seg->start + seg->size) > max_pfn ||
 241             seg->start + seg->size < seg->start)
 242                 return -ERANGE;
 243
 244         list_for_each_entry(tmp, &mem_segs, list) {
 245                 if (seg->start >= tmp->start + tmp->size)
 246                         continue;
 247                 if (seg->start + seg->size <= tmp->start)
 248                         continue;
 249                 return -ENOSPC;
 250         }
 251         list_add(&seg->list, &mem_segs);
 252         return 0;
 253 }
 254
 255 /*
 256  * Remove memory segment from the segment list.
 257  */
 258 static void remove_memory_segment(struct memory_segment *seg)
 259 {
 260         list_del(&seg->list);
 261 }
 262
 263 static void __remove_shared_memory(struct memory_segment *seg)
 264 {
 265         remove_memory_segment(seg);
 266         vmem_remove_range(seg->start, seg->size);
 267 }
 268
 269 int remove_shared_memory(unsigned long start, unsigned long size)
 270 {
 271         struct memory_segment *seg;
 272         int ret;
 273
 274         mutex_lock(&vmem_mutex);
 275
 276         ret = -ENOENT;
 277         list_for_each_entry(seg, &mem_segs, list) {
 278                 if (seg->start == start && seg->size == size)
 279                         break;
 280         }
 281
 282         if (seg->start != start || seg->size != size)
 283                 goto out;
 284
 285         ret = 0;
 286         __remove_shared_memory(seg);
 287         kfree(seg);
 288 out:
 289         mutex_unlock(&vmem_mutex);
 290         return ret;
 291 }
 292
 293 int add_shared_memory(unsigned long start, unsigned long size)
 294 {
 295         struct memory_segment *seg;
 296         struct page *page;
 297         unsigned long pfn, num_pfn, end_pfn;
 298         int ret;
 299
 300         mutex_lock(&vmem_mutex);
 301         ret = -ENOMEM;
 302         seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 303         if (!seg)
 304                 goto out;
 305         seg->start = start;
 306         seg->size = size;
 307
 308         ret = insert_memory_segment(seg);
 309         if (ret)
 310                 goto out_free;
 311
 312         ret = vmem_add_mem(start, size);
 313         if (ret)
 314                 goto out_remove;
 315
 316         pfn = PFN_DOWN(start);
 317         num_pfn = PFN_DOWN(size);
 318         end_pfn = pfn + num_pfn;
 319
 320         page = pfn_to_page(pfn);
 321         memset(page, 0, num_pfn * sizeof(struct page));
 322
 323         for (; pfn < end_pfn; pfn++) {
 324                 page = pfn_to_page(pfn);
 325                 init_page_count(page);
 326                 reset_page_mapcount(page);
 327                 SetPageReserved(page);
 328                 INIT_LIST_HEAD(&page->lru);
 329         }
 330         goto out;
 331
 332 out_remove:
 333         __remove_shared_memory(seg);
 334 out_free:
 335         kfree(seg);
 336 out:
 337         mutex_unlock(&vmem_mutex);
 338         return ret;
 339 }
 340
 341 /*
 342  * map whole physical memory to virtual memory (identity mapping)
 343  */
 344 void __init vmem_map_init(void)
 345 {
 346         unsigned long map_size;
 347         int i;
 348
 349         map_size = ALIGN(max_low_pfn, MAX_ORDER_NR_PAGES) * sizeof(struct page);
 350         vmalloc_end = PFN_ALIGN(VMALLOC_END_INIT) - PFN_ALIGN(map_size);
 351         vmem_map = (struct page *) vmalloc_end;
 352         NODE_DATA(0)->node_mem_map = vmem_map;
 353
 354         for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++)
 355                 vmem_add_mem(memory_chunk[i].addr, memory_chunk[i].size);
 356 }
 357
 358 /*
 359  * Convert memory chunk array to a memory segment list so there is a single
 360  * list that contains both r/w memory and shared memory segments.
 361  */
 362 static int __init vmem_convert_memory_chunk(void)
 363 {
 364         struct memory_segment *seg;
 365         int i;
 366
 367         mutex_lock(&vmem_mutex);
 368         for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) {
 369                 if (!memory_chunk[i].size)
 370                         continue;
 371                 seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 372                 if (!seg)
 373                         panic("Out of memory...\n");
 374                 seg->start = memory_chunk[i].addr;
 375                 seg->size = memory_chunk[i].size;
 376                 insert_memory_segment(seg);
 377         }
 378         mutex_unlock(&vmem_mutex);
 379         return 0;
 380 }
 381
 382 core_initcall(vmem_convert_memory_chunk);