arch/x86/mm/init_32.c

   1 /*
   2  *  linux/arch/i386/mm/init.c
   3  *
   4  *  Copyright (C) 1995  Linus Torvalds
   5  *
   6  *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/signal.h>
  11 #include <linux/sched.h>
  12 #include <linux/kernel.h>
  13 #include <linux/errno.h>
  14 #include <linux/string.h>
  15 #include <linux/types.h>
  16 #include <linux/ptrace.h>
  17 #include <linux/mman.h>
  18 #include <linux/mm.h>
  19 #include <linux/hugetlb.h>
  20 #include <linux/swap.h>
  21 #include <linux/smp.h>
  22 #include <linux/init.h>
  23 #include <linux/highmem.h>
  24 #include <linux/pagemap.h>
  25 #include <linux/pfn.h>
  26 #include <linux/poison.h>
  27 #include <linux/bootmem.h>
  28 #include <linux/slab.h>
  29 #include <linux/proc_fs.h>
  30 #include <linux/memory_hotplug.h>
  31 #include <linux/initrd.h>
  32 #include <linux/cpumask.h>
  33
  34 #include <asm/processor.h>
  35 #include <asm/system.h>
  36 #include <asm/uaccess.h>
  37 #include <asm/pgtable.h>
  38 #include <asm/dma.h>
  39 #include <asm/fixmap.h>
  40 #include <asm/e820.h>
  41 #include <asm/apic.h>
  42 #include <asm/bugs.h>
  43 #include <asm/tlb.h>
  44 #include <asm/tlbflush.h>
  45 #include <asm/pgalloc.h>
  46 #include <asm/sections.h>
  47 #include <asm/paravirt.h>
  48
  49 unsigned int __VMALLOC_RESERVE = 128 << 20;
  50
  51 DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
  52 unsigned long highstart_pfn, highend_pfn;
  53
  54 static noinline int do_test_wp_bit(void);
  55
  56 /*
  57  * Creates a middle page table and puts a pointer to it in the
  58  * given global directory entry. This only returns the gd entry
  59  * in non-PAE compilation mode, since the middle layer is folded.
  60  */
  61 static pmd_t * __init one_md_table_init(pgd_t *pgd)
  62 {
  63         pud_t *pud;
  64         pmd_t *pmd_table;
  65
  66 #ifdef CONFIG_X86_PAE
  67         if (!(pgd_val(*pgd) & _PAGE_PRESENT)) {
  68                 pmd_table = (pmd_t *) alloc_bootmem_low_pages(PAGE_SIZE);
  69
  70                 paravirt_alloc_pd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);
  71                 set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
  72                 pud = pud_offset(pgd, 0);
  73                 BUG_ON(pmd_table != pmd_offset(pud, 0));
  74         }
  75 #endif
  76         pud = pud_offset(pgd, 0);
  77         pmd_table = pmd_offset(pud, 0);
  78
  79         return pmd_table;
  80 }
  81
  82 /*
  83  * Create a page table and place a pointer to it in a middle page
  84  * directory entry:
  85  */
  86 static pte_t * __init one_page_table_init(pmd_t *pmd)
  87 {
  88         if (!(pmd_val(*pmd) & _PAGE_PRESENT)) {
  89                 pte_t *page_table = NULL;
  90
  91 #ifdef CONFIG_DEBUG_PAGEALLOC
  92                 page_table = (pte_t *) alloc_bootmem_pages(PAGE_SIZE);
  93 #endif
  94                 if (!page_table) {
  95                         page_table =
  96                                 (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE);
  97                 }
  98
  99                 paravirt_alloc_pt(&init_mm, __pa(page_table) >> PAGE_SHIFT);
 100                 set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE));
 101                 BUG_ON(page_table != pte_offset_kernel(pmd, 0));
 102         }
 103
 104         return pte_offset_kernel(pmd, 0);
 105 }
 106
 107 /*
 108  * This function initializes a certain range of kernel virtual memory
 109  * with new bootmem page tables, everywhere page tables are missing in
 110  * the given range.
 111  *
 112  * NOTE: The pagetables are allocated contiguous on the physical space
 113  * so we can cache the place of the first one and move around without
 114  * checking the pgd every time.
 115  */
 116 static void __init
 117 page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base)
 118 {
 119         int pgd_idx, pmd_idx;
 120         unsigned long vaddr;
 121         pgd_t *pgd;
 122         pmd_t *pmd;
 123
 124         vaddr = start;
 125         pgd_idx = pgd_index(vaddr);
 126         pmd_idx = pmd_index(vaddr);
 127         pgd = pgd_base + pgd_idx;
 128
 129         for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) {
 130                 pmd = one_md_table_init(pgd);
 131                 pmd = pmd + pmd_index(vaddr);
 132                 for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
 133                                                         pmd++, pmd_idx++) {
 134                         one_page_table_init(pmd);
 135
 136                         vaddr += PMD_SIZE;
 137                 }
 138                 pmd_idx = 0;
 139         }
 140 }
 141
 142 static inline int is_kernel_text(unsigned long addr)
 143 {
 144         if (addr >= PAGE_OFFSET && addr <= (unsigned long)__init_end)
 145                 return 1;
 146         return 0;
 147 }
 148
 149 /*
 150  * This maps the physical memory to kernel virtual address space, a total
 151  * of max_low_pfn pages, by creating page tables starting from address
 152  * PAGE_OFFSET:
 153  */
 154 static void __init kernel_physical_mapping_init(pgd_t *pgd_base)
 155 {
 156         int pgd_idx, pmd_idx, pte_ofs;
 157         unsigned long pfn;
 158         pgd_t *pgd;
 159         pmd_t *pmd;
 160         pte_t *pte;
 161
 162         pgd_idx = pgd_index(PAGE_OFFSET);
 163         pgd = pgd_base + pgd_idx;
 164         pfn = 0;
 165
 166         for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) {
 167                 pmd = one_md_table_init(pgd);
 168                 if (pfn >= max_low_pfn)
 169                         continue;
 170
 171                 for (pmd_idx = 0;
 172                      pmd_idx < PTRS_PER_PMD && pfn < max_low_pfn;
 173                      pmd++, pmd_idx++) {
 174                         unsigned int addr = pfn * PAGE_SIZE + PAGE_OFFSET;
 175
 176                         /*
 177                          * Map with big pages if possible, otherwise
 178                          * create normal page tables:
 179                          */
 180                         if (cpu_has_pse) {
 181                                 unsigned int addr2;
 182                                 pgprot_t prot = PAGE_KERNEL_LARGE;
 183
 184                                 addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE +
 185                                         PAGE_OFFSET + PAGE_SIZE-1;
 186
 187                                 if (is_kernel_text(addr) ||
 188                                     is_kernel_text(addr2))
 189                                         prot = PAGE_KERNEL_LARGE_EXEC;
 190
 191                                 set_pmd(pmd, pfn_pmd(pfn, prot));
 192
 193                                 pfn += PTRS_PER_PTE;
 194                                 continue;
 195                         }
 196                         pte = one_page_table_init(pmd);
 197
 198                         for (pte_ofs = 0;
 199                              pte_ofs < PTRS_PER_PTE && pfn < max_low_pfn;
 200                              pte++, pfn++, pte_ofs++, addr += PAGE_SIZE) {
 201                                 pgprot_t prot = PAGE_KERNEL;
 202
 203                                 if (is_kernel_text(addr))
 204                                         prot = PAGE_KERNEL_EXEC;
 205
 206                                 set_pte(pte, pfn_pte(pfn, prot));
 207                         }
 208                 }
 209         }
 210 }
 211
 212 static inline int page_kills_ppro(unsigned long pagenr)
 213 {
 214         if (pagenr >= 0x70000 && pagenr <= 0x7003F)
 215                 return 1;
 216         return 0;
 217 }
 218
 219 #ifdef CONFIG_HIGHMEM
 220 pte_t *kmap_pte;
 221 pgprot_t kmap_prot;
 222
 223 static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr)
 224 {
 225         return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),
 226                         vaddr), vaddr), vaddr);
 227 }
 228
 229 static void __init kmap_init(void)
 230 {
 231         unsigned long kmap_vstart;
 232
 233         /*
 234          * Cache the first kmap pte:
 235          */
 236         kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
 237         kmap_pte = kmap_get_fixmap_pte(kmap_vstart);
 238
 239         kmap_prot = PAGE_KERNEL;
 240 }
 241
 242 static void __init permanent_kmaps_init(pgd_t *pgd_base)
 243 {
 244         unsigned long vaddr;
 245         pgd_t *pgd;
 246         pud_t *pud;
 247         pmd_t *pmd;
 248         pte_t *pte;
 249
 250         vaddr = PKMAP_BASE;
 251         page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);
 252
 253         pgd = swapper_pg_dir + pgd_index(vaddr);
 254         pud = pud_offset(pgd, vaddr);
 255         pmd = pmd_offset(pud, vaddr);
 256         pte = pte_offset_kernel(pmd, vaddr);
 257         pkmap_page_table = pte;
 258 }
 259
 260 static void __meminit free_new_highpage(struct page *page)
 261 {
 262         init_page_count(page);
 263         __free_page(page);
 264         totalhigh_pages++;
 265 }
 266
 267 void __init add_one_highpage_init(struct page *page, int pfn, int bad_ppro)
 268 {
 269         if (page_is_ram(pfn) && !(bad_ppro && page_kills_ppro(pfn))) {
 270                 ClearPageReserved(page);
 271                 free_new_highpage(page);
 272         } else
 273                 SetPageReserved(page);
 274 }
 275
 276 static int __meminit
 277 add_one_highpage_hotplug(struct page *page, unsigned long pfn)
 278 {
 279         free_new_highpage(page);
 280         totalram_pages++;
 281 #ifdef CONFIG_FLATMEM
 282         max_mapnr = max(pfn, max_mapnr);
 283 #endif
 284         num_physpages++;
 285
 286         return 0;
 287 }
 288
 289 /*
 290  * Not currently handling the NUMA case.
 291  * Assuming single node and all memory that
 292  * has been added dynamically that would be
 293  * onlined here is in HIGHMEM.
 294  */
 295 void __meminit online_page(struct page *page)
 296 {
 297         ClearPageReserved(page);
 298         add_one_highpage_hotplug(page, page_to_pfn(page));
 299 }
 300
 301 #ifndef CONFIG_NUMA
 302 static void __init set_highmem_pages_init(int bad_ppro)
 303 {
 304         int pfn;
 305
 306         for (pfn = highstart_pfn; pfn < highend_pfn; pfn++) {
 307                 /*
 308                  * Holes under sparsemem might not have no mem_map[]:
 309                  */
 310                 if (pfn_valid(pfn))
 311                         add_one_highpage_init(pfn_to_page(pfn), pfn, bad_ppro);
 312         }
 313         totalram_pages += totalhigh_pages;
 314 }
 315 #endif /* !CONFIG_NUMA */
 316
 317 #else
 318 # define kmap_init()                            do { } while (0)
 319 # define permanent_kmaps_init(pgd_base)         do { } while (0)
 320 # define set_highmem_pages_init(bad_ppro)       do { } while (0)
 321 #endif /* CONFIG_HIGHMEM */
 322
 323 pteval_t __PAGE_KERNEL = _PAGE_KERNEL;
 324 EXPORT_SYMBOL(__PAGE_KERNEL);
 325
 326 pteval_t __PAGE_KERNEL_EXEC = _PAGE_KERNEL_EXEC;
 327
 328 void __init native_pagetable_setup_start(pgd_t *base)
 329 {
 330 #ifdef CONFIG_X86_PAE
 331         int i;
 332
 333         /*
 334          * Init entries of the first-level page table to the
 335          * zero page, if they haven't already been set up.
 336          *
 337          * In a normal native boot, we'll be running on a
 338          * pagetable rooted in swapper_pg_dir, but not in PAE
 339          * mode, so this will end up clobbering the mappings
 340          * for the lower 24Mbytes of the address space,
 341          * without affecting the kernel address space.
 342          */
 343         for (i = 0; i < USER_PTRS_PER_PGD; i++)
 344                 set_pgd(&base[i],
 345                         __pgd(__pa(empty_zero_page) | _PAGE_PRESENT));
 346
 347         /* Make sure kernel address space is empty so that a pagetable
 348            will be allocated for it. */
 349         memset(&base[USER_PTRS_PER_PGD], 0,
 350                KERNEL_PGD_PTRS * sizeof(pgd_t));
 351 #else
 352         paravirt_alloc_pd(&init_mm, __pa(base) >> PAGE_SHIFT);
 353 #endif
 354 }
 355
 356 void __init native_pagetable_setup_done(pgd_t *base)
 357 {
 358 #ifdef CONFIG_X86_PAE
 359         /*
 360          * Add low memory identity-mappings - SMP needs it when
 361          * starting up on an AP from real-mode. In the non-PAE
 362          * case we already have these mappings through head.S.
 363          * All user-space mappings are explicitly cleared after
 364          * SMP startup.
 365          */
 366         set_pgd(&base[0], base[USER_PTRS_PER_PGD]);
 367 #endif
 368 }
 369
 370 /*
 371  * Build a proper pagetable for the kernel mappings.  Up until this
 372  * point, we've been running on some set of pagetables constructed by
 373  * the boot process.
 374  *
 375  * If we're booting on native hardware, this will be a pagetable
 376  * constructed in arch/i386/kernel/head.S, and not running in PAE mode
 377  * (even if we'll end up running in PAE).  The root of the pagetable
 378  * will be swapper_pg_dir.
 379  *
 380  * If we're booting paravirtualized under a hypervisor, then there are
 381  * more options: we may already be running PAE, and the pagetable may
 382  * or may not be based in swapper_pg_dir.  In any case,
 383  * paravirt_pagetable_setup_start() will set up swapper_pg_dir
 384  * appropriately for the rest of the initialization to work.
 385  *
 386  * In general, pagetable_init() assumes that the pagetable may already
 387  * be partially populated, and so it avoids stomping on any existing
 388  * mappings.
 389  */
 390 static void __init pagetable_init(void)
 391 {
 392         pgd_t *pgd_base = swapper_pg_dir;
 393         unsigned long vaddr, end;
 394
 395         paravirt_pagetable_setup_start(pgd_base);
 396
 397         /* Enable PSE if available */
 398         if (cpu_has_pse)
 399                 set_in_cr4(X86_CR4_PSE);
 400
 401         /* Enable PGE if available */
 402         if (cpu_has_pge) {
 403                 set_in_cr4(X86_CR4_PGE);
 404                 __PAGE_KERNEL |= _PAGE_GLOBAL;
 405                 __PAGE_KERNEL_EXEC |= _PAGE_GLOBAL;
 406         }
 407
 408         kernel_physical_mapping_init(pgd_base);
 409         remap_numa_kva();
 410
 411         /*
 412          * Fixed mappings, only the page table structure has to be
 413          * created - mappings will be set by set_fixmap():
 414          */
 415         early_ioremap_clear();
 416         vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
 417         end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK;
 418         page_table_range_init(vaddr, end, pgd_base);
 419         early_ioremap_reset();
 420
 421         permanent_kmaps_init(pgd_base);
 422
 423         paravirt_pagetable_setup_done(pgd_base);
 424 }
 425
 426 #if defined(CONFIG_HIBERNATION) || defined(CONFIG_ACPI)
 427 /*
 428  * Swap suspend & friends need this for resume because things like the intel-agp
 429  * driver might have split up a kernel 4MB mapping.
 430  */
 431 char __nosavedata swsusp_pg_dir[PAGE_SIZE]
 432         __attribute__ ((aligned(PAGE_SIZE)));
 433
 434 static inline void save_pg_dir(void)
 435 {
 436         memcpy(swsusp_pg_dir, swapper_pg_dir, PAGE_SIZE);
 437 }
 438 #else
 439 static inline void save_pg_dir(void)
 440 {
 441 }
 442 #endif
 443
 444 void zap_low_mappings(void)
 445 {
 446         int i;
 447
 448         save_pg_dir();
 449
 450         /*
 451          * Zap initial low-memory mappings.
 452          *
 453          * Note that "pgd_clear()" doesn't do it for
 454          * us, because pgd_clear() is a no-op on i386.
 455          */
 456         for (i = 0; i < USER_PTRS_PER_PGD; i++) {
 457 #ifdef CONFIG_X86_PAE
 458                 set_pgd(swapper_pg_dir+i, __pgd(1 + __pa(empty_zero_page)));
 459 #else
 460                 set_pgd(swapper_pg_dir+i, __pgd(0));
 461 #endif
 462         }
 463         flush_tlb_all();
 464 }
 465
 466 int nx_enabled;
 467
 468 pteval_t __supported_pte_mask __read_mostly = ~_PAGE_NX;
 469 EXPORT_SYMBOL_GPL(__supported_pte_mask);
 470
 471 #ifdef CONFIG_X86_PAE
 472
 473 static int disable_nx __initdata;
 474
 475 /*
 476  * noexec = on|off
 477  *
 478  * Control non executable mappings.
 479  *
 480  * on      Enable
 481  * off     Disable
 482  */
 483 static int __init noexec_setup(char *str)
 484 {
 485         if (!str || !strcmp(str, "on")) {
 486                 if (cpu_has_nx) {
 487                         __supported_pte_mask |= _PAGE_NX;
 488                         disable_nx = 0;
 489                 }
 490         } else {
 491                 if (!strcmp(str, "off")) {
 492                         disable_nx = 1;
 493                         __supported_pte_mask &= ~_PAGE_NX;
 494                 } else {
 495                         return -EINVAL;
 496                 }
 497         }
 498
 499         return 0;
 500 }
 501 early_param("noexec", noexec_setup);
 502
 503 static void __init set_nx(void)
 504 {
 505         unsigned int v[4], l, h;
 506
 507         if (cpu_has_pae && (cpuid_eax(0x80000000) > 0x80000001)) {
 508                 cpuid(0x80000001, &v[0], &v[1], &v[2], &v[3]);
 509
 510                 if ((v[3] & (1 << 20)) && !disable_nx) {
 511                         rdmsr(MSR_EFER, l, h);
 512                         l |= EFER_NX;
 513                         wrmsr(MSR_EFER, l, h);
 514                         nx_enabled = 1;
 515                         __supported_pte_mask |= _PAGE_NX;
 516                 }
 517         }
 518 }
 519 #endif
 520
 521 /*
 522  * paging_init() sets up the page tables - note that the first 8MB are
 523  * already mapped by head.S.
 524  *
 525  * This routines also unmaps the page at virtual kernel address 0, so
 526  * that we can trap those pesky NULL-reference errors in the kernel.
 527  */
 528 void __init paging_init(void)
 529 {
 530 #ifdef CONFIG_X86_PAE
 531         set_nx();
 532         if (nx_enabled)
 533                 printk("NX (Execute Disable) protection: active\n");
 534 #endif
 535         pagetable_init();
 536
 537         load_cr3(swapper_pg_dir);
 538
 539 #ifdef CONFIG_X86_PAE
 540         /*
 541          * We will bail out later - printk doesn't work right now so
 542          * the user would just see a hanging kernel.
 543          */
 544         if (cpu_has_pae)
 545                 set_in_cr4(X86_CR4_PAE);
 546 #endif
 547         __flush_tlb_all();
 548
 549         kmap_init();
 550 }
 551
 552 /*
 553  * Test if the WP bit works in supervisor mode. It isn't supported on 386's
 554  * and also on some strange 486's (NexGen etc.). All 586+'s are OK. This
 555  * used to involve black magic jumps to work around some nasty CPU bugs,
 556  * but fortunately the switch to using exceptions got rid of all that.
 557  */
 558 static void __init test_wp_bit(void)
 559 {
 560         printk("Checking if this processor honours the WP bit even in supervisor mode... ");
 561
 562         /* Any page-aligned address will do, the test is non-destructive */
 563         __set_fixmap(FIX_WP_TEST, __pa(&swapper_pg_dir), PAGE_READONLY);
 564         boot_cpu_data.wp_works_ok = do_test_wp_bit();
 565         clear_fixmap(FIX_WP_TEST);
 566
 567         if (!boot_cpu_data.wp_works_ok) {
 568                 printk("No.\n");
 569 #ifdef CONFIG_X86_WP_WORKS_OK
 570                 panic("This kernel doesn't support CPU's with broken WP. Recompile it for a 386!");
 571 #endif
 572         } else {
 573                 printk("Ok.\n");
 574         }
 575 }
 576
 577 static struct kcore_list kcore_mem, kcore_vmalloc;
 578
 579 void __init mem_init(void)
 580 {
 581         int codesize, reservedpages, datasize, initsize;
 582         int tmp, bad_ppro;
 583
 584 #ifdef CONFIG_FLATMEM
 585         BUG_ON(!mem_map);
 586 #endif
 587         bad_ppro = ppro_with_ram_bug();
 588
 589 #ifdef CONFIG_HIGHMEM
 590         /* check that fixmap and pkmap do not overlap */
 591         if (PKMAP_BASE+LAST_PKMAP*PAGE_SIZE >= FIXADDR_START) {
 592                 printk(KERN_ERR "fixmap and kmap areas overlap - this will crash\n");
 593                 printk(KERN_ERR "pkstart: %lxh pkend: %lxh fixstart %lxh\n",
 594                                 PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE, FIXADDR_START);
 595                 BUG();
 596         }
 597 #endif
 598         /* this will put all low memory onto the freelists */
 599         totalram_pages += free_all_bootmem();
 600
 601         reservedpages = 0;
 602         for (tmp = 0; tmp < max_low_pfn; tmp++)
 603                 /*
 604                  * Only count reserved RAM pages:
 605                  */
 606                 if (page_is_ram(tmp) && PageReserved(pfn_to_page(tmp)))
 607                         reservedpages++;
 608
 609         set_highmem_pages_init(bad_ppro);
 610
 611         codesize =  (unsigned long) &_etext - (unsigned long) &_text;
 612         datasize =  (unsigned long) &_edata - (unsigned long) &_etext;
 613         initsize =  (unsigned long) &__init_end - (unsigned long) &__init_begin;
 614
 615         kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT);
 616         kclist_add(&kcore_vmalloc, (void *)VMALLOC_START,
 617                    VMALLOC_END-VMALLOC_START);
 618
 619         printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, "
 620                         "%dk reserved, %dk data, %dk init, %ldk highmem)\n",
 621                 (unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
 622                 num_physpages << (PAGE_SHIFT-10),
 623                 codesize >> 10,
 624                 reservedpages << (PAGE_SHIFT-10),
 625                 datasize >> 10,
 626                 initsize >> 10,
 627                 (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10))
 628                );
 629
 630 #if 1 /* double-sanity-check paranoia */
 631         printk("virtual kernel memory layout:\n"
 632                 "    fixmap  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
 633 #ifdef CONFIG_HIGHMEM
 634                 "    pkmap   : 0x%08lx - 0x%08lx   (%4ld kB)\n"
 635 #endif
 636                 "    vmalloc : 0x%08lx - 0x%08lx   (%4ld MB)\n"
 637                 "    lowmem  : 0x%08lx - 0x%08lx   (%4ld MB)\n"
 638                 "      .init : 0x%08lx - 0x%08lx   (%4ld kB)\n"
 639                 "      .data : 0x%08lx - 0x%08lx   (%4ld kB)\n"
 640                 "      .text : 0x%08lx - 0x%08lx   (%4ld kB)\n",
 641                 FIXADDR_START, FIXADDR_TOP,
 642                 (FIXADDR_TOP - FIXADDR_START) >> 10,
 643
 644 #ifdef CONFIG_HIGHMEM
 645                 PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE,
 646                 (LAST_PKMAP*PAGE_SIZE) >> 10,
 647 #endif
 648
 649                 VMALLOC_START, VMALLOC_END,
 650                 (VMALLOC_END - VMALLOC_START) >> 20,
 651
 652                 (unsigned long)__va(0), (unsigned long)high_memory,
 653                 ((unsigned long)high_memory - (unsigned long)__va(0)) >> 20,
 654
 655                 (unsigned long)&__init_begin, (unsigned long)&__init_end,
 656                 ((unsigned long)&__init_end -
 657                  (unsigned long)&__init_begin) >> 10,
 658
 659                 (unsigned long)&_etext, (unsigned long)&_edata,
 660                 ((unsigned long)&_edata - (unsigned long)&_etext) >> 10,
 661
 662                 (unsigned long)&_text, (unsigned long)&_etext,
 663                 ((unsigned long)&_etext - (unsigned long)&_text) >> 10);
 664
 665 #ifdef CONFIG_HIGHMEM
 666         BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE        > FIXADDR_START);
 667         BUG_ON(VMALLOC_END                              > PKMAP_BASE);
 668 #endif
 669         BUG_ON(VMALLOC_START                            > VMALLOC_END);
 670         BUG_ON((unsigned long)high_memory               > VMALLOC_START);
 671 #endif /* double-sanity-check paranoia */
 672
 673 #ifdef CONFIG_X86_PAE
 674         if (!cpu_has_pae)
 675                 panic("cannot execute a PAE-enabled kernel on a PAE-less CPU!");
 676 #endif
 677         if (boot_cpu_data.wp_works_ok < 0)
 678                 test_wp_bit();
 679
 680         /*
 681          * Subtle. SMP is doing it's boot stuff late (because it has to
 682          * fork idle threads) - but it also needs low mappings for the
 683          * protected-mode entry to work. We zap these entries only after
 684          * the WP-bit has been tested.
 685          */
 686 #ifndef CONFIG_SMP
 687         zap_low_mappings();
 688 #endif
 689 }
 690
 691 #ifdef CONFIG_MEMORY_HOTPLUG
 692 int arch_add_memory(int nid, u64 start, u64 size)
 693 {
 694         struct pglist_data *pgdata = NODE_DATA(nid);
 695         struct zone *zone = pgdata->node_zones + ZONE_HIGHMEM;
 696         unsigned long start_pfn = start >> PAGE_SHIFT;
 697         unsigned long nr_pages = size >> PAGE_SHIFT;
 698
 699         return __add_pages(zone, start_pfn, nr_pages);
 700 }
 701 #endif
 702
 703 struct kmem_cache *pmd_cache;
 704
 705 void __init pgtable_cache_init(void)
 706 {
 707         if (PTRS_PER_PMD > 1) {
 708                 pmd_cache = kmem_cache_create("pmd",
 709                                               PTRS_PER_PMD*sizeof(pmd_t),
 710                                               PTRS_PER_PMD*sizeof(pmd_t),
 711                                               SLAB_PANIC,
 712                                               pmd_ctor);
 713         }
 714 }
 715
 716 /*
 717  * This function cannot be __init, since exceptions don't work in that
 718  * section.  Put this after the callers, so that it cannot be inlined.
 719  */
 720 static noinline int do_test_wp_bit(void)
 721 {
 722         char tmp_reg;
 723         int flag;
 724
 725         __asm__ __volatile__(
 726                 "       movb %0, %1     \n"
 727                 "1:     movb %1, %0     \n"
 728                 "       xorl %2, %2     \n"
 729                 "2:                     \n"
 730                 ".section __ex_table, \"a\"\n"
 731                 "       .align 4        \n"
 732                 "       .long 1b, 2b    \n"
 733                 ".previous              \n"
 734                 :"=m" (*(char *)fix_to_virt(FIX_WP_TEST)),
 735                  "=q" (tmp_reg),
 736                  "=r" (flag)
 737                 :"2" (1)
 738                 :"memory");
 739
 740         return flag;
 741 }
 742
 743 #ifdef CONFIG_DEBUG_RODATA
 744 const int rodata_test_data = 0xC3;
 745 EXPORT_SYMBOL_GPL(rodata_test_data);
 746
 747 void mark_rodata_ro(void)
 748 {
 749         unsigned long start = PFN_ALIGN(_text);
 750         unsigned long size = PFN_ALIGN(_etext) - start;
 751
 752 #ifndef CONFIG_KPROBES
 753 #ifdef CONFIG_HOTPLUG_CPU
 754         /* It must still be possible to apply SMP alternatives. */
 755         if (num_possible_cpus() <= 1)
 756 #endif
 757         {
 758                 set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
 759                 printk("Write protecting the kernel text: %luk\n", size >> 10);
 760
 761 #ifdef CONFIG_CPA_DEBUG
 762                 printk("Testing CPA: Reverting %lx-%lx\n", start, start+size);
 763                 set_pages_rw(virt_to_page(start), size>>PAGE_SHIFT);
 764
 765                 printk("Testing CPA: write protecting again\n");
 766                 set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT);
 767 #endif
 768         }
 769 #endif
 770         start += size;
 771         size = (unsigned long)__end_rodata - start;
 772         set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
 773         printk("Write protecting the kernel read-only data: %luk\n",
 774                size >> 10);
 775         rodata_test();
 776
 777 #ifdef CONFIG_CPA_DEBUG
 778         printk("Testing CPA: undo %lx-%lx\n", start, start + size);
 779         set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT);
 780
 781         printk("Testing CPA: write protecting again\n");
 782         set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
 783 #endif
 784 }
 785 #endif
 786
 787 void free_init_pages(char *what, unsigned long begin, unsigned long end)
 788 {
 789 #ifdef CONFIG_DEBUG_PAGEALLOC
 790         /*
 791          * If debugging page accesses then do not free this memory but
 792          * mark them not present - any buggy init-section access will
 793          * create a kernel page fault:
 794          */
 795         printk(KERN_INFO "debug: unmapping init memory %08lx..%08lx\n",
 796                 begin, PAGE_ALIGN(end));
 797         set_memory_np(begin, (end - begin) >> PAGE_SHIFT);
 798 #else
 799         unsigned long addr;
 800
 801         /*
 802          * We just marked the kernel text read only above, now that
 803          * we are going to free part of that, we need to make that
 804          * writeable first.
 805          */
 806         set_memory_rw(begin, (end - begin) >> PAGE_SHIFT);
 807
 808         for (addr = begin; addr < end; addr += PAGE_SIZE) {
 809                 ClearPageReserved(virt_to_page(addr));
 810                 init_page_count(virt_to_page(addr));
 811                 memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
 812                 free_page(addr);
 813                 totalram_pages++;
 814         }
 815         printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10);
 816 #endif
 817 }
 818
 819 void free_initmem(void)
 820 {
 821         free_init_pages("unused kernel memory",
 822                         (unsigned long)(&__init_begin),
 823                         (unsigned long)(&__init_end));
 824 }
 825
 826 #ifdef CONFIG_BLK_DEV_INITRD
 827 void free_initrd_mem(unsigned long start, unsigned long end)
 828 {
 829         free_init_pages("initrd memory", start, end);
 830 }
 831 #endif