servers/vm/arch/i386/pagetable.c

   1
   2 #define _SYSTEM 1
   3 #define _POSIX_SOURCE 1
   4
   5 #include <minix/callnr.h>
   6 #include <minix/com.h>
   7 #include <minix/config.h>
   8 #include <minix/const.h>
   9 #include <minix/ds.h>
  10 #include <minix/endpoint.h>
  11 #include <minix/keymap.h>
  12 #include <minix/minlib.h>
  13 #include <minix/type.h>
  14 #include <minix/ipc.h>
  15 #include <minix/sysutil.h>
  16 #include <minix/syslib.h>
  17 #include <minix/safecopies.h>
  18 #include <minix/cpufeature.h>
  19 #include <minix/bitmap.h>
  20 #include <minix/debug.h>
  21
  22 #include <errno.h>
  23 #include <stdlib.h>
  24 #include <assert.h>
  25 #include <string.h>
  26 #include <env.h>
  27 #include <stdio.h>
  28 #include <fcntl.h>
  29 #include <stdlib.h>
  30
  31 #include "proto.h"
  32 #include "glo.h"
  33 #include "util.h"
  34 #include "vm.h"
  35 #include "sanitycheck.h"
  36
  37 static int vm_self_pages;
  38
  39 /* PDE used to map in kernel, kernel physical address. */
  40 static int pagedir_pde = -1;
  41 static u32_t global_bit = 0, pagedir_pde_val;
  42
  43 static multiboot_module_t *kern_mb_mod = NULL;
  44 static size_t kern_size = 0;
  45 static int kern_start_pde = -1;
  46
  47 /* big page size available in hardware? */
  48 static int bigpage_ok = 1;
  49
  50 /* Our process table entry. */
  51 struct vmproc *vmprocess = &vmproc[VM_PROC_NR];
  52
  53 /* Spare memory, ready to go after initialization, to avoid a
  54  * circular dependency on allocating memory and writing it into VM's
  55  * page table.
  56  */
  57 #if SANITYCHECKS
  58 #define SPAREPAGES 100
  59 #define STATIC_SPAREPAGES 90
  60 #else
  61 #ifdef __arm__
  62 # define SPAREPAGES 80
  63 # define STATIC_SPAREPAGES 75
  64 #else
  65 # define SPAREPAGES 20
  66 # define STATIC_SPAREPAGES 15
  67 #endif /* __arm__ */
  68 #endif
  69
  70 #define SPAREPAGEDIRS 11
  71 #define STATIC_SPAREPAGEDIRS 10
  72
  73 int missing_sparedirs = SPAREPAGEDIRS;
  74 static struct {
  75         void *pagedir;
  76         phys_bytes phys;
  77 } sparepagedirs[SPAREPAGEDIRS];
  78
  79 int missing_spares = SPAREPAGES;
  80 static struct {
  81         void *page;
  82         phys_bytes phys;
  83 } sparepages[SPAREPAGES];
  84
  85 extern char _end;
  86 #define is_staticaddr(v) ((vir_bytes) (v) < (vir_bytes) &_end)
  87
  88 #define MAX_KERNMAPPINGS 10
  89 static struct {
  90         phys_bytes      phys_addr;      /* Physical addr. */
  91         phys_bytes      len;            /* Length in bytes. */
  92         vir_bytes       vir_addr;       /* Offset in page table. */
  93         int             flags;
  94 } kern_mappings[MAX_KERNMAPPINGS];
  95 int kernmappings = 0;
  96
  97 /* Clicks must be pages, as
  98  *  - they must be page aligned to map them
  99  *  - they must be a multiple of the page size
 100  *  - it's inconvenient to have them bigger than pages, because we often want
 101  *    just one page
 102  * May as well require them to be equal then.
 103  */
 104 #if CLICK_SIZE != VM_PAGE_SIZE
 105 #error CLICK_SIZE must be page size.
 106 #endif
 107
 108 /* Page table that contains pointers to all page directories. */
 109 phys_bytes page_directories_phys;
 110 u32_t *page_directories = NULL;
 111
 112 static char static_sparepages[VM_PAGE_SIZE*STATIC_SPAREPAGES]
 113         __aligned(VM_PAGE_SIZE);
 114
 115 #if defined(__arm__)
 116 static char static_sparepagedirs[ARCH_PAGEDIR_SIZE*STATIC_SPAREPAGEDIRS + ARCH_PAGEDIR_SIZE] __aligned(ARCH_PAGEDIR_SIZE);
 117 #endif
 118
 119 #if SANITYCHECKS
 120 /*===========================================================================*
 121  *                              pt_sanitycheck                               *
 122  *===========================================================================*/
 123 void pt_sanitycheck(pt_t *pt, char *file, int line)
 124 {
 125 /* Basic pt sanity check. */
 126         int slot;
 127
 128         MYASSERT(pt);
 129         MYASSERT(pt->pt_dir);
 130         MYASSERT(pt->pt_dir_phys);
 131
 132         for(slot = 0; slot < ELEMENTS(vmproc); slot++) {
 133                 if(pt == &vmproc[slot].vm_pt)
 134                         break;
 135         }
 136
 137         if(slot >= ELEMENTS(vmproc)) {
 138                 panic("pt_sanitycheck: passed pt not in any proc");
 139         }
 140
 141         MYASSERT(usedpages_add(pt->pt_dir_phys, VM_PAGE_SIZE) == OK);
 142 }
 143 #endif
 144
 145 /*===========================================================================*
 146  *                              findhole                                     *
 147  *===========================================================================*/
 148 static u32_t findhole(int pages)
 149 {
 150 /* Find a space in the virtual address space of VM. */
 151         u32_t curv;
 152         int pde = 0, try_restart;
 153         static u32_t lastv = 0;
 154         pt_t *pt = &vmprocess->vm_pt;
 155         vir_bytes vmin, vmax;
 156 #if defined(__arm__)
 157         u32_t holev;
 158 #endif
 159
 160         vmin = (vir_bytes) (&_end); /* marks end of VM BSS */
 161         vmin += 1024*1024*1024; /* reserve 1GB virtual address space for VM heap */
 162         vmin &= ARCH_VM_ADDR_MASK;
 163         vmax = VM_STACKTOP;
 164
 165         /* Input sanity check. */
 166         assert(vmin + VM_PAGE_SIZE >= vmin);
 167         assert(vmax >= vmin + VM_PAGE_SIZE);
 168         assert((vmin % VM_PAGE_SIZE) == 0);
 169         assert((vmax % VM_PAGE_SIZE) == 0);
 170 #if defined(__arm__)
 171         assert(pages > 0);
 172 #endif
 173
 174         curv = lastv;
 175         if(curv < vmin || curv >= vmax)
 176                 curv = vmin;
 177
 178         try_restart = 1;
 179
 180         /* Start looking for a free page starting at vmin. */
 181         while(curv < vmax) {
 182                 int pte;
 183 #if defined(__arm__)
 184                 int i, nohole;
 185 #endif
 186
 187                 assert(curv >= vmin);
 188                 assert(curv < vmax);
 189
 190 #if defined(__i386__)
 191                 pde = I386_VM_PDE(curv);
 192                 pte = I386_VM_PTE(curv);
 193 #elif defined(__arm__)
 194                 holev = curv; /* the candidate hole */
 195                 nohole = 0;
 196                 for (i = 0; i < pages && !nohole; ++i) {
 197                     if(curv >= vmax) {
 198                         break;
 199                     }
 200 #endif
 201
 202 #if defined(__i386__)
 203                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT) ||
 204                    !(pt->pt_pt[pde][pte] & ARCH_VM_PAGE_PRESENT)) {
 205 #elif defined(__arm__)
 206                     pde = ARM_VM_PDE(curv);
 207                     pte = ARM_VM_PTE(curv);
 208
 209                     /* if page present, no hole */
 210                     if((pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT) &&
 211                        (pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT))
 212                         nohole = 1;
 213
 214                     /* if not contiguous, no hole */
 215                     if (curv != holev + i * VM_PAGE_SIZE)
 216                         nohole = 1;
 217
 218                     curv+=VM_PAGE_SIZE;
 219                 }
 220
 221                 /* there's a large enough hole */
 222                 if (!nohole && i == pages) {
 223 #endif
 224                         lastv = curv;
 225 #if defined(__i386__)
 226                         return curv;
 227 #elif defined(__arm__)
 228                         return holev;
 229 #endif
 230                 }
 231
 232 #if defined(__i386__)
 233                 curv+=VM_PAGE_SIZE;
 234
 235 #elif defined(__arm__)
 236                 /* Reset curv */
 237 #endif
 238                 if(curv >= vmax && try_restart) {
 239                         curv = vmin;
 240                         try_restart = 0;
 241                 }
 242         }
 243
 244         printf("VM: out of virtual address space in vm\n");
 245
 246         return NO_MEM;
 247 }
 248
 249 /*===========================================================================*
 250  *                              vm_freepages                                 *
 251  *===========================================================================*/
 252 void vm_freepages(vir_bytes vir, int pages)
 253 {
 254         assert(!(vir % VM_PAGE_SIZE));
 255
 256         if(is_staticaddr(vir)) {
 257                 printf("VM: not freeing static page\n");
 258                 return;
 259         }
 260
 261         if(pt_writemap(vmprocess, &vmprocess->vm_pt, vir,
 262                 MAP_NONE, pages*VM_PAGE_SIZE, 0,
 263                 WMF_OVERWRITE | WMF_FREE) != OK)
 264                 panic("vm_freepages: pt_writemap failed");
 265
 266         vm_self_pages--;
 267
 268 #if SANITYCHECKS
 269         /* If SANITYCHECKS are on, flush tlb so accessing freed pages is
 270          * always trapped, also if not in tlb.
 271          */
 272         if((sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 273                 panic("VMCTL_FLUSHTLB failed");
 274         }
 275 #endif
 276 }
 277
 278 /*===========================================================================*
 279  *                              vm_getsparepage                              *
 280  *===========================================================================*/
 281 static void *vm_getsparepage(phys_bytes *phys)
 282 {
 283         int s;
 284         assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
 285         for(s = 0; s < SPAREPAGES; s++) {
 286                 if(sparepages[s].page) {
 287                         void *sp;
 288                         sp = sparepages[s].page;
 289                         *phys = sparepages[s].phys;
 290                         sparepages[s].page = NULL;
 291                         missing_spares++;
 292                         assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
 293                         return sp;
 294                 }
 295         }
 296         printf("no spare found, %d missing\n", missing_spares);
 297         return NULL;
 298 }
 299
 300 /*===========================================================================*
 301  *                              vm_getsparepagedir                           *
 302  *===========================================================================*/
 303 static void *vm_getsparepagedir(phys_bytes *phys)
 304 {
 305         int s;
 306         assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
 307         for(s = 0; s < SPAREPAGEDIRS; s++) {
 308                 if(sparepagedirs[s].pagedir) {
 309                         void *sp;
 310                         sp = sparepagedirs[s].pagedir;
 311                         *phys = sparepagedirs[s].phys;
 312                         sparepagedirs[s].pagedir = NULL;
 313                         missing_sparedirs++;
 314                         assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
 315                         return sp;
 316                 }
 317         }
 318         return NULL;
 319 }
 320
 321 /*===========================================================================*
 322  *                              vm_checkspares                               *
 323  *===========================================================================*/
 324 static void *vm_checkspares(void)
 325 {
 326         int s, n = 0;
 327         static int total = 0, worst = 0;
 328         assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
 329         for(s = 0; s < SPAREPAGES && missing_spares > 0; s++) {
 330             if(!sparepages[s].page) {
 331                 n++;
 332                 if((sparepages[s].page = vm_allocpage(&sparepages[s].phys,
 333                         VMP_SPARE))) {
 334                         missing_spares--;
 335                         assert(missing_spares >= 0);
 336                         assert(missing_spares <= SPAREPAGES);
 337                 } else {
 338                         printf("VM: warning: couldn't get new spare page\n");
 339                 }
 340            }
 341         }
 342         if(worst < n) worst = n;
 343         total += n;
 344
 345         return NULL;
 346 }
 347
 348 #if defined(__arm__)
 349 /*===========================================================================*
 350  *                              vm_checksparedirs                            *
 351  *===========================================================================*/
 352 static void *vm_checksparedirs(void)
 353 {
 354         int s, n = 0;
 355         static int total = 0, worst = 0;
 356         assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
 357         for(s = 0; s < SPAREPAGEDIRS && missing_sparedirs > 0; s++)
 358             if(!sparepagedirs[s].pagedir) {
 359                 n++;
 360                 if((sparepagedirs[s].pagedir = vm_allocpage(&sparepagedirs[s].phys,
 361                         VMP_SPARE))) {
 362                         missing_sparedirs--;
 363                         assert(missing_sparedirs >= 0);
 364                         assert(missing_sparedirs <= SPAREPAGEDIRS);
 365                 } else {
 366                         printf("VM: warning: couldn't get new spare pagedir\n");
 367                 }
 368         }
 369         if(worst < n) worst = n;
 370         total += n;
 371
 372         return NULL;
 373 }
 374 #endif
 375
 376 static int pt_init_done;
 377
 378 /*===========================================================================*
 379  *                              vm_allocpage                                 *
 380  *===========================================================================*/
 381 void *vm_allocpages(phys_bytes *phys, int reason, int pages)
 382 {
 383 /* Allocate a page for use by VM itself. */
 384         phys_bytes newpage;
 385         vir_bytes loc;
 386         pt_t *pt;
 387         int r;
 388         static int level = 0;
 389         void *ret;
 390         u32_t mem_flags = 0;
 391
 392         pt = &vmprocess->vm_pt;
 393         assert(reason >= 0 && reason < VMP_CATEGORIES);
 394
 395         assert(pages > 0);
 396
 397         level++;
 398
 399         assert(level >= 1);
 400         assert(level <= 2);
 401
 402         if((level > 1) || !pt_init_done) {
 403                 void *s;
 404
 405                 if(pages == 1) s=vm_getsparepage(phys);
 406                 else if(pages == 4) s=vm_getsparepagedir(phys);
 407                 else panic("%d pages", pages);
 408
 409                 level--;
 410                 if(!s) {
 411                         util_stacktrace();
 412                         printf("VM: warning: out of spare pages\n");
 413                 }
 414                 if(!is_staticaddr(s)) vm_self_pages++;
 415                 return s;
 416         }
 417
 418 #if defined(__arm__)
 419         if (reason == VMP_PAGEDIR) {
 420                 mem_flags |= PAF_ALIGN16K;
 421         }
 422 #endif
 423
 424         /* VM does have a pagetable, so get a page and map it in there.
 425          * Where in our virtual address space can we put it?
 426          */
 427         loc = findhole(pages);
 428         if(loc == NO_MEM) {
 429                 level--;
 430                 printf("VM: vm_allocpage: findhole failed\n");
 431                 return NULL;
 432         }
 433
 434         /* Allocate page of memory for use by VM. As VM
 435          * is trusted, we don't have to pre-clear it.
 436          */
 437         if((newpage = alloc_mem(pages, mem_flags)) == NO_MEM) {
 438                 level--;
 439                 printf("VM: vm_allocpage: alloc_mem failed\n");
 440                 return NULL;
 441         }
 442
 443         *phys = CLICK2ABS(newpage);
 444
 445         /* Map this page into our address space. */
 446         if((r=pt_writemap(vmprocess, pt, loc, *phys, VM_PAGE_SIZE*pages,
 447                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW
 448 #if defined(__arm__)
 449                 | ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE
 450 #endif
 451                 , 0)) != OK) {
 452                 free_mem(newpage, pages);
 453                 printf("vm_allocpage writemap failed\n");
 454                 level--;
 455                 return NULL;
 456         }
 457
 458         if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 459                 panic("VMCTL_FLUSHTLB failed: %d", r);
 460         }
 461
 462         level--;
 463
 464         /* Return user-space-ready pointer to it. */
 465         ret = (void *) loc;
 466
 467         vm_self_pages++;
 468         return ret;
 469 }
 470
 471 void *vm_allocpage(phys_bytes *phys, int reason)
 472 {
 473         return vm_allocpages(phys, reason, 1);
 474 }
 475
 476 /*===========================================================================*
 477  *                              vm_pagelock                                  *
 478  *===========================================================================*/
 479 void vm_pagelock(void *vir, int lockflag)
 480 {
 481 /* Mark a page allocated by vm_allocpage() unwritable, i.e. only for VM. */
 482         vir_bytes m = (vir_bytes) vir;
 483         int r;
 484         u32_t flags = ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER;
 485         pt_t *pt;
 486
 487         pt = &vmprocess->vm_pt;
 488
 489         assert(!(m % VM_PAGE_SIZE));
 490
 491         if(!lockflag)
 492                 flags |= ARCH_VM_PTE_RW;
 493 #if defined(__arm__)
 494         else
 495                 flags |= ARCH_VM_PTE_RO;
 496         flags |= ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE;
 497 #endif
 498
 499         /* Update flags. */
 500         if((r=pt_writemap(vmprocess, pt, m, 0, VM_PAGE_SIZE,
 501                 flags, WMF_OVERWRITE | WMF_WRITEFLAGSONLY)) != OK) {
 502                 panic("vm_lockpage: pt_writemap failed");
 503         }
 504
 505         if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 506                 panic("VMCTL_FLUSHTLB failed: %d", r);
 507         }
 508
 509         return;
 510 }
 511
 512 /*===========================================================================*
 513  *                              vm_addrok                                    *
 514  *===========================================================================*/
 515 int vm_addrok(void *vir, int writeflag)
 516 {
 517         pt_t *pt = &vmprocess->vm_pt;
 518         int pde, pte;
 519         vir_bytes v = (vir_bytes) vir;
 520
 521 #if defined(__i386__)
 522         pde = I386_VM_PDE(v);
 523         pte = I386_VM_PTE(v);
 524 #elif defined(__arm__)
 525         pde = ARM_VM_PDE(v);
 526         pte = ARM_VM_PTE(v);
 527 #endif
 528
 529         if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 530                 printf("addr not ok: missing pde %d\n", pde);
 531                 return 0;
 532         }
 533
 534 #if defined(__i386__)
 535         if(writeflag &&
 536                 !(pt->pt_dir[pde] & ARCH_VM_PTE_RW)) {
 537                 printf("addr not ok: pde %d present but pde unwritable\n", pde);
 538                 return 0;
 539         }
 540
 541 #endif
 542         if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) {
 543                 printf("addr not ok: missing pde %d / pte %d\n",
 544                         pde, pte);
 545                 return 0;
 546         }
 547
 548 #if defined(__i386__)
 549         if(writeflag &&
 550                 !(pt->pt_pt[pde][pte] & ARCH_VM_PTE_RW)) {
 551                 printf("addr not ok: pde %d / pte %d present but unwritable\n",
 552 #elif defined(__arm__)
 553         if(!writeflag &&
 554                 !(pt->pt_pt[pde][pte] & ARCH_VM_PTE_RO)) {
 555                 printf("addr not ok: pde %d / pte %d present but writable\n",
 556 #endif
 557                         pde, pte);
 558                 return 0;
 559         }
 560
 561         return 1;
 562 }
 563
 564 /*===========================================================================*
 565  *                              pt_ptalloc                                   *
 566  *===========================================================================*/
 567 static int pt_ptalloc(pt_t *pt, int pde, u32_t flags)
 568 {
 569 /* Allocate a page table and write its address into the page directory. */
 570         int i;
 571         phys_bytes pt_phys;
 572
 573         /* Argument must make sense. */
 574         assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES);
 575         assert(!(flags & ~(PTF_ALLFLAGS)));
 576
 577         /* We don't expect to overwrite page directory entry, nor
 578          * storage for the page table.
 579          */
 580         assert(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT));
 581         assert(!pt->pt_pt[pde]);
 582
 583         /* Get storage for the page table. */
 584         if(!(pt->pt_pt[pde] = vm_allocpage(&pt_phys, VMP_PAGETABLE)))
 585                 return ENOMEM;
 586
 587         for(i = 0; i < ARCH_VM_PT_ENTRIES; i++)
 588                 pt->pt_pt[pde][i] = 0;  /* Empty entry. */
 589
 590         /* Make page directory entry.
 591          * The PDE is always 'present,' 'writable,' and 'user accessible,'
 592          * relying on the PTE for protection.
 593          */
 594 #if defined(__i386__)
 595         pt->pt_dir[pde] = (pt_phys & ARCH_VM_ADDR_MASK) | flags
 596                 | ARCH_VM_PDE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW;
 597 #elif defined(__arm__)
 598         pt->pt_dir[pde] = (pt_phys & ARCH_VM_PDE_MASK)
 599                 | ARCH_VM_PDE_PRESENT | ARM_VM_PDE_DOMAIN;
 600 #endif
 601
 602         return OK;
 603 }
 604
 605 /*===========================================================================*
 606  *                          pt_ptalloc_in_range                              *
 607  *===========================================================================*/
 608 int pt_ptalloc_in_range(pt_t *pt, vir_bytes start, vir_bytes end,
 609         u32_t flags, int verify)
 610 {
 611 /* Allocate all the page tables in the range specified. */
 612         int pde, first_pde, last_pde;
 613
 614 #if defined(__i386__)
 615         first_pde = I386_VM_PDE(start);
 616         last_pde = I386_VM_PDE(end-1);
 617 #elif defined(__arm__)
 618         first_pde = ARM_VM_PDE(start);
 619         last_pde = ARM_VM_PDE(end-1);
 620 #endif
 621         assert(first_pde >= 0);
 622         assert(last_pde < ARCH_VM_DIR_ENTRIES);
 623
 624         /* Scan all page-directory entries in the range. */
 625         for(pde = first_pde; pde <= last_pde; pde++) {
 626                 assert(!(pt->pt_dir[pde] & ARCH_VM_BIGPAGE));
 627                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 628                         int r;
 629                         if(verify) {
 630                                 printf("pt_ptalloc_in_range: no pde %d\n", pde);
 631                                 return EFAULT;
 632                         }
 633                         assert(!pt->pt_dir[pde]);
 634                         if((r=pt_ptalloc(pt, pde, flags)) != OK) {
 635                                 /* Couldn't do (complete) mapping.
 636                                  * Don't bother freeing any previously
 637                                  * allocated page tables, they're
 638                                  * still writable, don't point to nonsense,
 639                                  * and pt_ptalloc leaves the directory
 640                                  * and other data in a consistent state.
 641                                  */
 642                                 return r;
 643                         }
 644                         assert(pt->pt_pt[pde]);
 645                 }
 646                 assert(pt->pt_pt[pde]);
 647                 assert(pt->pt_dir[pde]);
 648                 assert(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT);
 649         }
 650
 651         return OK;
 652 }
 653
 654 static char *ptestr(u32_t pte)
 655 {
 656 #define FLAG(constant, name) {                                          \
 657         if(pte & (constant)) { strcat(str, name); strcat(str, " "); }   \
 658 }
 659
 660         static char str[30];
 661         if(!(pte & ARCH_VM_PTE_PRESENT)) {
 662                 return "not present";
 663         }
 664         str[0] = '\0';
 665 #if defined(__i386__)
 666         FLAG(ARCH_VM_PTE_RW, "W");
 667 #elif defined(__arm__)
 668         if(pte & ARCH_VM_PTE_RO) {
 669             strcat(str, "R ");
 670         } else {
 671             strcat(str, "W ");
 672         }
 673 #endif
 674         FLAG(ARCH_VM_PTE_USER, "U");
 675 #if defined(__i386__)
 676         FLAG(I386_VM_PWT, "PWT");
 677         FLAG(I386_VM_PCD, "PCD");
 678         FLAG(I386_VM_ACC, "ACC");
 679         FLAG(I386_VM_DIRTY, "DIRTY");
 680         FLAG(I386_VM_PS, "PS");
 681         FLAG(I386_VM_GLOBAL, "G");
 682         FLAG(I386_VM_PTAVAIL1, "AV1");
 683         FLAG(I386_VM_PTAVAIL2, "AV2");
 684         FLAG(I386_VM_PTAVAIL3, "AV3");
 685 #elif defined(__arm__)
 686         FLAG(ARM_VM_PTE_SUPER, "S");
 687         FLAG(ARM_VM_PTE_SHAREABLE, "SH");
 688         FLAG(ARM_VM_PTE_WB, "WB");
 689         FLAG(ARM_VM_PTE_WT, "WT");
 690 #endif
 691
 692         return str;
 693 }
 694
 695 /*===========================================================================*
 696  *                           pt_map_in_range                                 *
 697  *===========================================================================*/
 698 int pt_map_in_range(struct vmproc *src_vmp, struct vmproc *dst_vmp,
 699         vir_bytes start, vir_bytes end)
 700 {
 701 /* Transfer all the mappings from the pt of the source process to the pt of
 702  * the destination process in the range specified.
 703  */
 704         int pde, pte;
 705         vir_bytes viraddr;
 706         pt_t *pt, *dst_pt;
 707
 708         pt = &src_vmp->vm_pt;
 709         dst_pt = &dst_vmp->vm_pt;
 710
 711         end = end ? end : VM_DATATOP;
 712         assert(start % VM_PAGE_SIZE == 0);
 713         assert(end % VM_PAGE_SIZE == 0);
 714 #if defined(__i386__)
 715         assert(start <= end);
 716         assert(I386_VM_PDE(end) < ARCH_VM_DIR_ENTRIES);
 717 #elif defined(__arm__)
 718         assert(ARM_VM_PDE(start) >= 0 && start <= end);
 719         assert(ARM_VM_PDE(end) < ARCH_VM_DIR_ENTRIES);
 720 #endif
 721
 722 #if LU_DEBUG
 723         printf("VM: pt_map_in_range: src = %d, dst = %d\n",
 724                 src_vmp->vm_endpoint, dst_vmp->vm_endpoint);
 725         printf("VM: pt_map_in_range: transferring from 0x%08x (pde %d pte %d) to 0x%08x (pde %d pte %d)\n",
 726 #if defined(__i386__)
 727                 start, I386_VM_PDE(start), I386_VM_PTE(start),
 728                 end, I386_VM_PDE(end), I386_VM_PTE(end));
 729 #elif defined(__arm__)
 730                 start, ARM_VM_PDE(start), ARM_VM_PTE(start),
 731                 end, ARM_VM_PDE(end), ARM_VM_PTE(end));
 732 #endif
 733 #endif
 734
 735         /* Scan all page-table entries in the range. */
 736         for(viraddr = start; viraddr <= end; viraddr += VM_PAGE_SIZE) {
 737 #if defined(__i386__)
 738                 pde = I386_VM_PDE(viraddr);
 739 #elif defined(__arm__)
 740                 pde = ARM_VM_PDE(viraddr);
 741 #endif
 742                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 743                         if(viraddr == VM_DATATOP) break;
 744                         continue;
 745                 }
 746 #if defined(__i386__)
 747                 pte = I386_VM_PTE(viraddr);
 748 #elif defined(__arm__)
 749                 pte = ARM_VM_PTE(viraddr);
 750 #endif
 751                 if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) {
 752                         if(viraddr == VM_DATATOP) break;
 753                         continue;
 754                 }
 755
 756                 /* Transfer the mapping. */
 757                 dst_pt->pt_pt[pde][pte] = pt->pt_pt[pde][pte];
 758
 759                 if(viraddr == VM_DATATOP) break;
 760         }
 761
 762         return OK;
 763 }
 764
 765 /*===========================================================================*
 766  *                              pt_ptmap                                     *
 767  *===========================================================================*/
 768 int pt_ptmap(struct vmproc *src_vmp, struct vmproc *dst_vmp)
 769 {
 770 /* Transfer mappings to page dir and page tables from source process and
 771  * destination process. Make sure all the mappings are above the stack, not
 772  * to corrupt valid mappings in the data segment of the destination process.
 773  */
 774         int pde, r;
 775         phys_bytes physaddr;
 776         vir_bytes viraddr;
 777         pt_t *pt;
 778
 779         pt = &src_vmp->vm_pt;
 780
 781 #if LU_DEBUG
 782         printf("VM: pt_ptmap: src = %d, dst = %d\n",
 783                 src_vmp->vm_endpoint, dst_vmp->vm_endpoint);
 784 #endif
 785
 786         /* Transfer mapping to the page directory. */
 787         viraddr = (vir_bytes) pt->pt_dir;
 788         physaddr = pt->pt_dir_phys & ARCH_VM_ADDR_MASK;
 789 #if defined(__i386__)
 790         if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, VM_PAGE_SIZE,
 791                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW,
 792 #elif defined(__arm__)
 793         if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, ARCH_PAGEDIR_SIZE,
 794                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW |
 795                 ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE,
 796 #endif
 797                 WMF_OVERWRITE)) != OK) {
 798                 return r;
 799         }
 800 #if LU_DEBUG
 801         printf("VM: pt_ptmap: transferred mapping to page dir: 0x%08x (0x%08x)\n",
 802                 viraddr, physaddr);
 803 #endif
 804
 805         /* Scan all non-reserved page-directory entries. */
 806         for(pde=0; pde < ARCH_VM_DIR_ENTRIES; pde++) {
 807                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 808                         continue;
 809                 }
 810
 811                 /* Transfer mapping to the page table. */
 812                 viraddr = (vir_bytes) pt->pt_pt[pde];
 813 #if defined(__i386__)
 814                 physaddr = pt->pt_dir[pde] & ARCH_VM_ADDR_MASK;
 815 #elif defined(__arm__)
 816                 physaddr = pt->pt_dir[pde] & ARCH_VM_PDE_MASK;
 817 #endif
 818                 if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, VM_PAGE_SIZE,
 819                         ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW
 820 #ifdef __arm__
 821                         | ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW |
 822                         ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE
 823 #endif
 824                         ,
 825                         WMF_OVERWRITE)) != OK) {
 826                         return r;
 827                 }
 828         }
 829
 830         return OK;
 831 }
 832
 833 void pt_clearmapcache(void)
 834 {
 835         /* Make sure kernel will invalidate tlb when using current
 836          * pagetable (i.e. vm's) to make new mappings before new cr3
 837          * is loaded.
 838          */
 839         if(sys_vmctl(SELF, VMCTL_CLEARMAPCACHE, 0) != OK)
 840                 panic("VMCTL_CLEARMAPCACHE failed");
 841 }
 842
 843 /*===========================================================================*
 844  *                              pt_writemap                                  *
 845  *===========================================================================*/
 846 int pt_writemap(struct vmproc * vmp,
 847                         pt_t *pt,
 848                         vir_bytes v,
 849                         phys_bytes physaddr,
 850                         size_t bytes,
 851                         u32_t flags,
 852                         u32_t writemapflags)
 853 {
 854 /* Write mapping into page table. Allocate a new page table if necessary. */
 855 /* Page directory and table entries for this virtual address. */
 856         int p, pages;
 857         int verify = 0;
 858         int ret = OK;
 859
 860 #ifdef CONFIG_SMP
 861         int vminhibit_clear = 0;
 862         /* FIXME
 863          * don't do it everytime, stop the process only on the first change and
 864          * resume the execution on the last change. Do in a wrapper of this
 865          * function
 866          */
 867         if (vmp && vmp->vm_endpoint != NONE && vmp->vm_endpoint != VM_PROC_NR &&
 868                         !(vmp->vm_flags & VMF_EXITING)) {
 869                 sys_vmctl(vmp->vm_endpoint, VMCTL_VMINHIBIT_SET, 0);
 870                 vminhibit_clear = 1;
 871         }
 872 #endif
 873
 874         if(writemapflags & WMF_VERIFY)
 875                 verify = 1;
 876
 877         assert(!(bytes % VM_PAGE_SIZE));
 878         assert(!(flags & ~(PTF_ALLFLAGS)));
 879
 880         pages = bytes / VM_PAGE_SIZE;
 881
 882         /* MAP_NONE means to clear the mapping. It doesn't matter
 883          * what's actually written into the PTE if PRESENT
 884          * isn't on, so we can just write MAP_NONE into it.
 885          */
 886         assert(physaddr == MAP_NONE || (flags & ARCH_VM_PTE_PRESENT));
 887         assert(physaddr != MAP_NONE || !flags);
 888
 889         /* First make sure all the necessary page tables are allocated,
 890          * before we start writing in any of them, because it's a pain
 891          * to undo our work properly.
 892          */
 893         ret = pt_ptalloc_in_range(pt, v, v + VM_PAGE_SIZE*pages, flags, verify);
 894         if(ret != OK) {
 895                 printf("VM: writemap: pt_ptalloc_in_range failed\n");
 896                 goto resume_exit;
 897         }
 898
 899         /* Now write in them. */
 900         for(p = 0; p < pages; p++) {
 901                 u32_t entry;
 902 #if defined(__i386__)
 903                 int pde = I386_VM_PDE(v);
 904                 int pte = I386_VM_PTE(v);
 905 #elif defined(__arm__)
 906                 int pde = ARM_VM_PDE(v);
 907                 int pte = ARM_VM_PTE(v);
 908 #endif
 909
 910                 assert(!(v % VM_PAGE_SIZE));
 911                 assert(pte >= 0 && pte < ARCH_VM_PT_ENTRIES);
 912                 assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES);
 913
 914                 /* Page table has to be there. */
 915                 assert(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT);
 916
 917                 /* We do not expect it to be a bigpage. */
 918                 assert(!(pt->pt_dir[pde] & ARCH_VM_BIGPAGE));
 919
 920                 /* Make sure page directory entry for this page table
 921                  * is marked present and page table entry is available.
 922                  */
 923                 assert(pt->pt_pt[pde]);
 924
 925 #if SANITYCHECKS
 926                 /* We don't expect to overwrite a page. */
 927                 if(!(writemapflags & (WMF_OVERWRITE|WMF_VERIFY)))
 928                         assert(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT));
 929 #endif
 930                 if(writemapflags & (WMF_WRITEFLAGSONLY|WMF_FREE)) {
 931 #if defined(__i386__)
 932                         physaddr = pt->pt_pt[pde][pte] & ARCH_VM_ADDR_MASK;
 933 #elif defined(__arm__)
 934                         physaddr = pt->pt_pt[pde][pte] & ARM_VM_PTE_MASK;
 935 #endif
 936                 }
 937
 938                 if(writemapflags & WMF_FREE) {
 939                         free_mem(ABS2CLICK(physaddr), 1);
 940                 }
 941
 942                 /* Entry we will write. */
 943 #if defined(__i386__)
 944                 entry = (physaddr & ARCH_VM_ADDR_MASK) | flags;
 945 #elif defined(__arm__)
 946                 entry = (physaddr & ARM_VM_PTE_MASK) | flags;
 947 #endif
 948
 949                 if(verify) {
 950                         u32_t maskedentry;
 951                         maskedentry = pt->pt_pt[pde][pte];
 952 #if defined(__i386__)
 953                         maskedentry &= ~(I386_VM_ACC|I386_VM_DIRTY);
 954 #endif
 955                         /* Verify pagetable entry. */
 956 #if defined(__i386__)
 957                         if(entry & ARCH_VM_PTE_RW) {
 958                                 /* If we expect a writable page, allow a readonly page. */
 959                                 maskedentry |= ARCH_VM_PTE_RW;
 960                         }
 961 #elif defined(__arm__)
 962                         if(!(entry & ARCH_VM_PTE_RO)) {
 963                                 /* If we expect a writable page, allow a readonly page. */
 964                                 maskedentry &= ~ARCH_VM_PTE_RO;
 965                         }
 966 #endif
 967                         if(maskedentry != entry) {
 968                                 printf("pt_writemap: mismatch: ");
 969 #if defined(__i386__)
 970                                 if((entry & ARCH_VM_ADDR_MASK) !=
 971                                         (maskedentry & ARCH_VM_ADDR_MASK)) {
 972 #elif defined(__arm__)
 973                                 if((entry & ARM_VM_PTE_MASK) !=
 974                                         (maskedentry & ARM_VM_PTE_MASK)) {
 975 #endif
 976                                         printf("pt_writemap: physaddr mismatch (0x%lx, 0x%lx); ",
 977                                                 (long)entry, (long)maskedentry);
 978                                 } else printf("phys ok; ");
 979                                 printf(" flags: found %s; ",
 980                                         ptestr(pt->pt_pt[pde][pte]));
 981                                 printf(" masked %s; ",
 982                                         ptestr(maskedentry));
 983                                 printf(" expected %s\n", ptestr(entry));
 984                                 printf("found 0x%x, wanted 0x%x\n",
 985                                         pt->pt_pt[pde][pte], entry);
 986                                 ret = EFAULT;
 987                                 goto resume_exit;
 988                         }
 989                 } else {
 990                         /* Write pagetable entry. */
 991                         pt->pt_pt[pde][pte] = entry;
 992                 }
 993
 994                 physaddr += VM_PAGE_SIZE;
 995                 v += VM_PAGE_SIZE;
 996         }
 997
 998 resume_exit:
 999
1000 #ifdef CONFIG_SMP
1001         if (vminhibit_clear) {
1002                 assert(vmp && vmp->vm_endpoint != NONE && vmp->vm_endpoint != VM_PROC_NR &&
1003                         !(vmp->vm_flags & VMF_EXITING));
1004                 sys_vmctl(vmp->vm_endpoint, VMCTL_VMINHIBIT_CLEAR, 0);
1005         }
1006 #endif
1007
1008         return ret;
1009 }
1010
1011 /*===========================================================================*
1012  *                              pt_checkrange                                *
1013  *===========================================================================*/
1014 int pt_checkrange(pt_t *pt, vir_bytes v,  size_t bytes,
1015         int write)
1016 {
1017         int p, pages;
1018
1019         assert(!(bytes % VM_PAGE_SIZE));
1020
1021         pages = bytes / VM_PAGE_SIZE;
1022
1023         for(p = 0; p < pages; p++) {
1024 #if defined(__i386__)
1025                 int pde = I386_VM_PDE(v);
1026                 int pte = I386_VM_PTE(v);
1027 #elif defined(__arm__)
1028                 int pde = ARM_VM_PDE(v);
1029                 int pte = ARM_VM_PTE(v);
1030 #endif
1031
1032                 assert(!(v % VM_PAGE_SIZE));
1033                 assert(pte >= 0 && pte < ARCH_VM_PT_ENTRIES);
1034                 assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES);
1035
1036                 /* Page table has to be there. */
1037                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT))
1038                         return EFAULT;
1039
1040                 /* Make sure page directory entry for this page table
1041                  * is marked present and page table entry is available.
1042                  */
1043                 assert((pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT) && pt->pt_pt[pde]);
1044
1045                 if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) {
1046                         return EFAULT;
1047                 }
1048
1049 #if defined(__i386__)
1050                 if(write && !(pt->pt_pt[pde][pte] & ARCH_VM_PTE_RW)) {
1051 #elif defined(__arm__)
1052                 if(write && (pt->pt_pt[pde][pte] & ARCH_VM_PTE_RO)) {
1053 #endif
1054                         return EFAULT;
1055                 }
1056
1057                 v += VM_PAGE_SIZE;
1058         }
1059
1060         return OK;
1061 }
1062
1063 /*===========================================================================*
1064  *                              pt_new                                       *
1065  *===========================================================================*/
1066 int pt_new(pt_t *pt)
1067 {
1068 /* Allocate a pagetable root. Allocate a page-aligned page directory
1069  * and set them to 0 (indicating no page tables are allocated). Lookup
1070  * its physical address as we'll need that in the future. Verify it's
1071  * page-aligned.
1072  */
1073         int i, r;
1074
1075         /* Don't ever re-allocate/re-move a certain process slot's
1076          * page directory once it's been created. This is a fraction
1077          * faster, but also avoids having to invalidate the page
1078          * mappings from in-kernel page tables pointing to
1079          * the page directories (the page_directories data).
1080          */
1081         if(!pt->pt_dir &&
1082           !(pt->pt_dir = vm_allocpages((phys_bytes *)&pt->pt_dir_phys,
1083                 VMP_PAGEDIR, ARCH_PAGEDIR_SIZE/VM_PAGE_SIZE))) {
1084                 return ENOMEM;
1085         }
1086
1087         assert(!((u32_t)pt->pt_dir_phys % ARCH_PAGEDIR_SIZE));
1088
1089         for(i = 0; i < ARCH_VM_DIR_ENTRIES; i++) {
1090                 pt->pt_dir[i] = 0; /* invalid entry (PRESENT bit = 0) */
1091                 pt->pt_pt[i] = NULL;
1092         }
1093
1094         /* Where to start looking for free virtual address space? */
1095         pt->pt_virtop = 0;
1096
1097         /* Map in kernel. */
1098         if((r=pt_mapkernel(pt)) != OK)
1099                 return r;
1100
1101         return OK;
1102 }
1103
1104 static int freepde(void)
1105 {
1106         int p = kernel_boot_info.freepde_start++;
1107         assert(kernel_boot_info.freepde_start < ARCH_VM_DIR_ENTRIES);
1108         return p;
1109 }
1110
1111 /*===========================================================================*
1112  *                              pt_init                                      *
1113  *===========================================================================*/
1114 void pt_init(void)
1115 {
1116         pt_t *newpt;
1117         int s, r, p;
1118         int global_bit_ok = 0;
1119         vir_bytes sparepages_mem;
1120 #if defined(__arm__)
1121         vir_bytes sparepagedirs_mem;
1122 #endif
1123         static u32_t currentpagedir[ARCH_VM_DIR_ENTRIES];
1124         int m = kernel_boot_info.kern_mod;
1125 #if defined(__i386__)
1126         u32_t mypdbr; /* Page Directory Base Register (cr3) value */
1127 #elif defined(__arm__)
1128         u32_t myttbr;
1129 #endif
1130
1131         /* Find what the physical location of the kernel is. */
1132         assert(m >= 0);
1133         assert(m < kernel_boot_info.mods_with_kernel);
1134         assert(kernel_boot_info.mods_with_kernel < MULTIBOOT_MAX_MODS);
1135         kern_mb_mod = &kernel_boot_info.module_list[m];
1136         kern_size = kern_mb_mod->mod_end - kern_mb_mod->mod_start;
1137         assert(!(kern_mb_mod->mod_start % ARCH_BIG_PAGE_SIZE));
1138         assert(!(kernel_boot_info.vir_kern_start % ARCH_BIG_PAGE_SIZE));
1139         kern_start_pde = kernel_boot_info.vir_kern_start / ARCH_BIG_PAGE_SIZE;
1140
1141         /* Get ourselves spare pages. */
1142         sparepages_mem = (vir_bytes) static_sparepages;
1143         assert(!(sparepages_mem % VM_PAGE_SIZE));
1144
1145 #if defined(__arm__)
1146         /* Get ourselves spare pagedirs. */
1147         sparepagedirs_mem = (vir_bytes) static_sparepagedirs;
1148         assert(!(sparepagedirs_mem % ARCH_PAGEDIR_SIZE));
1149 #endif
1150
1151         /* Spare pages are used to allocate memory before VM has its own page
1152          * table that things (i.e. arbitrary physical memory) can be mapped into.
1153          * We get it by pre-allocating it in our bss (allocated and mapped in by
1154          * the kernel) in static_sparepages. We also need the physical addresses
1155          * though; we look them up now so they are ready for use.
1156          */
1157 #if defined(__arm__)
1158         missing_sparedirs = 0;
1159         assert(STATIC_SPAREPAGEDIRS < SPAREPAGEDIRS);
1160         for(s = 0; s < SPAREPAGEDIRS; s++) {
1161                 vir_bytes v = (sparepagedirs_mem + s*ARCH_PAGEDIR_SIZE);;
1162                 phys_bytes ph;
1163                 if((r=sys_umap(SELF, VM_D, (vir_bytes) v,
1164                         ARCH_PAGEDIR_SIZE, &ph)) != OK)
1165                                 panic("pt_init: sys_umap failed: %d", r);
1166                 if(s >= STATIC_SPAREPAGEDIRS) {
1167                         sparepagedirs[s].pagedir = NULL;
1168                         missing_sparedirs++;
1169                         continue;
1170                 }
1171                 sparepagedirs[s].pagedir = (void *) v;
1172                 sparepagedirs[s].phys = ph;
1173         }
1174 #endif
1175
1176         missing_spares = 0;
1177         assert(STATIC_SPAREPAGES < SPAREPAGES);
1178         for(s = 0; s < SPAREPAGES; s++) {
1179                 vir_bytes v = (sparepages_mem + s*VM_PAGE_SIZE);;
1180                 phys_bytes ph;
1181                 if((r=sys_umap(SELF, VM_D, (vir_bytes) v,
1182                         VM_PAGE_SIZE*SPAREPAGES, &ph)) != OK)
1183                                 panic("pt_init: sys_umap failed: %d", r);
1184                 if(s >= STATIC_SPAREPAGES) {
1185                         sparepages[s].page = NULL;
1186                         missing_spares++;
1187                         continue;
1188                 }
1189                 sparepages[s].page = (void *) v;
1190                 sparepages[s].phys = ph;
1191         }
1192
1193 #if defined(__i386__)
1194         /* global bit and 4MB pages available? */
1195         global_bit_ok = _cpufeature(_CPUF_I386_PGE);
1196         bigpage_ok = _cpufeature(_CPUF_I386_PSE);
1197
1198         /* Set bit for PTE's and PDE's if available. */
1199         if(global_bit_ok)
1200                 global_bit = I386_VM_GLOBAL;
1201 #endif
1202
1203         /* Allocate us a page table in which to remember page directory
1204          * pointers.
1205          */
1206         if(!(page_directories = vm_allocpage(&page_directories_phys,
1207                 VMP_PAGETABLE)))
1208                 panic("no virt addr for vm mappings");
1209
1210         memset(page_directories, 0, VM_PAGE_SIZE);
1211
1212         /* Now reserve another pde for kernel's own mappings. */
1213         {
1214                 int kernmap_pde;
1215                 phys_bytes addr, len;
1216                 int flags, index = 0;
1217                 u32_t offset = 0;
1218
1219                 kernmap_pde = freepde();
1220                 offset = kernmap_pde * ARCH_BIG_PAGE_SIZE;
1221
1222                 while(sys_vmctl_get_mapping(index, &addr, &len,
1223                         &flags) == OK)  {
1224                         int usedpde;
1225                         vir_bytes vir;
1226                         if(index >= MAX_KERNMAPPINGS)
1227                                 panic("VM: too many kernel mappings: %d", index);
1228                         kern_mappings[index].phys_addr = addr;
1229                         kern_mappings[index].len = len;
1230                         kern_mappings[index].flags = flags;
1231                         kern_mappings[index].vir_addr = offset;
1232                         kern_mappings[index].flags =
1233                                 ARCH_VM_PTE_PRESENT;
1234                         if(flags & VMMF_UNCACHED)
1235 #if defined(__i386__)
1236                                 kern_mappings[index].flags |= PTF_NOCACHE;
1237 #elif defined(__arm__)
1238                                 kern_mappings[index].flags |= ARM_VM_PTE_DEVICE;
1239                         else
1240                                 kern_mappings[index].flags |=
1241                                     ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE;
1242 #endif
1243                         if(flags & VMMF_USER)
1244                                 kern_mappings[index].flags |= ARCH_VM_PTE_USER;
1245 #if defined(__arm__)
1246                         else
1247                                 kern_mappings[index].flags |= ARM_VM_PTE_SUPER;
1248 #endif
1249                         if(flags & VMMF_WRITE)
1250                                 kern_mappings[index].flags |= ARCH_VM_PTE_RW;
1251 #if defined(__i386__)
1252                         if(flags & VMMF_GLO)
1253                                 kern_mappings[index].flags |= I386_VM_GLOBAL;
1254 #elif defined(__arm__)
1255                         else
1256                                 kern_mappings[index].flags |= ARCH_VM_PTE_RO;
1257 #endif
1258                         if(addr % VM_PAGE_SIZE)
1259                                 panic("VM: addr unaligned: %d", addr);
1260                         if(len % VM_PAGE_SIZE)
1261                                 panic("VM: len unaligned: %d", len);
1262                         vir = offset;
1263                         if(sys_vmctl_reply_mapping(index, vir) != OK)
1264                                 panic("VM: reply failed");
1265                         offset += len;
1266                         index++;
1267                         kernmappings++;
1268
1269 #if defined(__i386__)
1270                         usedpde = I386_VM_PDE(offset);
1271 #elif defined(__arm__)
1272                         usedpde = ARM_VM_PDE(offset);
1273 #endif
1274                         while(usedpde > kernmap_pde) {
1275                                 int newpde = freepde();
1276                                 assert(newpde == kernmap_pde+1);
1277                                 kernmap_pde = newpde;
1278                         }
1279                 }
1280         }
1281
1282         /* Find a PDE below processes available for mapping in the
1283          * page directories.
1284          */
1285         pagedir_pde = freepde();
1286 #if defined(__i386__)
1287         pagedir_pde_val = (page_directories_phys & ARCH_VM_ADDR_MASK) |
1288                         ARCH_VM_PDE_PRESENT | ARCH_VM_PTE_RW;
1289 #elif defined(__arm__)
1290         pagedir_pde_val = (page_directories_phys & ARCH_VM_PDE_MASK) |
1291                         ARCH_VM_PDE_PRESENT | ARM_VM_PDE_DOMAIN;
1292 #endif
1293
1294         /* Allright. Now. We have to make our own page directory and page tables,
1295          * that the kernel has already set up, accessible to us. It's easier to
1296          * understand if we just copy all the required pages (i.e. page directory
1297          * and page tables), and set up the pointers as if VM had done it itself.
1298          *
1299          * This allocation will happen without using any page table, and just
1300          * uses spare pages.
1301          */
1302         newpt = &vmprocess->vm_pt;
1303         if(pt_new(newpt) != OK)
1304                 panic("vm pt_new failed");
1305
1306         /* Get our current pagedir so we can see it. */
1307 #if defined(__i386__)
1308         if(sys_vmctl_get_pdbr(SELF, &mypdbr) != OK)
1309 #elif defined(__arm__)
1310         if(sys_vmctl_get_pdbr(SELF, &myttbr) != OK)
1311 #endif
1312                 panic("VM: sys_vmctl_get_pdbr failed");
1313 #if defined(__i386__)
1314         if(sys_vircopy(NONE, mypdbr, SELF,
1315                 (vir_bytes) currentpagedir, VM_PAGE_SIZE) != OK)
1316 #elif defined(__arm__)
1317         if(sys_vircopy(NONE, myttbr, SELF,
1318                 (vir_bytes) currentpagedir, ARCH_PAGEDIR_SIZE) != OK)
1319 #endif
1320                 panic("VM: sys_vircopy failed");
1321
1322         /* We have mapped in kernel ourselves; now copy mappings for VM
1323          * that kernel made, including allocations for BSS. Skip identity
1324          * mapping bits; just map in VM.
1325          */
1326         for(p = 0; p < ARCH_VM_DIR_ENTRIES; p++) {
1327                 u32_t entry = currentpagedir[p];
1328                 phys_bytes ptaddr_kern, ptaddr_us;
1329
1330                 /* BIGPAGEs are kernel mapping (do ourselves) or boot
1331                  * identity mapping (don't want).
1332                  */
1333                 if(!(entry & ARCH_VM_PDE_PRESENT)) continue;
1334                 if((entry & ARCH_VM_BIGPAGE)) continue;
1335
1336                 if(pt_ptalloc(newpt, p, 0) != OK)
1337                         panic("pt_ptalloc failed");
1338                 assert(newpt->pt_dir[p] & ARCH_VM_PDE_PRESENT);
1339
1340 #if defined(__i386__)
1341                 ptaddr_kern = entry & ARCH_VM_ADDR_MASK;
1342                 ptaddr_us = newpt->pt_dir[p] & ARCH_VM_ADDR_MASK;
1343 #elif defined(__arm__)
1344                 ptaddr_kern = entry & ARCH_VM_PDE_MASK;
1345                 ptaddr_us = newpt->pt_dir[p] & ARCH_VM_PDE_MASK;
1346 #endif
1347
1348                 /* Copy kernel-initialized pagetable contents into our
1349                  * normally accessible pagetable.
1350                  */
1351                 if(sys_abscopy(ptaddr_kern, ptaddr_us, VM_PAGE_SIZE) != OK)
1352                         panic("pt_init: abscopy failed");
1353         }
1354
1355         /* Inform kernel vm has a newly built page table. */
1356         assert(vmproc[VM_PROC_NR].vm_endpoint == VM_PROC_NR);
1357         pt_bind(newpt, &vmproc[VM_PROC_NR]);
1358
1359         pt_init_done = 1;
1360
1361         vm_checkspares();
1362
1363         /* All OK. */
1364         return;
1365 }
1366
1367 /*===========================================================================*
1368  *                              pt_bind                                      *
1369  *===========================================================================*/
1370 int pt_bind(pt_t *pt, struct vmproc *who)
1371 {
1372         int slot;
1373         u32_t phys;
1374         void *pdes;
1375         int pages_per_pagedir = ARCH_PAGEDIR_SIZE/VM_PAGE_SIZE;
1376
1377         /* Basic sanity checks. */
1378         assert(who);
1379         assert(who->vm_flags & VMF_INUSE);
1380         assert(pt);
1381
1382         assert(pagedir_pde >= 0);
1383
1384         slot = who->vm_slot;
1385         assert(slot >= 0);
1386         assert(slot < ELEMENTS(vmproc));
1387         assert(slot < ARCH_VM_PT_ENTRIES / pages_per_pagedir);
1388
1389 #if defined(__i386__)
1390         phys = pt->pt_dir_phys & ARCH_VM_ADDR_MASK;
1391 #elif defined(__arm__)
1392         phys = pt->pt_dir_phys & ARM_VM_PTE_MASK;
1393 #endif
1394         assert(pt->pt_dir_phys == phys);
1395         assert(!(pt->pt_dir_phys % ARCH_PAGEDIR_SIZE));
1396
1397         /* Update "page directory pagetable." */
1398 #if defined(__i386__)
1399         page_directories[slot] = phys | ARCH_VM_PDE_PRESENT|ARCH_VM_PTE_RW;
1400 #elif defined(__arm__)
1401         {
1402         int i;
1403         for (i = 0; i < pages_per_pagedir; i++)
1404             page_directories[slot*pages_per_pagedir+i] =
1405                 (phys+i*VM_PAGE_SIZE) |
1406                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_RW |
1407                 ARCH_VM_PTE_USER;
1408         }
1409 #endif
1410
1411         /* This is where the PDE's will be visible to the kernel
1412          * in its address space.
1413          */
1414         pdes = (void *) (pagedir_pde*ARCH_BIG_PAGE_SIZE +
1415 #if defined(__i386__)
1416                         slot * VM_PAGE_SIZE);
1417 #elif defined(__arm__)
1418                         slot * ARCH_PAGEDIR_SIZE);
1419 #endif
1420
1421 #if 0
1422         printf("VM: slot %d endpoint %d has pde val 0x%lx at kernel address 0x%lx\n",
1423                 slot, who->vm_endpoint, page_directories[slot], pdes);
1424 #endif
1425         /* Tell kernel about new page table root. */
1426         return sys_vmctl_set_addrspace(who->vm_endpoint, pt->pt_dir_phys, pdes);
1427 }
1428
1429 /*===========================================================================*
1430  *                              pt_free                                      *
1431  *===========================================================================*/
1432 void pt_free(pt_t *pt)
1433 {
1434 /* Free memory associated with this pagetable. */
1435         int i;
1436
1437         for(i = 0; i < ARCH_VM_DIR_ENTRIES; i++)
1438                 if(pt->pt_pt[i])
1439                         vm_freepages((vir_bytes) pt->pt_pt[i], 1);
1440
1441         return;
1442 }
1443
1444 /*===========================================================================*
1445  *                              pt_mapkernel                                 *
1446  *===========================================================================*/
1447 int pt_mapkernel(pt_t *pt)
1448 {
1449         int i;
1450         int kern_pde = kern_start_pde;
1451         phys_bytes addr, mapped = 0;
1452
1453         /* Any page table needs to map in the kernel address space. */
1454         assert(bigpage_ok);
1455         assert(pagedir_pde >= 0);
1456         assert(kern_pde >= 0);
1457
1458         /* pt_init() has made sure this is ok. */
1459         addr = kern_mb_mod->mod_start;
1460
1461         /* Actually mapping in kernel */
1462         while(mapped < kern_size) {
1463 #if defined(__i386__)
1464                 pt->pt_dir[kern_pde] = addr | ARCH_VM_PDE_PRESENT |
1465                         ARCH_VM_BIGPAGE | ARCH_VM_PTE_RW | global_bit;
1466 #elif defined(__arm__)
1467                 pt->pt_dir[kern_pde] = (addr & ARCH_VM_PDE_MASK) |
1468                         ARM_VM_SECTION |
1469                         ARM_VM_SECTION_DOMAIN | ARM_VM_SECTION_WB |
1470                         ARM_VM_SECTION_SHAREABLE | ARM_VM_SECTION_SUPER;
1471 #endif
1472                 kern_pde++;
1473                 mapped += ARCH_BIG_PAGE_SIZE;
1474                 addr += ARCH_BIG_PAGE_SIZE;
1475         }
1476
1477         /* Kernel also wants to know about all page directories. */
1478         assert(pagedir_pde > kern_pde);
1479         pt->pt_dir[pagedir_pde] = pagedir_pde_val;
1480
1481         /* Kernel also wants various mappings of its own. */
1482         for(i = 0; i < kernmappings; i++) {
1483                 int r;
1484                 if((r=pt_writemap(NULL, pt,
1485                         kern_mappings[i].vir_addr,
1486                         kern_mappings[i].phys_addr,
1487                         kern_mappings[i].len,
1488                         kern_mappings[i].flags, 0)) != OK) {
1489                         return r;
1490                 }
1491
1492 #if defined(__arm__)
1493                 if(kern_mappings[i].phys_addr == 0x48000000) {
1494                         if((r=pt_writemap(NULL, pt,
1495                                 kern_mappings[i].phys_addr,
1496                                 kern_mappings[i].phys_addr,
1497                                 kern_mappings[i].len,
1498                                 kern_mappings[i].flags, 0)) != OK) {
1499                                 return r;
1500                         }
1501                 }
1502 #endif
1503         }
1504
1505         return OK;
1506 }
1507
1508 /*===========================================================================*
1509  *                              pt_cycle                                     *
1510  *===========================================================================*/
1511 void pt_cycle(void)
1512 {
1513         vm_checkspares();
1514 #if defined(__arm__)
1515         vm_checksparedirs();
1516 #endif
1517 }
1518
1519 int get_vm_self_pages(void) { return vm_self_pages; }