servers/vm/arch/i386/pagetable.c

   1
   2 #define _SYSTEM 1
   3 #define _POSIX_SOURCE 1
   4
   5 #include <minix/callnr.h>
   6 #include <minix/com.h>
   7 #include <minix/config.h>
   8 #include <minix/const.h>
   9 #include <minix/ds.h>
  10 #include <minix/endpoint.h>
  11 #include <minix/keymap.h>
  12 #include <minix/minlib.h>
  13 #include <minix/type.h>
  14 #include <minix/ipc.h>
  15 #include <minix/sysutil.h>
  16 #include <minix/syslib.h>
  17 #include <minix/safecopies.h>
  18 #include <minix/cpufeature.h>
  19 #include <minix/bitmap.h>
  20 #include <minix/debug.h>
  21
  22 #include <errno.h>
  23 #include <stdlib.h>
  24 #include <assert.h>
  25 #include <string.h>
  26 #include <env.h>
  27 #include <stdio.h>
  28 #include <fcntl.h>
  29 #include <stdlib.h>
  30
  31 #include "proto.h"
  32 #include "glo.h"
  33 #include "util.h"
  34 #include "vm.h"
  35 #include "sanitycheck.h"
  36
  37 static int vm_self_pages;
  38
  39 /* PDE used to map in kernel, kernel physical address. */
  40 static int pagedir_pde = -1;
  41 static u32_t global_bit = 0, pagedir_pde_val;
  42
  43 static multiboot_module_t *kern_mb_mod = NULL;
  44 static size_t kern_size = 0;
  45 static int kern_start_pde = -1;
  46
  47 /* big page size available in hardware? */
  48 static int bigpage_ok = 1;
  49
  50 /* Our process table entry. */
  51 struct vmproc *vmprocess = &vmproc[VM_PROC_NR];
  52
  53 /* Spare memory, ready to go after initialization, to avoid a
  54  * circular dependency on allocating memory and writing it into VM's
  55  * page table.
  56  */
  57 #if SANITYCHECKS
  58 #define SPAREPAGES 100
  59 #define STATIC_SPAREPAGES 90
  60 #else
  61 #ifdef __arm__
  62 # define SPAREPAGES 80
  63 # define STATIC_SPAREPAGES 75
  64 #else
  65 # define SPAREPAGES 20
  66 # define STATIC_SPAREPAGES 15
  67 #endif /* __arm__ */
  68 #endif
  69
  70 #define SPAREPAGEDIRS 11
  71 #define STATIC_SPAREPAGEDIRS 10
  72
  73 int missing_sparedirs = SPAREPAGEDIRS;
  74 static struct {
  75         void *pagedir;
  76         phys_bytes phys;
  77 } sparepagedirs[SPAREPAGEDIRS];
  78
  79 int missing_spares = SPAREPAGES;
  80 static struct {
  81         void *page;
  82         phys_bytes phys;
  83 } sparepages[SPAREPAGES];
  84
  85 extern char _end;
  86 #define is_staticaddr(v) ((vir_bytes) (v) < (vir_bytes) &_end)
  87
  88 #define MAX_KERNMAPPINGS 10
  89 static struct {
  90         phys_bytes      phys_addr;      /* Physical addr. */
  91         phys_bytes      len;            /* Length in bytes. */
  92         vir_bytes       vir_addr;       /* Offset in page table. */
  93         int             flags;
  94 } kern_mappings[MAX_KERNMAPPINGS];
  95 int kernmappings = 0;
  96
  97 /* Clicks must be pages, as
  98  *  - they must be page aligned to map them
  99  *  - they must be a multiple of the page size
 100  *  - it's inconvenient to have them bigger than pages, because we often want
 101  *    just one page
 102  * May as well require them to be equal then.
 103  */
 104 #if CLICK_SIZE != VM_PAGE_SIZE
 105 #error CLICK_SIZE must be page size.
 106 #endif
 107
 108 /* Page table that contains pointers to all page directories. */
 109 phys_bytes page_directories_phys;
 110 u32_t *page_directories = NULL;
 111
 112 static char static_sparepages[VM_PAGE_SIZE*STATIC_SPAREPAGES]
 113         __aligned(VM_PAGE_SIZE);
 114
 115 #if defined(__arm__)
 116 static char static_sparepagedirs[ARCH_PAGEDIR_SIZE*STATIC_SPAREPAGEDIRS + ARCH_PAGEDIR_SIZE] __aligned(ARCH_PAGEDIR_SIZE);
 117 #endif
 118
 119 #if SANITYCHECKS
 120 /*===========================================================================*
 121  *                              pt_sanitycheck                               *
 122  *===========================================================================*/
 123 void pt_sanitycheck(pt_t *pt, char *file, int line)
 124 {
 125 /* Basic pt sanity check. */
 126         int slot;
 127
 128         MYASSERT(pt);
 129         MYASSERT(pt->pt_dir);
 130         MYASSERT(pt->pt_dir_phys);
 131
 132         for(slot = 0; slot < ELEMENTS(vmproc); slot++) {
 133                 if(pt == &vmproc[slot].vm_pt)
 134                         break;
 135         }
 136
 137         if(slot >= ELEMENTS(vmproc)) {
 138                 panic("pt_sanitycheck: passed pt not in any proc");
 139         }
 140
 141         MYASSERT(usedpages_add(pt->pt_dir_phys, VM_PAGE_SIZE) == OK);
 142 }
 143 #endif
 144
 145 /*===========================================================================*
 146  *                              findhole                                     *
 147  *===========================================================================*/
 148 static u32_t findhole(int pages)
 149 {
 150 /* Find a space in the virtual address space of VM. */
 151         u32_t curv;
 152         int pde = 0, try_restart;
 153         static u32_t lastv = 0;
 154         pt_t *pt = &vmprocess->vm_pt;
 155         vir_bytes vmin, vmax;
 156 #if defined(__arm__)
 157         u32_t holev;
 158 #endif
 159
 160         vmin = (vir_bytes) (&_end); /* marks end of VM BSS */
 161         vmin += 1024*1024*1024; /* reserve 1GB virtual address space for VM heap */
 162         vmin &= ARCH_VM_ADDR_MASK;
 163         vmax = VM_STACKTOP;
 164
 165         /* Input sanity check. */
 166         assert(vmin + VM_PAGE_SIZE >= vmin);
 167         assert(vmax >= vmin + VM_PAGE_SIZE);
 168         assert((vmin % VM_PAGE_SIZE) == 0);
 169         assert((vmax % VM_PAGE_SIZE) == 0);
 170 #if defined(__arm__)
 171         assert(pages > 0);
 172 #endif
 173
 174 #if SANITYCHECKS
 175         curv = ((u32_t) random()) % ((vmax - vmin)/VM_PAGE_SIZE);
 176         curv *= VM_PAGE_SIZE;
 177         curv += vmin;
 178 #else
 179         curv = lastv;
 180         if(curv < vmin || curv >= vmax)
 181                 curv = vmin;
 182 #endif
 183         try_restart = 1;
 184
 185         /* Start looking for a free page starting at vmin. */
 186         while(curv < vmax) {
 187                 int pte;
 188 #if defined(__arm__)
 189                 int i, nohole;
 190 #endif
 191
 192                 assert(curv >= vmin);
 193                 assert(curv < vmax);
 194
 195 #if defined(__i386__)
 196                 pde = I386_VM_PDE(curv);
 197                 pte = I386_VM_PTE(curv);
 198 #elif defined(__arm__)
 199                 holev = curv; /* the candidate hole */
 200                 nohole = 0;
 201                 for (i = 0; i < pages && !nohole; ++i) {
 202                     if(curv >= vmax) {
 203                         break;
 204                     }
 205 #endif
 206
 207 #if defined(__i386__)
 208                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT) ||
 209                    !(pt->pt_pt[pde][pte] & ARCH_VM_PAGE_PRESENT)) {
 210 #elif defined(__arm__)
 211                     pde = ARM_VM_PDE(curv);
 212                     pte = ARM_VM_PTE(curv);
 213
 214                     /* if page present, no hole */
 215                     if((pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT) &&
 216                        (pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT))
 217                         nohole = 1;
 218
 219                     /* if not contiguous, no hole */
 220                     if (curv != holev + i * VM_PAGE_SIZE)
 221                         nohole = 1;
 222
 223                     curv+=VM_PAGE_SIZE;
 224                 }
 225
 226                 /* there's a large enough hole */
 227                 if (!nohole && i == pages) {
 228 #endif
 229                         lastv = curv;
 230 #if defined(__i386__)
 231                         return curv;
 232 #elif defined(__arm__)
 233                         return holev;
 234 #endif
 235                 }
 236
 237 #if defined(__i386__)
 238                 curv+=VM_PAGE_SIZE;
 239
 240 #elif defined(__arm__)
 241                 /* Reset curv */
 242 #endif
 243                 if(curv >= vmax && try_restart) {
 244                         curv = vmin;
 245                         try_restart = 0;
 246                 }
 247         }
 248
 249         printf("VM: out of virtual address space in vm\n");
 250
 251         return NO_MEM;
 252 }
 253
 254 /*===========================================================================*
 255  *                              vm_freepages                                 *
 256  *===========================================================================*/
 257 void vm_freepages(vir_bytes vir, int pages)
 258 {
 259         assert(!(vir % VM_PAGE_SIZE));
 260
 261         if(is_staticaddr(vir)) {
 262                 printf("VM: not freeing static page\n");
 263                 return;
 264         }
 265
 266         if(pt_writemap(vmprocess, &vmprocess->vm_pt, vir,
 267                 MAP_NONE, pages*VM_PAGE_SIZE, 0,
 268                 WMF_OVERWRITE | WMF_FREE) != OK)
 269                 panic("vm_freepages: pt_writemap failed");
 270
 271         vm_self_pages--;
 272
 273 #if SANITYCHECKS
 274         /* If SANITYCHECKS are on, flush tlb so accessing freed pages is
 275          * always trapped, also if not in tlb.
 276          */
 277         if((sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 278                 panic("VMCTL_FLUSHTLB failed");
 279         }
 280 #endif
 281 }
 282
 283 /*===========================================================================*
 284  *                              vm_getsparepage                              *
 285  *===========================================================================*/
 286 static void *vm_getsparepage(phys_bytes *phys)
 287 {
 288         int s;
 289         assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
 290         for(s = 0; s < SPAREPAGES; s++) {
 291                 if(sparepages[s].page) {
 292                         void *sp;
 293                         sp = sparepages[s].page;
 294                         *phys = sparepages[s].phys;
 295                         sparepages[s].page = NULL;
 296                         missing_spares++;
 297                         assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
 298                         return sp;
 299                 }
 300         }
 301         printf("no spare found, %d missing\n", missing_spares);
 302         return NULL;
 303 }
 304
 305 /*===========================================================================*
 306  *                              vm_getsparepagedir                           *
 307  *===========================================================================*/
 308 static void *vm_getsparepagedir(phys_bytes *phys)
 309 {
 310         int s;
 311         assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
 312         for(s = 0; s < SPAREPAGEDIRS; s++) {
 313                 if(sparepagedirs[s].pagedir) {
 314                         void *sp;
 315                         sp = sparepagedirs[s].pagedir;
 316                         *phys = sparepagedirs[s].phys;
 317                         sparepagedirs[s].pagedir = NULL;
 318                         missing_sparedirs++;
 319                         assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
 320                         return sp;
 321                 }
 322         }
 323         return NULL;
 324 }
 325
 326 /*===========================================================================*
 327  *                              vm_checkspares                               *
 328  *===========================================================================*/
 329 static void *vm_checkspares(void)
 330 {
 331         int s, n = 0;
 332         static int total = 0, worst = 0;
 333         assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
 334         for(s = 0; s < SPAREPAGES && missing_spares > 0; s++) {
 335             if(!sparepages[s].page) {
 336                 n++;
 337                 if((sparepages[s].page = vm_allocpage(&sparepages[s].phys,
 338                         VMP_SPARE))) {
 339                         missing_spares--;
 340                         assert(missing_spares >= 0);
 341                         assert(missing_spares <= SPAREPAGES);
 342                 } else {
 343                         printf("VM: warning: couldn't get new spare page\n");
 344                 }
 345            }
 346         }
 347         if(worst < n) worst = n;
 348         total += n;
 349
 350         return NULL;
 351 }
 352
 353 #if defined(__arm__)
 354 /*===========================================================================*
 355  *                              vm_checksparedirs                            *
 356  *===========================================================================*/
 357 static void *vm_checksparedirs(void)
 358 {
 359         int s, n = 0;
 360         static int total = 0, worst = 0;
 361         assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
 362         for(s = 0; s < SPAREPAGEDIRS && missing_sparedirs > 0; s++)
 363             if(!sparepagedirs[s].pagedir) {
 364                 n++;
 365                 if((sparepagedirs[s].pagedir = vm_allocpage(&sparepagedirs[s].phys,
 366                         VMP_SPARE))) {
 367                         missing_sparedirs--;
 368                         assert(missing_sparedirs >= 0);
 369                         assert(missing_sparedirs <= SPAREPAGEDIRS);
 370                 } else {
 371                         printf("VM: warning: couldn't get new spare pagedir\n");
 372                 }
 373         }
 374         if(worst < n) worst = n;
 375         total += n;
 376
 377         return NULL;
 378 }
 379 #endif
 380
 381 static int pt_init_done;
 382
 383 /*===========================================================================*
 384  *                              vm_allocpage                                 *
 385  *===========================================================================*/
 386 void *vm_allocpages(phys_bytes *phys, int reason, int pages)
 387 {
 388 /* Allocate a page for use by VM itself. */
 389         phys_bytes newpage;
 390         vir_bytes loc;
 391         pt_t *pt;
 392         int r;
 393         static int level = 0;
 394         void *ret;
 395         u32_t mem_flags = 0;
 396
 397         pt = &vmprocess->vm_pt;
 398         assert(reason >= 0 && reason < VMP_CATEGORIES);
 399
 400         assert(pages > 0);
 401
 402         level++;
 403
 404         assert(level >= 1);
 405         assert(level <= 2);
 406
 407         if((level > 1) || !pt_init_done) {
 408                 void *s;
 409
 410                 if(pages == 1) s=vm_getsparepage(phys);
 411                 else if(pages == 4) s=vm_getsparepagedir(phys);
 412                 else panic("%d pages", pages);
 413
 414                 level--;
 415                 if(!s) {
 416                         util_stacktrace();
 417                         printf("VM: warning: out of spare pages\n");
 418                 }
 419                 if(!is_staticaddr(s)) vm_self_pages++;
 420                 return s;
 421         }
 422
 423 #if defined(__arm__)
 424         if (reason == VMP_PAGEDIR) {
 425                 mem_flags |= PAF_ALIGN16K;
 426         }
 427 #endif
 428
 429         /* VM does have a pagetable, so get a page and map it in there.
 430          * Where in our virtual address space can we put it?
 431          */
 432         loc = findhole(pages);
 433         if(loc == NO_MEM) {
 434                 level--;
 435                 printf("VM: vm_allocpage: findhole failed\n");
 436                 return NULL;
 437         }
 438
 439         /* Allocate page of memory for use by VM. As VM
 440          * is trusted, we don't have to pre-clear it.
 441          */
 442         if((newpage = alloc_mem(pages, mem_flags)) == NO_MEM) {
 443                 level--;
 444                 printf("VM: vm_allocpage: alloc_mem failed\n");
 445                 return NULL;
 446         }
 447
 448         *phys = CLICK2ABS(newpage);
 449
 450         /* Map this page into our address space. */
 451         if((r=pt_writemap(vmprocess, pt, loc, *phys, VM_PAGE_SIZE*pages,
 452                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW
 453 #if defined(__arm__)
 454                 | ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE
 455 #endif
 456                 , 0)) != OK) {
 457                 free_mem(newpage, pages);
 458                 printf("vm_allocpage writemap failed\n");
 459                 level--;
 460                 return NULL;
 461         }
 462
 463         if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 464                 panic("VMCTL_FLUSHTLB failed: %d", r);
 465         }
 466
 467         level--;
 468
 469         /* Return user-space-ready pointer to it. */
 470         ret = (void *) loc;
 471
 472         vm_self_pages++;
 473         return ret;
 474 }
 475
 476 void *vm_allocpage(phys_bytes *phys, int reason)
 477 {
 478         return vm_allocpages(phys, reason, 1);
 479 }
 480
 481 /*===========================================================================*
 482  *                              vm_pagelock                                  *
 483  *===========================================================================*/
 484 void vm_pagelock(void *vir, int lockflag)
 485 {
 486 /* Mark a page allocated by vm_allocpage() unwritable, i.e. only for VM. */
 487         vir_bytes m = (vir_bytes) vir;
 488         int r;
 489         u32_t flags = ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER;
 490         pt_t *pt;
 491
 492         pt = &vmprocess->vm_pt;
 493
 494         assert(!(m % VM_PAGE_SIZE));
 495
 496         if(!lockflag)
 497                 flags |= ARCH_VM_PTE_RW;
 498 #if defined(__arm__)
 499         else
 500                 flags |= ARCH_VM_PTE_RO;
 501         flags |= ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE;
 502 #endif
 503
 504         /* Update flags. */
 505         if((r=pt_writemap(vmprocess, pt, m, 0, VM_PAGE_SIZE,
 506                 flags, WMF_OVERWRITE | WMF_WRITEFLAGSONLY)) != OK) {
 507                 panic("vm_lockpage: pt_writemap failed");
 508         }
 509
 510         if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 511                 panic("VMCTL_FLUSHTLB failed: %d", r);
 512         }
 513
 514         return;
 515 }
 516
 517 /*===========================================================================*
 518  *                              vm_addrok                                    *
 519  *===========================================================================*/
 520 int vm_addrok(void *vir, int writeflag)
 521 {
 522         pt_t *pt = &vmprocess->vm_pt;
 523         int pde, pte;
 524         vir_bytes v = (vir_bytes) vir;
 525
 526 #if defined(__i386__)
 527         pde = I386_VM_PDE(v);
 528         pte = I386_VM_PTE(v);
 529 #elif defined(__arm__)
 530         pde = ARM_VM_PDE(v);
 531         pte = ARM_VM_PTE(v);
 532 #endif
 533
 534         if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 535                 printf("addr not ok: missing pde %d\n", pde);
 536                 return 0;
 537         }
 538
 539 #if defined(__i386__)
 540         if(writeflag &&
 541                 !(pt->pt_dir[pde] & ARCH_VM_PTE_RW)) {
 542                 printf("addr not ok: pde %d present but pde unwritable\n", pde);
 543                 return 0;
 544         }
 545
 546 #endif
 547         if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) {
 548                 printf("addr not ok: missing pde %d / pte %d\n",
 549                         pde, pte);
 550                 return 0;
 551         }
 552
 553 #if defined(__i386__)
 554         if(writeflag &&
 555                 !(pt->pt_pt[pde][pte] & ARCH_VM_PTE_RW)) {
 556                 printf("addr not ok: pde %d / pte %d present but unwritable\n",
 557 #elif defined(__arm__)
 558         if(!writeflag &&
 559                 !(pt->pt_pt[pde][pte] & ARCH_VM_PTE_RO)) {
 560                 printf("addr not ok: pde %d / pte %d present but writable\n",
 561 #endif
 562                         pde, pte);
 563                 return 0;
 564         }
 565
 566         return 1;
 567 }
 568
 569 /*===========================================================================*
 570  *                              pt_ptalloc                                   *
 571  *===========================================================================*/
 572 static int pt_ptalloc(pt_t *pt, int pde, u32_t flags)
 573 {
 574 /* Allocate a page table and write its address into the page directory. */
 575         int i;
 576         phys_bytes pt_phys;
 577
 578         /* Argument must make sense. */
 579         assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES);
 580         assert(!(flags & ~(PTF_ALLFLAGS)));
 581
 582         /* We don't expect to overwrite page directory entry, nor
 583          * storage for the page table.
 584          */
 585         assert(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT));
 586         assert(!pt->pt_pt[pde]);
 587
 588         /* Get storage for the page table. */
 589         if(!(pt->pt_pt[pde] = vm_allocpage(&pt_phys, VMP_PAGETABLE)))
 590                 return ENOMEM;
 591
 592         for(i = 0; i < ARCH_VM_PT_ENTRIES; i++)
 593                 pt->pt_pt[pde][i] = 0;  /* Empty entry. */
 594
 595         /* Make page directory entry.
 596          * The PDE is always 'present,' 'writable,' and 'user accessible,'
 597          * relying on the PTE for protection.
 598          */
 599 #if defined(__i386__)
 600         pt->pt_dir[pde] = (pt_phys & ARCH_VM_ADDR_MASK) | flags
 601                 | ARCH_VM_PDE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW;
 602 #elif defined(__arm__)
 603         pt->pt_dir[pde] = (pt_phys & ARCH_VM_PDE_MASK)
 604                 | ARCH_VM_PDE_PRESENT | ARM_VM_PDE_DOMAIN;
 605 #endif
 606
 607         return OK;
 608 }
 609
 610 /*===========================================================================*
 611  *                          pt_ptalloc_in_range                              *
 612  *===========================================================================*/
 613 int pt_ptalloc_in_range(pt_t *pt, vir_bytes start, vir_bytes end,
 614         u32_t flags, int verify)
 615 {
 616 /* Allocate all the page tables in the range specified. */
 617         int pde, first_pde, last_pde;
 618
 619 #if defined(__i386__)
 620         first_pde = I386_VM_PDE(start);
 621         last_pde = I386_VM_PDE(end-1);
 622 #elif defined(__arm__)
 623         first_pde = ARM_VM_PDE(start);
 624         last_pde = ARM_VM_PDE(end-1);
 625 #endif
 626         assert(first_pde >= 0);
 627         assert(last_pde < ARCH_VM_DIR_ENTRIES);
 628
 629         /* Scan all page-directory entries in the range. */
 630         for(pde = first_pde; pde <= last_pde; pde++) {
 631                 assert(!(pt->pt_dir[pde] & ARCH_VM_BIGPAGE));
 632                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 633                         int r;
 634                         if(verify) {
 635                                 printf("pt_ptalloc_in_range: no pde %d\n", pde);
 636                                 return EFAULT;
 637                         }
 638                         assert(!pt->pt_dir[pde]);
 639                         if((r=pt_ptalloc(pt, pde, flags)) != OK) {
 640                                 /* Couldn't do (complete) mapping.
 641                                  * Don't bother freeing any previously
 642                                  * allocated page tables, they're
 643                                  * still writable, don't point to nonsense,
 644                                  * and pt_ptalloc leaves the directory
 645                                  * and other data in a consistent state.
 646                                  */
 647                                 return r;
 648                         }
 649                         assert(pt->pt_pt[pde]);
 650                 }
 651                 assert(pt->pt_pt[pde]);
 652                 assert(pt->pt_dir[pde]);
 653                 assert(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT);
 654         }
 655
 656         return OK;
 657 }
 658
 659 static char *ptestr(u32_t pte)
 660 {
 661 #define FLAG(constant, name) {                                          \
 662         if(pte & (constant)) { strcat(str, name); strcat(str, " "); }   \
 663 }
 664
 665         static char str[30];
 666         if(!(pte & ARCH_VM_PTE_PRESENT)) {
 667                 return "not present";
 668         }
 669         str[0] = '\0';
 670 #if defined(__i386__)
 671         FLAG(ARCH_VM_PTE_RW, "W");
 672 #elif defined(__arm__)
 673         if(pte & ARCH_VM_PTE_RO) {
 674             strcat(str, "R ");
 675         } else {
 676             strcat(str, "W ");
 677         }
 678 #endif
 679         FLAG(ARCH_VM_PTE_USER, "U");
 680 #if defined(__i386__)
 681         FLAG(I386_VM_PWT, "PWT");
 682         FLAG(I386_VM_PCD, "PCD");
 683         FLAG(I386_VM_ACC, "ACC");
 684         FLAG(I386_VM_DIRTY, "DIRTY");
 685         FLAG(I386_VM_PS, "PS");
 686         FLAG(I386_VM_GLOBAL, "G");
 687         FLAG(I386_VM_PTAVAIL1, "AV1");
 688         FLAG(I386_VM_PTAVAIL2, "AV2");
 689         FLAG(I386_VM_PTAVAIL3, "AV3");
 690 #elif defined(__arm__)
 691         FLAG(ARM_VM_PTE_SUPER, "S");
 692         FLAG(ARM_VM_PTE_SHAREABLE, "SH");
 693         FLAG(ARM_VM_PTE_WB, "WB");
 694         FLAG(ARM_VM_PTE_WT, "WT");
 695 #endif
 696
 697         return str;
 698 }
 699
 700 /*===========================================================================*
 701  *                           pt_map_in_range                                 *
 702  *===========================================================================*/
 703 int pt_map_in_range(struct vmproc *src_vmp, struct vmproc *dst_vmp,
 704         vir_bytes start, vir_bytes end)
 705 {
 706 /* Transfer all the mappings from the pt of the source process to the pt of
 707  * the destination process in the range specified.
 708  */
 709         int pde, pte;
 710         vir_bytes viraddr;
 711         pt_t *pt, *dst_pt;
 712
 713         pt = &src_vmp->vm_pt;
 714         dst_pt = &dst_vmp->vm_pt;
 715
 716         end = end ? end : VM_DATATOP;
 717         assert(start % VM_PAGE_SIZE == 0);
 718         assert(end % VM_PAGE_SIZE == 0);
 719 #if defined(__i386__)
 720         assert(start <= end);
 721         assert(I386_VM_PDE(end) < ARCH_VM_DIR_ENTRIES);
 722 #elif defined(__arm__)
 723         assert(ARM_VM_PDE(start) >= 0 && start <= end);
 724         assert(ARM_VM_PDE(end) < ARCH_VM_DIR_ENTRIES);
 725 #endif
 726
 727 #if LU_DEBUG
 728         printf("VM: pt_map_in_range: src = %d, dst = %d\n",
 729                 src_vmp->vm_endpoint, dst_vmp->vm_endpoint);
 730         printf("VM: pt_map_in_range: transferring from 0x%08x (pde %d pte %d) to 0x%08x (pde %d pte %d)\n",
 731 #if defined(__i386__)
 732                 start, I386_VM_PDE(start), I386_VM_PTE(start),
 733                 end, I386_VM_PDE(end), I386_VM_PTE(end));
 734 #elif defined(__arm__)
 735                 start, ARM_VM_PDE(start), ARM_VM_PTE(start),
 736                 end, ARM_VM_PDE(end), ARM_VM_PTE(end));
 737 #endif
 738 #endif
 739
 740         /* Scan all page-table entries in the range. */
 741         for(viraddr = start; viraddr <= end; viraddr += VM_PAGE_SIZE) {
 742 #if defined(__i386__)
 743                 pde = I386_VM_PDE(viraddr);
 744 #elif defined(__arm__)
 745                 pde = ARM_VM_PDE(viraddr);
 746 #endif
 747                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 748                         if(viraddr == VM_DATATOP) break;
 749                         continue;
 750                 }
 751 #if defined(__i386__)
 752                 pte = I386_VM_PTE(viraddr);
 753 #elif defined(__arm__)
 754                 pte = ARM_VM_PTE(viraddr);
 755 #endif
 756                 if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) {
 757                         if(viraddr == VM_DATATOP) break;
 758                         continue;
 759                 }
 760
 761                 /* Transfer the mapping. */
 762                 dst_pt->pt_pt[pde][pte] = pt->pt_pt[pde][pte];
 763
 764                 if(viraddr == VM_DATATOP) break;
 765         }
 766
 767         return OK;
 768 }
 769
 770 /*===========================================================================*
 771  *                              pt_ptmap                                     *
 772  *===========================================================================*/
 773 int pt_ptmap(struct vmproc *src_vmp, struct vmproc *dst_vmp)
 774 {
 775 /* Transfer mappings to page dir and page tables from source process and
 776  * destination process. Make sure all the mappings are above the stack, not
 777  * to corrupt valid mappings in the data segment of the destination process.
 778  */
 779         int pde, r;
 780         phys_bytes physaddr;
 781         vir_bytes viraddr;
 782         pt_t *pt;
 783
 784         pt = &src_vmp->vm_pt;
 785
 786 #if LU_DEBUG
 787         printf("VM: pt_ptmap: src = %d, dst = %d\n",
 788                 src_vmp->vm_endpoint, dst_vmp->vm_endpoint);
 789 #endif
 790
 791         /* Transfer mapping to the page directory. */
 792         viraddr = (vir_bytes) pt->pt_dir;
 793         physaddr = pt->pt_dir_phys & ARCH_VM_ADDR_MASK;
 794 #if defined(__i386__)
 795         if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, VM_PAGE_SIZE,
 796                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW,
 797 #elif defined(__arm__)
 798         if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, ARCH_PAGEDIR_SIZE,
 799                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW |
 800                 ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE,
 801 #endif
 802                 WMF_OVERWRITE)) != OK) {
 803                 return r;
 804         }
 805 #if LU_DEBUG
 806         printf("VM: pt_ptmap: transferred mapping to page dir: 0x%08x (0x%08x)\n",
 807                 viraddr, physaddr);
 808 #endif
 809
 810         /* Scan all non-reserved page-directory entries. */
 811         for(pde=0; pde < ARCH_VM_DIR_ENTRIES; pde++) {
 812                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 813                         continue;
 814                 }
 815
 816                 /* Transfer mapping to the page table. */
 817                 viraddr = (vir_bytes) pt->pt_pt[pde];
 818 #if defined(__i386__)
 819                 physaddr = pt->pt_dir[pde] & ARCH_VM_ADDR_MASK;
 820 #elif defined(__arm__)
 821                 physaddr = pt->pt_dir[pde] & ARCH_VM_PDE_MASK;
 822 #endif
 823                 if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, VM_PAGE_SIZE,
 824                         ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW
 825 #ifdef __arm__
 826                         | ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW |
 827                         ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE
 828 #endif
 829                         ,
 830                         WMF_OVERWRITE)) != OK) {
 831                         return r;
 832                 }
 833         }
 834
 835         return OK;
 836 }
 837
 838 void pt_clearmapcache(void)
 839 {
 840         /* Make sure kernel will invalidate tlb when using current
 841          * pagetable (i.e. vm's) to make new mappings before new cr3
 842          * is loaded.
 843          */
 844         if(sys_vmctl(SELF, VMCTL_CLEARMAPCACHE, 0) != OK)
 845                 panic("VMCTL_CLEARMAPCACHE failed");
 846 }
 847
 848 /*===========================================================================*
 849  *                              pt_writemap                                  *
 850  *===========================================================================*/
 851 int pt_writemap(struct vmproc * vmp,
 852                         pt_t *pt,
 853                         vir_bytes v,
 854                         phys_bytes physaddr,
 855                         size_t bytes,
 856                         u32_t flags,
 857                         u32_t writemapflags)
 858 {
 859 /* Write mapping into page table. Allocate a new page table if necessary. */
 860 /* Page directory and table entries for this virtual address. */
 861         int p, pages;
 862         int verify = 0;
 863         int ret = OK;
 864
 865 #ifdef CONFIG_SMP
 866         int vminhibit_clear = 0;
 867         /* FIXME
 868          * don't do it everytime, stop the process only on the first change and
 869          * resume the execution on the last change. Do in a wrapper of this
 870          * function
 871          */
 872         if (vmp && vmp->vm_endpoint != NONE && vmp->vm_endpoint != VM_PROC_NR &&
 873                         !(vmp->vm_flags & VMF_EXITING)) {
 874                 sys_vmctl(vmp->vm_endpoint, VMCTL_VMINHIBIT_SET, 0);
 875                 vminhibit_clear = 1;
 876         }
 877 #endif
 878
 879         if(writemapflags & WMF_VERIFY)
 880                 verify = 1;
 881
 882         assert(!(bytes % VM_PAGE_SIZE));
 883         assert(!(flags & ~(PTF_ALLFLAGS)));
 884
 885         pages = bytes / VM_PAGE_SIZE;
 886
 887         /* MAP_NONE means to clear the mapping. It doesn't matter
 888          * what's actually written into the PTE if PRESENT
 889          * isn't on, so we can just write MAP_NONE into it.
 890          */
 891         assert(physaddr == MAP_NONE || (flags & ARCH_VM_PTE_PRESENT));
 892         assert(physaddr != MAP_NONE || !flags);
 893
 894         /* First make sure all the necessary page tables are allocated,
 895          * before we start writing in any of them, because it's a pain
 896          * to undo our work properly.
 897          */
 898         ret = pt_ptalloc_in_range(pt, v, v + VM_PAGE_SIZE*pages, flags, verify);
 899         if(ret != OK) {
 900                 printf("VM: writemap: pt_ptalloc_in_range failed\n");
 901                 goto resume_exit;
 902         }
 903
 904         /* Now write in them. */
 905         for(p = 0; p < pages; p++) {
 906                 u32_t entry;
 907 #if defined(__i386__)
 908                 int pde = I386_VM_PDE(v);
 909                 int pte = I386_VM_PTE(v);
 910 #elif defined(__arm__)
 911                 int pde = ARM_VM_PDE(v);
 912                 int pte = ARM_VM_PTE(v);
 913 #endif
 914
 915                 assert(!(v % VM_PAGE_SIZE));
 916                 assert(pte >= 0 && pte < ARCH_VM_PT_ENTRIES);
 917                 assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES);
 918
 919                 /* Page table has to be there. */
 920                 assert(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT);
 921
 922                 /* We do not expect it to be a bigpage. */
 923                 assert(!(pt->pt_dir[pde] & ARCH_VM_BIGPAGE));
 924
 925                 /* Make sure page directory entry for this page table
 926                  * is marked present and page table entry is available.
 927                  */
 928                 assert(pt->pt_pt[pde]);
 929
 930 #if SANITYCHECKS
 931                 /* We don't expect to overwrite a page. */
 932                 if(!(writemapflags & (WMF_OVERWRITE|WMF_VERIFY)))
 933                         assert(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT));
 934 #endif
 935                 if(writemapflags & (WMF_WRITEFLAGSONLY|WMF_FREE)) {
 936 #if defined(__i386__)
 937                         physaddr = pt->pt_pt[pde][pte] & ARCH_VM_ADDR_MASK;
 938 #elif defined(__arm__)
 939                         physaddr = pt->pt_pt[pde][pte] & ARM_VM_PTE_MASK;
 940 #endif
 941                 }
 942
 943                 if(writemapflags & WMF_FREE) {
 944                         free_mem(ABS2CLICK(physaddr), 1);
 945                 }
 946
 947                 /* Entry we will write. */
 948 #if defined(__i386__)
 949                 entry = (physaddr & ARCH_VM_ADDR_MASK) | flags;
 950 #elif defined(__arm__)
 951                 entry = (physaddr & ARM_VM_PTE_MASK) | flags;
 952 #endif
 953
 954                 if(verify) {
 955                         u32_t maskedentry;
 956                         maskedentry = pt->pt_pt[pde][pte];
 957 #if defined(__i386__)
 958                         maskedentry &= ~(I386_VM_ACC|I386_VM_DIRTY);
 959 #endif
 960                         /* Verify pagetable entry. */
 961                         if(entry & ARCH_VM_PTE_RW) {
 962                                 /* If we expect a writable page, allow a readonly page. */
 963                                 maskedentry |= ARCH_VM_PTE_RW;
 964                         }
 965                         if(maskedentry != entry) {
 966                                 printf("pt_writemap: mismatch: ");
 967 #if defined(__i386__)
 968                                 if((entry & ARCH_VM_ADDR_MASK) !=
 969                                         (maskedentry & ARCH_VM_ADDR_MASK)) {
 970 #elif defined(__arm__)
 971                                 if((entry & ARM_VM_PTE_MASK) !=
 972                                         (maskedentry & ARM_VM_PTE_MASK)) {
 973 #endif
 974                                         printf("pt_writemap: physaddr mismatch (0x%lx, 0x%lx); ",
 975                                                 (long)entry, (long)maskedentry);
 976                                 } else printf("phys ok; ");
 977                                 printf(" flags: found %s; ",
 978                                         ptestr(pt->pt_pt[pde][pte]));
 979                                 printf(" masked %s; ",
 980                                         ptestr(maskedentry));
 981                                 printf(" expected %s\n", ptestr(entry));
 982                                 ret = EFAULT;
 983                                 goto resume_exit;
 984                         }
 985                 } else {
 986                         /* Write pagetable entry. */
 987                         pt->pt_pt[pde][pte] = entry;
 988                 }
 989
 990                 physaddr += VM_PAGE_SIZE;
 991                 v += VM_PAGE_SIZE;
 992         }
 993
 994 resume_exit:
 995
 996 #ifdef CONFIG_SMP
 997         if (vminhibit_clear) {
 998                 assert(vmp && vmp->vm_endpoint != NONE && vmp->vm_endpoint != VM_PROC_NR &&
 999                         !(vmp->vm_flags & VMF_EXITING));
1000                 sys_vmctl(vmp->vm_endpoint, VMCTL_VMINHIBIT_CLEAR, 0);
1001         }
1002 #endif
1003
1004         return ret;
1005 }
1006
1007 /*===========================================================================*
1008  *                              pt_checkrange                                *
1009  *===========================================================================*/
1010 int pt_checkrange(pt_t *pt, vir_bytes v,  size_t bytes,
1011         int write)
1012 {
1013         int p, pages;
1014
1015         assert(!(bytes % VM_PAGE_SIZE));
1016
1017         pages = bytes / VM_PAGE_SIZE;
1018
1019         for(p = 0; p < pages; p++) {
1020 #if defined(__i386__)
1021                 int pde = I386_VM_PDE(v);
1022                 int pte = I386_VM_PTE(v);
1023 #elif defined(__arm__)
1024                 int pde = ARM_VM_PDE(v);
1025                 int pte = ARM_VM_PTE(v);
1026 #endif
1027
1028                 assert(!(v % VM_PAGE_SIZE));
1029                 assert(pte >= 0 && pte < ARCH_VM_PT_ENTRIES);
1030                 assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES);
1031
1032                 /* Page table has to be there. */
1033                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT))
1034                         return EFAULT;
1035
1036                 /* Make sure page directory entry for this page table
1037                  * is marked present and page table entry is available.
1038                  */
1039                 assert((pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT) && pt->pt_pt[pde]);
1040
1041                 if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) {
1042                         return EFAULT;
1043                 }
1044
1045 #if defined(__i386__)
1046                 if(write && !(pt->pt_pt[pde][pte] & ARCH_VM_PTE_RW)) {
1047 #elif defined(__arm__)
1048                 if(write && (pt->pt_pt[pde][pte] & ARCH_VM_PTE_RO)) {
1049 #endif
1050                         return EFAULT;
1051                 }
1052
1053                 v += VM_PAGE_SIZE;
1054         }
1055
1056         return OK;
1057 }
1058
1059 /*===========================================================================*
1060  *                              pt_new                                       *
1061  *===========================================================================*/
1062 int pt_new(pt_t *pt)
1063 {
1064 /* Allocate a pagetable root. Allocate a page-aligned page directory
1065  * and set them to 0 (indicating no page tables are allocated). Lookup
1066  * its physical address as we'll need that in the future. Verify it's
1067  * page-aligned.
1068  */
1069         int i, r;
1070
1071         /* Don't ever re-allocate/re-move a certain process slot's
1072          * page directory once it's been created. This is a fraction
1073          * faster, but also avoids having to invalidate the page
1074          * mappings from in-kernel page tables pointing to
1075          * the page directories (the page_directories data).
1076          */
1077         if(!pt->pt_dir &&
1078           !(pt->pt_dir = vm_allocpages((phys_bytes *)&pt->pt_dir_phys,
1079                 VMP_PAGEDIR, ARCH_PAGEDIR_SIZE/VM_PAGE_SIZE))) {
1080                 return ENOMEM;
1081         }
1082
1083         assert(!((u32_t)pt->pt_dir_phys % ARCH_PAGEDIR_SIZE));
1084
1085         for(i = 0; i < ARCH_VM_DIR_ENTRIES; i++) {
1086                 pt->pt_dir[i] = 0; /* invalid entry (PRESENT bit = 0) */
1087                 pt->pt_pt[i] = NULL;
1088         }
1089
1090         /* Where to start looking for free virtual address space? */
1091         pt->pt_virtop = 0;
1092
1093         /* Map in kernel. */
1094         if((r=pt_mapkernel(pt)) != OK)
1095                 return r;
1096
1097         return OK;
1098 }
1099
1100 static int freepde(void)
1101 {
1102         int p = kernel_boot_info.freepde_start++;
1103         assert(kernel_boot_info.freepde_start < ARCH_VM_DIR_ENTRIES);
1104         return p;
1105 }
1106
1107 /*===========================================================================*
1108  *                              pt_init                                      *
1109  *===========================================================================*/
1110 void pt_init(void)
1111 {
1112         pt_t *newpt;
1113         int s, r, p;
1114         int global_bit_ok = 0;
1115         vir_bytes sparepages_mem;
1116 #if defined(__arm__)
1117         vir_bytes sparepagedirs_mem;
1118 #endif
1119         static u32_t currentpagedir[ARCH_VM_DIR_ENTRIES];
1120         int m = kernel_boot_info.kern_mod;
1121 #if defined(__i386__)
1122         u32_t mypdbr; /* Page Directory Base Register (cr3) value */
1123 #elif defined(__arm__)
1124         u32_t myttbr;
1125 #endif
1126
1127         /* Find what the physical location of the kernel is. */
1128         assert(m >= 0);
1129         assert(m < kernel_boot_info.mods_with_kernel);
1130         assert(kernel_boot_info.mods_with_kernel < MULTIBOOT_MAX_MODS);
1131         kern_mb_mod = &kernel_boot_info.module_list[m];
1132         kern_size = kern_mb_mod->mod_end - kern_mb_mod->mod_start;
1133         assert(!(kern_mb_mod->mod_start % ARCH_BIG_PAGE_SIZE));
1134         assert(!(kernel_boot_info.vir_kern_start % ARCH_BIG_PAGE_SIZE));
1135         kern_start_pde = kernel_boot_info.vir_kern_start / ARCH_BIG_PAGE_SIZE;
1136
1137         /* Get ourselves spare pages. */
1138         sparepages_mem = (vir_bytes) static_sparepages;
1139         assert(!(sparepages_mem % VM_PAGE_SIZE));
1140
1141 #if defined(__arm__)
1142         /* Get ourselves spare pagedirs. */
1143         sparepagedirs_mem = (vir_bytes) static_sparepagedirs;
1144         assert(!(sparepagedirs_mem % ARCH_PAGEDIR_SIZE));
1145 #endif
1146
1147         /* Spare pages are used to allocate memory before VM has its own page
1148          * table that things (i.e. arbitrary physical memory) can be mapped into.
1149          * We get it by pre-allocating it in our bss (allocated and mapped in by
1150          * the kernel) in static_sparepages. We also need the physical addresses
1151          * though; we look them up now so they are ready for use.
1152          */
1153 #if defined(__arm__)
1154         missing_sparedirs = 0;
1155         assert(STATIC_SPAREPAGEDIRS < SPAREPAGEDIRS);
1156         for(s = 0; s < SPAREPAGEDIRS; s++) {
1157                 vir_bytes v = (sparepagedirs_mem + s*ARCH_PAGEDIR_SIZE);;
1158                 phys_bytes ph;
1159                 if((r=sys_umap(SELF, VM_D, (vir_bytes) v,
1160                         ARCH_PAGEDIR_SIZE, &ph)) != OK)
1161                                 panic("pt_init: sys_umap failed: %d", r);
1162                 if(s >= STATIC_SPAREPAGEDIRS) {
1163                         sparepagedirs[s].pagedir = NULL;
1164                         missing_sparedirs++;
1165                         continue;
1166                 }
1167                 sparepagedirs[s].pagedir = (void *) v;
1168                 sparepagedirs[s].phys = ph;
1169         }
1170 #endif
1171
1172         missing_spares = 0;
1173         assert(STATIC_SPAREPAGES < SPAREPAGES);
1174         for(s = 0; s < SPAREPAGES; s++) {
1175                 vir_bytes v = (sparepages_mem + s*VM_PAGE_SIZE);;
1176                 phys_bytes ph;
1177                 if((r=sys_umap(SELF, VM_D, (vir_bytes) v,
1178                         VM_PAGE_SIZE*SPAREPAGES, &ph)) != OK)
1179                                 panic("pt_init: sys_umap failed: %d", r);
1180                 if(s >= STATIC_SPAREPAGES) {
1181                         sparepages[s].page = NULL;
1182                         missing_spares++;
1183                         continue;
1184                 }
1185                 sparepages[s].page = (void *) v;
1186                 sparepages[s].phys = ph;
1187         }
1188
1189 #if defined(__i386__)
1190         /* global bit and 4MB pages available? */
1191         global_bit_ok = _cpufeature(_CPUF_I386_PGE);
1192         bigpage_ok = _cpufeature(_CPUF_I386_PSE);
1193
1194         /* Set bit for PTE's and PDE's if available. */
1195         if(global_bit_ok)
1196                 global_bit = I386_VM_GLOBAL;
1197 #endif
1198
1199         /* Allocate us a page table in which to remember page directory
1200          * pointers.
1201          */
1202         if(!(page_directories = vm_allocpage(&page_directories_phys,
1203                 VMP_PAGETABLE)))
1204                 panic("no virt addr for vm mappings");
1205
1206         memset(page_directories, 0, VM_PAGE_SIZE);
1207
1208         /* Now reserve another pde for kernel's own mappings. */
1209         {
1210                 int kernmap_pde;
1211                 phys_bytes addr, len;
1212                 int flags, index = 0;
1213                 u32_t offset = 0;
1214
1215                 kernmap_pde = freepde();
1216                 offset = kernmap_pde * ARCH_BIG_PAGE_SIZE;
1217
1218                 while(sys_vmctl_get_mapping(index, &addr, &len,
1219                         &flags) == OK)  {
1220                         int usedpde;
1221                         vir_bytes vir;
1222                         if(index >= MAX_KERNMAPPINGS)
1223                                 panic("VM: too many kernel mappings: %d", index);
1224                         kern_mappings[index].phys_addr = addr;
1225                         kern_mappings[index].len = len;
1226                         kern_mappings[index].flags = flags;
1227                         kern_mappings[index].vir_addr = offset;
1228                         kern_mappings[index].flags =
1229                                 ARCH_VM_PTE_PRESENT;
1230                         if(flags & VMMF_UNCACHED)
1231 #if defined(__i386__)
1232                                 kern_mappings[index].flags |= PTF_NOCACHE;
1233 #elif defined(__arm__)
1234                                 kern_mappings[index].flags |= ARM_VM_PTE_DEVICE;
1235                         else
1236                                 kern_mappings[index].flags |=
1237                                     ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE;
1238 #endif
1239                         if(flags & VMMF_USER)
1240                                 kern_mappings[index].flags |= ARCH_VM_PTE_USER;
1241 #if defined(__arm__)
1242                         else
1243                                 kern_mappings[index].flags |= ARM_VM_PTE_SUPER;
1244 #endif
1245                         if(flags & VMMF_WRITE)
1246                                 kern_mappings[index].flags |= ARCH_VM_PTE_RW;
1247 #if defined(__i386__)
1248                         if(flags & VMMF_GLO)
1249                                 kern_mappings[index].flags |= I386_VM_GLOBAL;
1250 #elif defined(__arm__)
1251                         else
1252                                 kern_mappings[index].flags |= ARCH_VM_PTE_RO;
1253 #endif
1254                         if(addr % VM_PAGE_SIZE)
1255                                 panic("VM: addr unaligned: %d", addr);
1256                         if(len % VM_PAGE_SIZE)
1257                                 panic("VM: len unaligned: %d", len);
1258                         vir = offset;
1259                         if(sys_vmctl_reply_mapping(index, vir) != OK)
1260                                 panic("VM: reply failed");
1261                         offset += len;
1262                         index++;
1263                         kernmappings++;
1264
1265 #if defined(__i386__)
1266                         usedpde = I386_VM_PDE(offset);
1267 #elif defined(__arm__)
1268                         usedpde = ARM_VM_PDE(offset);
1269 #endif
1270                         while(usedpde > kernmap_pde) {
1271                                 int newpde = freepde();
1272                                 assert(newpde == kernmap_pde+1);
1273                                 kernmap_pde = newpde;
1274                         }
1275                 }
1276         }
1277
1278         /* Find a PDE below processes available for mapping in the
1279          * page directories.
1280          */
1281         pagedir_pde = freepde();
1282 #if defined(__i386__)
1283         pagedir_pde_val = (page_directories_phys & ARCH_VM_ADDR_MASK) |
1284                         ARCH_VM_PDE_PRESENT | ARCH_VM_PTE_RW;
1285 #elif defined(__arm__)
1286         pagedir_pde_val = (page_directories_phys & ARCH_VM_PDE_MASK) |
1287                         ARCH_VM_PDE_PRESENT | ARM_VM_PDE_DOMAIN;
1288 #endif
1289
1290         /* Allright. Now. We have to make our own page directory and page tables,
1291          * that the kernel has already set up, accessible to us. It's easier to
1292          * understand if we just copy all the required pages (i.e. page directory
1293          * and page tables), and set up the pointers as if VM had done it itself.
1294          *
1295          * This allocation will happen without using any page table, and just
1296          * uses spare pages.
1297          */
1298         newpt = &vmprocess->vm_pt;
1299         if(pt_new(newpt) != OK)
1300                 panic("vm pt_new failed");
1301
1302         /* Get our current pagedir so we can see it. */
1303 #if defined(__i386__)
1304         if(sys_vmctl_get_pdbr(SELF, &mypdbr) != OK)
1305 #elif defined(__arm__)
1306         if(sys_vmctl_get_pdbr(SELF, &myttbr) != OK)
1307 #endif
1308                 panic("VM: sys_vmctl_get_pdbr failed");
1309 #if defined(__i386__)
1310         if(sys_vircopy(NONE, mypdbr, SELF,
1311                 (vir_bytes) currentpagedir, VM_PAGE_SIZE) != OK)
1312 #elif defined(__arm__)
1313         if(sys_vircopy(NONE, myttbr, SELF,
1314                 (vir_bytes) currentpagedir, ARCH_PAGEDIR_SIZE) != OK)
1315 #endif
1316                 panic("VM: sys_vircopy failed");
1317
1318         /* We have mapped in kernel ourselves; now copy mappings for VM
1319          * that kernel made, including allocations for BSS. Skip identity
1320          * mapping bits; just map in VM.
1321          */
1322         for(p = 0; p < ARCH_VM_DIR_ENTRIES; p++) {
1323                 u32_t entry = currentpagedir[p];
1324                 phys_bytes ptaddr_kern, ptaddr_us;
1325
1326                 /* BIGPAGEs are kernel mapping (do ourselves) or boot
1327                  * identity mapping (don't want).
1328                  */
1329                 if(!(entry & ARCH_VM_PDE_PRESENT)) continue;
1330                 if((entry & ARCH_VM_BIGPAGE)) continue;
1331
1332                 if(pt_ptalloc(newpt, p, 0) != OK)
1333                         panic("pt_ptalloc failed");
1334                 assert(newpt->pt_dir[p] & ARCH_VM_PDE_PRESENT);
1335
1336 #if defined(__i386__)
1337                 ptaddr_kern = entry & ARCH_VM_ADDR_MASK;
1338                 ptaddr_us = newpt->pt_dir[p] & ARCH_VM_ADDR_MASK;
1339 #elif defined(__arm__)
1340                 ptaddr_kern = entry & ARCH_VM_PDE_MASK;
1341                 ptaddr_us = newpt->pt_dir[p] & ARCH_VM_PDE_MASK;
1342 #endif
1343
1344                 /* Copy kernel-initialized pagetable contents into our
1345                  * normally accessible pagetable.
1346                  */
1347                 if(sys_abscopy(ptaddr_kern, ptaddr_us, VM_PAGE_SIZE) != OK)
1348                         panic("pt_init: abscopy failed");
1349         }
1350
1351         /* Inform kernel vm has a newly built page table. */
1352         assert(vmproc[VM_PROC_NR].vm_endpoint == VM_PROC_NR);
1353         pt_bind(newpt, &vmproc[VM_PROC_NR]);
1354
1355         pt_init_done = 1;
1356
1357         vm_checkspares();
1358
1359         /* All OK. */
1360         return;
1361 }
1362
1363 /*===========================================================================*
1364  *                              pt_bind                                      *
1365  *===========================================================================*/
1366 int pt_bind(pt_t *pt, struct vmproc *who)
1367 {
1368         int slot;
1369         u32_t phys;
1370         void *pdes;
1371         int pages_per_pagedir = ARCH_PAGEDIR_SIZE/VM_PAGE_SIZE;
1372
1373         /* Basic sanity checks. */
1374         assert(who);
1375         assert(who->vm_flags & VMF_INUSE);
1376         assert(pt);
1377
1378         assert(pagedir_pde >= 0);
1379
1380         slot = who->vm_slot;
1381         assert(slot >= 0);
1382         assert(slot < ELEMENTS(vmproc));
1383         assert(slot < ARCH_VM_PT_ENTRIES / pages_per_pagedir);
1384
1385 #if defined(__i386__)
1386         phys = pt->pt_dir_phys & ARCH_VM_ADDR_MASK;
1387 #elif defined(__arm__)
1388         phys = pt->pt_dir_phys & ARM_VM_PTE_MASK;
1389 #endif
1390         assert(pt->pt_dir_phys == phys);
1391         assert(!(pt->pt_dir_phys % ARCH_PAGEDIR_SIZE));
1392
1393         /* Update "page directory pagetable." */
1394 #if defined(__i386__)
1395         page_directories[slot] = phys | ARCH_VM_PDE_PRESENT|ARCH_VM_PTE_RW;
1396 #elif defined(__arm__)
1397         {
1398         int i;
1399         for (i = 0; i < pages_per_pagedir; i++)
1400             page_directories[slot*pages_per_pagedir+i] =
1401                 (phys+i*VM_PAGE_SIZE) |
1402                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_RW |
1403                 ARCH_VM_PTE_USER;
1404         }
1405 #endif
1406
1407         /* This is where the PDE's will be visible to the kernel
1408          * in its address space.
1409          */
1410         pdes = (void *) (pagedir_pde*ARCH_BIG_PAGE_SIZE +
1411 #if defined(__i386__)
1412                         slot * VM_PAGE_SIZE);
1413 #elif defined(__arm__)
1414                         slot * ARCH_PAGEDIR_SIZE);
1415 #endif
1416
1417 #if 0
1418         printf("VM: slot %d endpoint %d has pde val 0x%lx at kernel address 0x%lx\n",
1419                 slot, who->vm_endpoint, page_directories[slot], pdes);
1420 #endif
1421         /* Tell kernel about new page table root. */
1422         return sys_vmctl_set_addrspace(who->vm_endpoint, pt->pt_dir_phys, pdes);
1423 }
1424
1425 /*===========================================================================*
1426  *                              pt_free                                      *
1427  *===========================================================================*/
1428 void pt_free(pt_t *pt)
1429 {
1430 /* Free memory associated with this pagetable. */
1431         int i;
1432
1433         for(i = 0; i < ARCH_VM_DIR_ENTRIES; i++)
1434                 if(pt->pt_pt[i])
1435                         vm_freepages((vir_bytes) pt->pt_pt[i], 1);
1436
1437         return;
1438 }
1439
1440 /*===========================================================================*
1441  *                              pt_mapkernel                                 *
1442  *===========================================================================*/
1443 int pt_mapkernel(pt_t *pt)
1444 {
1445         int i;
1446         int kern_pde = kern_start_pde;
1447         phys_bytes addr, mapped = 0;
1448
1449         /* Any page table needs to map in the kernel address space. */
1450         assert(bigpage_ok);
1451         assert(pagedir_pde >= 0);
1452         assert(kern_pde >= 0);
1453
1454         /* pt_init() has made sure this is ok. */
1455         addr = kern_mb_mod->mod_start;
1456
1457         /* Actually mapping in kernel */
1458         while(mapped < kern_size) {
1459 #if defined(__i386__)
1460                 pt->pt_dir[kern_pde] = addr | ARCH_VM_PDE_PRESENT |
1461                         ARCH_VM_BIGPAGE | ARCH_VM_PTE_RW | global_bit;
1462 #elif defined(__arm__)
1463                 pt->pt_dir[kern_pde] = (addr & ARCH_VM_PDE_MASK) |
1464                         ARM_VM_SECTION |
1465                         ARM_VM_SECTION_DOMAIN | ARM_VM_SECTION_WB |
1466                         ARM_VM_SECTION_SHAREABLE | ARM_VM_SECTION_SUPER;
1467 #endif
1468                 kern_pde++;
1469                 mapped += ARCH_BIG_PAGE_SIZE;
1470                 addr += ARCH_BIG_PAGE_SIZE;
1471         }
1472
1473         /* Kernel also wants to know about all page directories. */
1474         assert(pagedir_pde > kern_pde);
1475         pt->pt_dir[pagedir_pde] = pagedir_pde_val;
1476
1477         /* Kernel also wants various mappings of its own. */
1478         for(i = 0; i < kernmappings; i++) {
1479                 int r;
1480                 if((r=pt_writemap(NULL, pt,
1481                         kern_mappings[i].vir_addr,
1482                         kern_mappings[i].phys_addr,
1483                         kern_mappings[i].len,
1484                         kern_mappings[i].flags, 0)) != OK) {
1485                         return r;
1486                 }
1487
1488 #if defined(__arm__)
1489                 if(kern_mappings[i].phys_addr == 0x48000000) {
1490                         if((r=pt_writemap(NULL, pt,
1491                                 kern_mappings[i].phys_addr,
1492                                 kern_mappings[i].phys_addr,
1493                                 kern_mappings[i].len,
1494                                 kern_mappings[i].flags, 0)) != OK) {
1495                                 return r;
1496                         }
1497                 }
1498 #endif
1499         }
1500
1501         return OK;
1502 }
1503
1504 /*===========================================================================*
1505  *                              pt_cycle                                     *
1506  *===========================================================================*/
1507 void pt_cycle(void)
1508 {
1509         vm_checkspares();
1510 #if defined(__arm__)
1511         vm_checksparedirs();
1512 #endif
1513 }
1514
1515 int get_vm_self_pages(void) { return vm_self_pages; }