servers/vm/arch/i386/pagetable.c

   1
   2 #define _SYSTEM 1
   3 #define _POSIX_SOURCE 1
   4
   5 #include <minix/callnr.h>
   6 #include <minix/com.h>
   7 #include <minix/config.h>
   8 #include <minix/const.h>
   9 #include <minix/ds.h>
  10 #include <minix/endpoint.h>
  11 #include <minix/keymap.h>
  12 #include <minix/minlib.h>
  13 #include <minix/type.h>
  14 #include <minix/ipc.h>
  15 #include <minix/sysutil.h>
  16 #include <minix/syslib.h>
  17 #include <minix/safecopies.h>
  18 #include <minix/cpufeature.h>
  19 #include <minix/bitmap.h>
  20 #include <minix/debug.h>
  21
  22 #include <errno.h>
  23 #include <stdlib.h>
  24 #include <assert.h>
  25 #include <string.h>
  26 #include <env.h>
  27 #include <stdio.h>
  28 #include <fcntl.h>
  29 #include <stdlib.h>
  30
  31 #include "proto.h"
  32 #include "glo.h"
  33 #include "util.h"
  34 #include "vm.h"
  35 #include "sanitycheck.h"
  36
  37 static int vm_self_pages;
  38
  39 /* PDE used to map in kernel, kernel physical address. */
  40 static int pagedir_pde = -1;
  41 static u32_t global_bit = 0, pagedir_pde_val;
  42
  43 static multiboot_module_t *kern_mb_mod = NULL;
  44 static size_t kern_size = 0;
  45 static int kern_start_pde = -1;
  46
  47 /* big page size available in hardware? */
  48 static int bigpage_ok = 1;
  49
  50 /* Our process table entry. */
  51 struct vmproc *vmprocess = &vmproc[VM_PROC_NR];
  52
  53 /* Spare memory, ready to go after initialization, to avoid a
  54  * circular dependency on allocating memory and writing it into VM's
  55  * page table.
  56  */
  57 #if defined(__i386__)
  58 #if SANITYCHECKS
  59 #define SPAREPAGES 100
  60 #define STATIC_SPAREPAGES 90
  61 #else
  62 #define SPAREPAGES 15
  63 #define STATIC_SPAREPAGES 10
  64 #endif
  65 #elif defined(__arm__)
  66 #define SPAREPAGEDIRS 11
  67 #define STATIC_SPAREPAGEDIRS 10
  68 #define SPAREPAGES 250
  69 #define STATIC_SPAREPAGES 100
  70 int missing_sparedirs = SPAREPAGEDIRS;
  71 static struct {
  72         void *pagedir;
  73         phys_bytes phys;
  74 } sparepagedirs[SPAREPAGEDIRS];
  75 #endif
  76
  77 int missing_spares = SPAREPAGES;
  78 static struct {
  79         void *page;
  80         phys_bytes phys;
  81 } sparepages[SPAREPAGES];
  82
  83 extern char _end;
  84 #define is_staticaddr(v) ((vir_bytes) (v) < (vir_bytes) &_end)
  85
  86 #define MAX_KERNMAPPINGS 10
  87 static struct {
  88         phys_bytes      phys_addr;      /* Physical addr. */
  89         phys_bytes      len;            /* Length in bytes. */
  90         vir_bytes       vir_addr;       /* Offset in page table. */
  91         int             flags;
  92 } kern_mappings[MAX_KERNMAPPINGS];
  93 int kernmappings = 0;
  94
  95 /* Clicks must be pages, as
  96  *  - they must be page aligned to map them
  97  *  - they must be a multiple of the page size
  98  *  - it's inconvenient to have them bigger than pages, because we often want
  99  *    just one page
 100  * May as well require them to be equal then.
 101  */
 102 #if CLICK_SIZE != VM_PAGE_SIZE
 103 #error CLICK_SIZE must be page size.
 104 #endif
 105
 106 /* Page table that contains pointers to all page directories. */
 107 phys_bytes page_directories_phys;
 108 u32_t *page_directories = NULL;
 109
 110 static char static_sparepages[VM_PAGE_SIZE*STATIC_SPAREPAGES]
 111         __aligned(VM_PAGE_SIZE);
 112
 113 #if defined(__arm__)
 114 static char static_sparepagedirs[ARCH_PAGEDIR_SIZE*STATIC_SPAREPAGEDIRS + ARCH_PAGEDIR_SIZE] __aligned(ARCH_PAGEDIR_SIZE);
 115 #endif
 116
 117 #if SANITYCHECKS
 118 /*===========================================================================*
 119  *                              pt_sanitycheck                               *
 120  *===========================================================================*/
 121 void pt_sanitycheck(pt_t *pt, char *file, int line)
 122 {
 123 /* Basic pt sanity check. */
 124         int slot;
 125
 126         MYASSERT(pt);
 127         MYASSERT(pt->pt_dir);
 128         MYASSERT(pt->pt_dir_phys);
 129
 130         for(slot = 0; slot < ELEMENTS(vmproc); slot++) {
 131                 if(pt == &vmproc[slot].vm_pt)
 132                         break;
 133         }
 134
 135         if(slot >= ELEMENTS(vmproc)) {
 136                 panic("pt_sanitycheck: passed pt not in any proc");
 137         }
 138
 139         MYASSERT(usedpages_add(pt->pt_dir_phys, VM_PAGE_SIZE) == OK);
 140 }
 141 #endif
 142
 143 /*===========================================================================*
 144  *                              findhole                                     *
 145  *===========================================================================*/
 146 #if defined(__i386__)
 147 static u32_t findhole(void)
 148 #elif defined(__arm__)
 149 static u32_t findhole(int pages)
 150 #endif
 151 {
 152 /* Find a space in the virtual address space of VM. */
 153         u32_t curv;
 154         int pde = 0, try_restart;
 155         static u32_t lastv = 0;
 156         pt_t *pt = &vmprocess->vm_pt;
 157         vir_bytes vmin, vmax;
 158 #if defined(__arm__)
 159         u32_t holev;
 160 #endif
 161
 162         vmin = (vir_bytes) (&_end) & ARCH_VM_ADDR_MASK; /* marks end of VM BSS */
 163         vmax = VM_STACKTOP;
 164
 165         /* Input sanity check. */
 166         assert(vmin + VM_PAGE_SIZE >= vmin);
 167         assert(vmax >= vmin + VM_PAGE_SIZE);
 168         assert((vmin % VM_PAGE_SIZE) == 0);
 169         assert((vmax % VM_PAGE_SIZE) == 0);
 170 #if defined(__arm__)
 171         assert(pages > 0);
 172 #endif
 173
 174 #if SANITYCHECKS
 175         curv = ((u32_t) random()) % ((vmax - vmin)/VM_PAGE_SIZE);
 176         curv *= VM_PAGE_SIZE;
 177         curv += vmin;
 178 #else
 179         curv = lastv;
 180         if(curv < vmin || curv >= vmax)
 181                 curv = vmin;
 182 #endif
 183         try_restart = 1;
 184
 185         /* Start looking for a free page starting at vmin. */
 186         while(curv < vmax) {
 187                 int pte;
 188 #if defined(__arm__)
 189                 int i, nohole;
 190 #endif
 191
 192                 assert(curv >= vmin);
 193                 assert(curv < vmax);
 194
 195 #if defined(__i386__)
 196                 pde = I386_VM_PDE(curv);
 197                 pte = I386_VM_PTE(curv);
 198 #elif defined(__arm__)
 199                 holev = curv; /* the candidate hole */
 200                 nohole = 0;
 201                 for (i = 0; i < pages && !nohole; ++i) {
 202                     if(curv >= vmax) {
 203                         break;
 204                     }
 205 #endif
 206
 207 #if defined(__i386__)
 208                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT) ||
 209                    !(pt->pt_pt[pde][pte] & ARCH_VM_PAGE_PRESENT)) {
 210 #elif defined(__arm__)
 211                     pde = ARM_VM_PDE(curv);
 212                     pte = ARM_VM_PTE(curv);
 213
 214                     /* if page present, no hole */
 215                     if((pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT) &&
 216                        (pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT))
 217                         nohole = 1;
 218
 219                     /* if not contiguous, no hole */
 220                     if (curv != holev + i * VM_PAGE_SIZE)
 221                         nohole = 1;
 222
 223                     curv+=VM_PAGE_SIZE;
 224                 }
 225
 226                 /* there's a large enough hole */
 227                 if (!nohole && i == pages) {
 228 #endif
 229                         lastv = curv;
 230 #if defined(__i386__)
 231                         return curv;
 232 #elif defined(__arm__)
 233                         return holev;
 234 #endif
 235                 }
 236
 237 #if defined(__i386__)
 238                 curv+=VM_PAGE_SIZE;
 239
 240 #elif defined(__arm__)
 241                 /* Reset curv */
 242 #endif
 243                 if(curv >= vmax && try_restart) {
 244                         curv = vmin;
 245                         try_restart = 0;
 246                 }
 247         }
 248
 249         printf("VM: out of virtual address space in vm\n");
 250
 251         return NO_MEM;
 252 }
 253
 254 /*===========================================================================*
 255  *                              vm_freepages                                 *
 256  *===========================================================================*/
 257 void vm_freepages(vir_bytes vir, int pages)
 258 {
 259         assert(!(vir % VM_PAGE_SIZE));
 260
 261         if(is_staticaddr(vir)) {
 262                 printf("VM: not freeing static page\n");
 263                 return;
 264         }
 265
 266         if(pt_writemap(vmprocess, &vmprocess->vm_pt, vir,
 267                 MAP_NONE, pages*VM_PAGE_SIZE, 0,
 268                 WMF_OVERWRITE | WMF_FREE) != OK)
 269                 panic("vm_freepages: pt_writemap failed");
 270
 271         vm_self_pages--;
 272
 273 #if SANITYCHECKS
 274         /* If SANITYCHECKS are on, flush tlb so accessing freed pages is
 275          * always trapped, also if not in tlb.
 276          */
 277         if((sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 278                 panic("VMCTL_FLUSHTLB failed");
 279         }
 280 #endif
 281 }
 282
 283 /*===========================================================================*
 284  *                              vm_getsparepage                              *
 285  *===========================================================================*/
 286 static void *vm_getsparepage(phys_bytes *phys)
 287 {
 288         int s;
 289         assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
 290         for(s = 0; s < SPAREPAGES; s++) {
 291                 if(sparepages[s].page) {
 292                         void *sp;
 293                         sp = sparepages[s].page;
 294                         *phys = sparepages[s].phys;
 295                         sparepages[s].page = NULL;
 296                         missing_spares++;
 297                         assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
 298                         return sp;
 299                 }
 300         }
 301         return NULL;
 302 }
 303
 304 #if defined(__arm__)
 305 /*===========================================================================*
 306  *                              vm_getsparepagedir                           *
 307  *===========================================================================*/
 308 static void *vm_getsparepagedir(phys_bytes *phys)
 309 {
 310         int s;
 311         assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
 312         for(s = 0; s < SPAREPAGEDIRS; s++) {
 313                 if(sparepagedirs[s].pagedir) {
 314                         void *sp;
 315                         sp = sparepagedirs[s].pagedir;
 316                         *phys = sparepagedirs[s].phys;
 317                         sparepagedirs[s].pagedir = NULL;
 318                         missing_sparedirs++;
 319                         assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
 320                         return sp;
 321                 }
 322         }
 323         return NULL;
 324 }
 325 #endif
 326
 327 /*===========================================================================*
 328  *                              vm_checkspares                               *
 329  *===========================================================================*/
 330 static void *vm_checkspares(void)
 331 {
 332         int s, n = 0;
 333         static int total = 0, worst = 0;
 334         assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
 335         for(s = 0; s < SPAREPAGES && missing_spares > 0; s++)
 336             if(!sparepages[s].page) {
 337                 n++;
 338                 if((sparepages[s].page = vm_allocpage(&sparepages[s].phys,
 339                         VMP_SPARE))) {
 340                         missing_spares--;
 341                         assert(missing_spares >= 0);
 342                         assert(missing_spares <= SPAREPAGES);
 343                 } else {
 344                         printf("VM: warning: couldn't get new spare page\n");
 345                 }
 346         }
 347         if(worst < n) worst = n;
 348         total += n;
 349
 350         return NULL;
 351 }
 352
 353 #if defined(__arm__)
 354 /*===========================================================================*
 355  *                              vm_checksparedirs                            *
 356  *===========================================================================*/
 357 static void *vm_checksparedirs(void)
 358 {
 359         int s, n = 0;
 360         static int total = 0, worst = 0;
 361         assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
 362         for(s = 0; s < SPAREPAGEDIRS && missing_sparedirs > 0; s++)
 363             if(!sparepagedirs[s].pagedir) {
 364                 n++;
 365                 if((sparepagedirs[s].pagedir = vm_allocpage(&sparepagedirs[s].phys,
 366                         VMP_SPARE))) {
 367                         missing_sparedirs--;
 368                         assert(missing_sparedirs >= 0);
 369                         assert(missing_sparedirs <= SPAREPAGEDIRS);
 370                 } else {
 371                         printf("VM: warning: couldn't get new spare pagedir\n");
 372                 }
 373         }
 374         if(worst < n) worst = n;
 375         total += n;
 376
 377         return NULL;
 378 }
 379
 380 #endif
 381 static int pt_init_done;
 382
 383 /*===========================================================================*
 384  *                              vm_allocpage                                 *
 385  *===========================================================================*/
 386 void *vm_allocpage(phys_bytes *phys, int reason)
 387 {
 388 /* Allocate a page for use by VM itself. */
 389         phys_bytes newpage;
 390         vir_bytes loc;
 391         pt_t *pt;
 392         int r;
 393         static int level = 0;
 394         void *ret;
 395 #if defined(__arm__)
 396         u32_t mem_bytes, mem_clicks, mem_flags;
 397 #endif
 398
 399         pt = &vmprocess->vm_pt;
 400         assert(reason >= 0 && reason < VMP_CATEGORIES);
 401
 402         level++;
 403
 404         assert(level >= 1);
 405         assert(level <= 2);
 406
 407         if((level > 1) || !pt_init_done) {
 408                 void *s;
 409 #if defined(__i386__)
 410                 s=vm_getsparepage(phys);
 411 #elif defined(__arm__)
 412
 413                 if (reason == VMP_PAGEDIR)
 414                         s=vm_getsparepagedir(phys);
 415                 else
 416                         s=vm_getsparepage(phys);
 417
 418 #endif
 419                 level--;
 420                 if(!s) {
 421                         util_stacktrace();
 422                         printf("VM: warning: out of spare pages\n");
 423                 }
 424                 if(!is_staticaddr(s)) vm_self_pages++;
 425                 return s;
 426         }
 427
 428 #if defined(__arm__)
 429         if (reason == VMP_PAGEDIR) {
 430                 mem_bytes = ARCH_PAGEDIR_SIZE;
 431                 mem_flags = PAF_ALIGN16K;
 432         } else {
 433                 mem_bytes = VM_PAGE_SIZE;
 434                 mem_flags = 0;
 435         }
 436         mem_clicks = mem_bytes / VM_PAGE_SIZE * CLICKSPERPAGE;
 437
 438 #endif
 439         /* VM does have a pagetable, so get a page and map it in there.
 440          * Where in our virtual address space can we put it?
 441          */
 442 #if defined(__i386__)
 443         loc = findhole();
 444 #elif defined(__arm__)
 445         loc = findhole(mem_bytes / VM_PAGE_SIZE);
 446 #endif
 447         if(loc == NO_MEM) {
 448                 level--;
 449                 printf("VM: vm_allocpage: findhole failed\n");
 450                 return NULL;
 451         }
 452
 453         /* Allocate page of memory for use by VM. As VM
 454          * is trusted, we don't have to pre-clear it.
 455          */
 456 #if defined(__i386__)
 457         if((newpage = alloc_mem(CLICKSPERPAGE, 0)) == NO_MEM) {
 458 #elif defined(__arm__)
 459         if((newpage = alloc_mem(mem_clicks, mem_flags)) == NO_MEM) {
 460 #endif
 461                 level--;
 462                 printf("VM: vm_allocpage: alloc_mem failed\n");
 463                 return NULL;
 464         }
 465
 466         *phys = CLICK2ABS(newpage);
 467
 468         /* Map this page into our address space. */
 469 #if defined(__i386__)
 470         if((r=pt_writemap(vmprocess, pt, loc, *phys, VM_PAGE_SIZE,
 471                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW, 0)) != OK) {
 472                 free_mem(newpage, CLICKSPERPAGE);
 473 #elif defined(__arm__)
 474         if((r=pt_writemap(vmprocess, pt, loc, *phys, mem_bytes,
 475                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW |
 476                 ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE, 0)) != OK) {
 477                 free_mem(newpage, mem_clicks);
 478 #endif
 479                 printf("vm_allocpage writemap failed\n");
 480                 level--;
 481                 return NULL;
 482         }
 483
 484         if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 485                 panic("VMCTL_FLUSHTLB failed: %d", r);
 486         }
 487
 488         level--;
 489
 490         /* Return user-space-ready pointer to it. */
 491         ret = (void *) loc;
 492
 493         vm_self_pages++;
 494         return ret;
 495 }
 496
 497 /*===========================================================================*
 498  *                              vm_pagelock                                  *
 499  *===========================================================================*/
 500 void vm_pagelock(void *vir, int lockflag)
 501 {
 502 /* Mark a page allocated by vm_allocpage() unwritable, i.e. only for VM. */
 503         vir_bytes m = (vir_bytes) vir;
 504         int r;
 505         u32_t flags = ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER;
 506         pt_t *pt;
 507
 508         pt = &vmprocess->vm_pt;
 509
 510         assert(!(m % VM_PAGE_SIZE));
 511
 512         if(!lockflag)
 513                 flags |= ARCH_VM_PTE_RW;
 514 #if defined(__arm__)
 515         else
 516                 flags |= ARCH_VM_PTE_RO;
 517         flags |= ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE;
 518 #endif
 519
 520         /* Update flags. */
 521         if((r=pt_writemap(vmprocess, pt, m, 0, VM_PAGE_SIZE,
 522                 flags, WMF_OVERWRITE | WMF_WRITEFLAGSONLY)) != OK) {
 523                 panic("vm_lockpage: pt_writemap failed");
 524         }
 525
 526         if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 527                 panic("VMCTL_FLUSHTLB failed: %d", r);
 528         }
 529
 530         return;
 531 }
 532
 533 /*===========================================================================*
 534  *                              vm_addrok                                    *
 535  *===========================================================================*/
 536 int vm_addrok(void *vir, int writeflag)
 537 {
 538         pt_t *pt = &vmprocess->vm_pt;
 539         int pde, pte;
 540         vir_bytes v = (vir_bytes) vir;
 541
 542 #if defined(__i386__)
 543         pde = I386_VM_PDE(v);
 544         pte = I386_VM_PTE(v);
 545 #elif defined(__arm__)
 546         pde = ARM_VM_PDE(v);
 547         pte = ARM_VM_PTE(v);
 548 #endif
 549
 550         if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 551                 printf("addr not ok: missing pde %d\n", pde);
 552                 return 0;
 553         }
 554
 555 #if defined(__i386__)
 556         if(writeflag &&
 557                 !(pt->pt_dir[pde] & ARCH_VM_PTE_RW)) {
 558                 printf("addr not ok: pde %d present but pde unwritable\n", pde);
 559                 return 0;
 560         }
 561
 562 #endif
 563         if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) {
 564                 printf("addr not ok: missing pde %d / pte %d\n",
 565                         pde, pte);
 566                 return 0;
 567         }
 568
 569 #if defined(__i386__)
 570         if(writeflag &&
 571                 !(pt->pt_pt[pde][pte] & ARCH_VM_PTE_RW)) {
 572                 printf("addr not ok: pde %d / pte %d present but unwritable\n",
 573 #elif defined(__arm__)
 574         if(!writeflag &&
 575                 !(pt->pt_pt[pde][pte] & ARCH_VM_PTE_RO)) {
 576                 printf("addr not ok: pde %d / pte %d present but writable\n",
 577 #endif
 578                         pde, pte);
 579                 return 0;
 580         }
 581
 582         return 1;
 583 }
 584
 585 /*===========================================================================*
 586  *                              pt_ptalloc                                   *
 587  *===========================================================================*/
 588 static int pt_ptalloc(pt_t *pt, int pde, u32_t flags)
 589 {
 590 /* Allocate a page table and write its address into the page directory. */
 591         int i;
 592         phys_bytes pt_phys;
 593
 594         /* Argument must make sense. */
 595         assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES);
 596         assert(!(flags & ~(PTF_ALLFLAGS)));
 597
 598         /* We don't expect to overwrite page directory entry, nor
 599          * storage for the page table.
 600          */
 601         assert(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT));
 602         assert(!pt->pt_pt[pde]);
 603
 604         /* Get storage for the page table. */
 605         if(!(pt->pt_pt[pde] = vm_allocpage(&pt_phys, VMP_PAGETABLE)))
 606                 return ENOMEM;
 607
 608         for(i = 0; i < ARCH_VM_PT_ENTRIES; i++)
 609                 pt->pt_pt[pde][i] = 0;  /* Empty entry. */
 610
 611         /* Make page directory entry.
 612          * The PDE is always 'present,' 'writable,' and 'user accessible,'
 613          * relying on the PTE for protection.
 614          */
 615 #if defined(__i386__)
 616         pt->pt_dir[pde] = (pt_phys & ARCH_VM_ADDR_MASK) | flags
 617                 | ARCH_VM_PDE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW;
 618 #elif defined(__arm__)
 619         pt->pt_dir[pde] = (pt_phys & ARCH_VM_PDE_MASK)
 620                 | ARCH_VM_PDE_PRESENT | ARM_VM_PDE_DOMAIN;
 621 #endif
 622
 623         return OK;
 624 }
 625
 626 /*===========================================================================*
 627  *                          pt_ptalloc_in_range                              *
 628  *===========================================================================*/
 629 int pt_ptalloc_in_range(pt_t *pt, vir_bytes start, vir_bytes end,
 630         u32_t flags, int verify)
 631 {
 632 /* Allocate all the page tables in the range specified. */
 633         int pde, first_pde, last_pde;
 634
 635 #if defined(__i386__)
 636         first_pde = I386_VM_PDE(start);
 637         last_pde = I386_VM_PDE(end-1);
 638 #elif defined(__arm__)
 639         first_pde = ARM_VM_PDE(start);
 640         last_pde = ARM_VM_PDE(end-1);
 641 #endif
 642         assert(first_pde >= 0);
 643         assert(last_pde < ARCH_VM_DIR_ENTRIES);
 644
 645         /* Scan all page-directory entries in the range. */
 646         for(pde = first_pde; pde <= last_pde; pde++) {
 647                 assert(!(pt->pt_dir[pde] & ARCH_VM_BIGPAGE));
 648                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 649                         int r;
 650                         if(verify) {
 651                                 printf("pt_ptalloc_in_range: no pde %d\n", pde);
 652                                 return EFAULT;
 653                         }
 654                         assert(!pt->pt_dir[pde]);
 655                         if((r=pt_ptalloc(pt, pde, flags)) != OK) {
 656                                 /* Couldn't do (complete) mapping.
 657                                  * Don't bother freeing any previously
 658                                  * allocated page tables, they're
 659                                  * still writable, don't point to nonsense,
 660                                  * and pt_ptalloc leaves the directory
 661                                  * and other data in a consistent state.
 662                                  */
 663                                 return r;
 664                         }
 665                 }
 666                 assert(pt->pt_dir[pde]);
 667                 assert(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT);
 668         }
 669
 670         return OK;
 671 }
 672
 673 static char *ptestr(u32_t pte)
 674 {
 675 #define FLAG(constant, name) {                                          \
 676         if(pte & (constant)) { strcat(str, name); strcat(str, " "); }   \
 677 }
 678
 679         static char str[30];
 680         if(!(pte & ARCH_VM_PTE_PRESENT)) {
 681                 return "not present";
 682         }
 683         str[0] = '\0';
 684 #if defined(__i386__)
 685         FLAG(ARCH_VM_PTE_RW, "W");
 686 #elif defined(__arm__)
 687         if(pte & ARCH_VM_PTE_RO) {
 688             strcat(str, "R ");
 689         } else {
 690             strcat(str, "W ");
 691         }
 692 #endif
 693         FLAG(ARCH_VM_PTE_USER, "U");
 694 #if defined(__i386__)
 695         FLAG(I386_VM_PWT, "PWT");
 696         FLAG(I386_VM_PCD, "PCD");
 697         FLAG(I386_VM_ACC, "ACC");
 698         FLAG(I386_VM_DIRTY, "DIRTY");
 699         FLAG(I386_VM_PS, "PS");
 700         FLAG(I386_VM_GLOBAL, "G");
 701         FLAG(I386_VM_PTAVAIL1, "AV1");
 702         FLAG(I386_VM_PTAVAIL2, "AV2");
 703         FLAG(I386_VM_PTAVAIL3, "AV3");
 704 #elif defined(__arm__)
 705         FLAG(ARM_VM_PTE_SUPER, "S");
 706         FLAG(ARM_VM_PTE_SHAREABLE, "SH");
 707         FLAG(ARM_VM_PTE_WB, "WB");
 708         FLAG(ARM_VM_PTE_WT, "WT");
 709 #endif
 710
 711         return str;
 712 }
 713
 714 /*===========================================================================*
 715  *                           pt_map_in_range                                 *
 716  *===========================================================================*/
 717 int pt_map_in_range(struct vmproc *src_vmp, struct vmproc *dst_vmp,
 718         vir_bytes start, vir_bytes end)
 719 {
 720 /* Transfer all the mappings from the pt of the source process to the pt of
 721  * the destination process in the range specified.
 722  */
 723         int pde, pte;
 724         vir_bytes viraddr;
 725         pt_t *pt, *dst_pt;
 726
 727         pt = &src_vmp->vm_pt;
 728         dst_pt = &dst_vmp->vm_pt;
 729
 730         end = end ? end : VM_DATATOP;
 731         assert(start % VM_PAGE_SIZE == 0);
 732         assert(end % VM_PAGE_SIZE == 0);
 733 #if defined(__i386__)
 734         assert(start <= end);
 735         assert(I386_VM_PDE(end) < ARCH_VM_DIR_ENTRIES);
 736 #elif defined(__arm__)
 737         assert(ARM_VM_PDE(start) >= 0 && start <= end);
 738         assert(ARM_VM_PDE(end) < ARCH_VM_DIR_ENTRIES);
 739 #endif
 740
 741 #if LU_DEBUG
 742         printf("VM: pt_map_in_range: src = %d, dst = %d\n",
 743                 src_vmp->vm_endpoint, dst_vmp->vm_endpoint);
 744         printf("VM: pt_map_in_range: transferring from 0x%08x (pde %d pte %d) to 0x%08x (pde %d pte %d)\n",
 745 #if defined(__i386__)
 746                 start, I386_VM_PDE(start), I386_VM_PTE(start),
 747                 end, I386_VM_PDE(end), I386_VM_PTE(end));
 748 #elif defined(__arm__)
 749                 start, ARM_VM_PDE(start), ARM_VM_PTE(start),
 750                 end, ARM_VM_PDE(end), ARM_VM_PTE(end));
 751 #endif
 752 #endif
 753
 754         /* Scan all page-table entries in the range. */
 755         for(viraddr = start; viraddr <= end; viraddr += VM_PAGE_SIZE) {
 756 #if defined(__i386__)
 757                 pde = I386_VM_PDE(viraddr);
 758 #elif defined(__arm__)
 759                 pde = ARM_VM_PDE(viraddr);
 760 #endif
 761                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 762                         if(viraddr == VM_DATATOP) break;
 763                         continue;
 764                 }
 765 #if defined(__i386__)
 766                 pte = I386_VM_PTE(viraddr);
 767 #elif defined(__arm__)
 768                 pte = ARM_VM_PTE(viraddr);
 769 #endif
 770                 if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) {
 771                         if(viraddr == VM_DATATOP) break;
 772                         continue;
 773                 }
 774
 775                 /* Transfer the mapping. */
 776                 dst_pt->pt_pt[pde][pte] = pt->pt_pt[pde][pte];
 777
 778                 if(viraddr == VM_DATATOP) break;
 779         }
 780
 781         return OK;
 782 }
 783
 784 /*===========================================================================*
 785  *                              pt_ptmap                                     *
 786  *===========================================================================*/
 787 int pt_ptmap(struct vmproc *src_vmp, struct vmproc *dst_vmp)
 788 {
 789 /* Transfer mappings to page dir and page tables from source process and
 790  * destination process. Make sure all the mappings are above the stack, not
 791  * to corrupt valid mappings in the data segment of the destination process.
 792  */
 793         int pde, r;
 794         phys_bytes physaddr;
 795         vir_bytes viraddr;
 796         pt_t *pt;
 797
 798         pt = &src_vmp->vm_pt;
 799
 800 #if LU_DEBUG
 801         printf("VM: pt_ptmap: src = %d, dst = %d\n",
 802                 src_vmp->vm_endpoint, dst_vmp->vm_endpoint);
 803 #endif
 804
 805         /* Transfer mapping to the page directory. */
 806         viraddr = (vir_bytes) pt->pt_dir;
 807         physaddr = pt->pt_dir_phys & ARCH_VM_ADDR_MASK;
 808 #if defined(__i386__)
 809         if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, VM_PAGE_SIZE,
 810                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW,
 811 #elif defined(__arm__)
 812         if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, ARCH_PAGEDIR_SIZE,
 813                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW |
 814                 ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE,
 815 #endif
 816                 WMF_OVERWRITE)) != OK) {
 817                 return r;
 818         }
 819 #if LU_DEBUG
 820         printf("VM: pt_ptmap: transferred mapping to page dir: 0x%08x (0x%08x)\n",
 821                 viraddr, physaddr);
 822 #endif
 823
 824         /* Scan all non-reserved page-directory entries. */
 825         for(pde=0; pde < ARCH_VM_DIR_ENTRIES; pde++) {
 826                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 827                         continue;
 828                 }
 829
 830                 /* Transfer mapping to the page table. */
 831                 viraddr = (vir_bytes) pt->pt_pt[pde];
 832 #if defined(__i386__)
 833                 physaddr = pt->pt_dir[pde] & ARCH_VM_ADDR_MASK;
 834 #elif defined(__arm__)
 835                 physaddr = pt->pt_dir[pde] & ARCH_VM_PDE_MASK;
 836 #endif
 837                 if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, VM_PAGE_SIZE,
 838                         ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW
 839 #ifdef __arm__
 840                         | ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW |
 841                         ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE
 842 #endif
 843                         ,
 844                         WMF_OVERWRITE)) != OK) {
 845                         return r;
 846                 }
 847         }
 848
 849         return OK;
 850 }
 851
 852 void pt_clearmapcache(void)
 853 {
 854         /* Make sure kernel will invalidate tlb when using current
 855          * pagetable (i.e. vm's) to make new mappings before new cr3
 856          * is loaded.
 857          */
 858         if(sys_vmctl(SELF, VMCTL_CLEARMAPCACHE, 0) != OK)
 859                 panic("VMCTL_CLEARMAPCACHE failed");
 860 }
 861
 862 /*===========================================================================*
 863  *                              pt_writemap                                  *
 864  *===========================================================================*/
 865 int pt_writemap(struct vmproc * vmp,
 866                         pt_t *pt,
 867                         vir_bytes v,
 868                         phys_bytes physaddr,
 869                         size_t bytes,
 870                         u32_t flags,
 871                         u32_t writemapflags)
 872 {
 873 /* Write mapping into page table. Allocate a new page table if necessary. */
 874 /* Page directory and table entries for this virtual address. */
 875         int p, pages;
 876         int verify = 0;
 877         int ret = OK;
 878
 879 #ifdef CONFIG_SMP
 880         int vminhibit_clear = 0;
 881         /* FIXME
 882          * don't do it everytime, stop the process only on the first change and
 883          * resume the execution on the last change. Do in a wrapper of this
 884          * function
 885          */
 886         if (vmp && vmp->vm_endpoint != NONE && vmp->vm_endpoint != VM_PROC_NR &&
 887                         !(vmp->vm_flags & VMF_EXITING)) {
 888                 sys_vmctl(vmp->vm_endpoint, VMCTL_VMINHIBIT_SET, 0);
 889                 vminhibit_clear = 1;
 890         }
 891 #endif
 892
 893         if(writemapflags & WMF_VERIFY)
 894                 verify = 1;
 895
 896         assert(!(bytes % VM_PAGE_SIZE));
 897         assert(!(flags & ~(PTF_ALLFLAGS)));
 898
 899         pages = bytes / VM_PAGE_SIZE;
 900
 901         /* MAP_NONE means to clear the mapping. It doesn't matter
 902          * what's actually written into the PTE if PRESENT
 903          * isn't on, so we can just write MAP_NONE into it.
 904          */
 905         assert(physaddr == MAP_NONE || (flags & ARCH_VM_PTE_PRESENT));
 906         assert(physaddr != MAP_NONE || !flags);
 907
 908         /* First make sure all the necessary page tables are allocated,
 909          * before we start writing in any of them, because it's a pain
 910          * to undo our work properly.
 911          */
 912         ret = pt_ptalloc_in_range(pt, v, v + VM_PAGE_SIZE*pages, flags, verify);
 913         if(ret != OK) {
 914                 printf("VM: writemap: pt_ptalloc_in_range failed\n");
 915                 goto resume_exit;
 916         }
 917
 918         /* Now write in them. */
 919         for(p = 0; p < pages; p++) {
 920                 u32_t entry;
 921 #if defined(__i386__)
 922                 int pde = I386_VM_PDE(v);
 923                 int pte = I386_VM_PTE(v);
 924 #elif defined(__arm__)
 925                 int pde = ARM_VM_PDE(v);
 926                 int pte = ARM_VM_PTE(v);
 927 #endif
 928
 929                 assert(!(v % VM_PAGE_SIZE));
 930                 assert(pte >= 0 && pte < ARCH_VM_PT_ENTRIES);
 931                 assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES);
 932
 933                 /* Page table has to be there. */
 934                 assert(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT);
 935
 936                 /* We do not expect it to be a bigpage. */
 937                 assert(!(pt->pt_dir[pde] & ARCH_VM_BIGPAGE));
 938
 939                 /* Make sure page directory entry for this page table
 940                  * is marked present and page table entry is available.
 941                  */
 942                 assert(pt->pt_pt[pde]);
 943
 944 #if SANITYCHECKS
 945                 /* We don't expect to overwrite a page. */
 946                 if(!(writemapflags & (WMF_OVERWRITE|WMF_VERIFY)))
 947                         assert(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT));
 948 #endif
 949                 if(writemapflags & (WMF_WRITEFLAGSONLY|WMF_FREE)) {
 950 #if defined(__i386__)
 951                         physaddr = pt->pt_pt[pde][pte] & ARCH_VM_ADDR_MASK;
 952 #elif defined(__arm__)
 953                         physaddr = pt->pt_pt[pde][pte] & ARM_VM_PTE_MASK;
 954 #endif
 955                 }
 956
 957                 if(writemapflags & WMF_FREE) {
 958                         free_mem(ABS2CLICK(physaddr), 1);
 959                 }
 960
 961                 /* Entry we will write. */
 962 #if defined(__i386__)
 963                 entry = (physaddr & ARCH_VM_ADDR_MASK) | flags;
 964 #elif defined(__arm__)
 965                 entry = (physaddr & ARM_VM_PTE_MASK) | flags;
 966 #endif
 967
 968                 if(verify) {
 969                         u32_t maskedentry;
 970                         maskedentry = pt->pt_pt[pde][pte];
 971 #if defined(__i386__)
 972                         maskedentry &= ~(I386_VM_ACC|I386_VM_DIRTY);
 973 #endif
 974                         /* Verify pagetable entry. */
 975                         if(entry & ARCH_VM_PTE_RW) {
 976                                 /* If we expect a writable page, allow a readonly page. */
 977                                 maskedentry |= ARCH_VM_PTE_RW;
 978                         }
 979                         if(maskedentry != entry) {
 980                                 printf("pt_writemap: mismatch: ");
 981 #if defined(__i386__)
 982                                 if((entry & ARCH_VM_ADDR_MASK) !=
 983                                         (maskedentry & ARCH_VM_ADDR_MASK)) {
 984 #elif defined(__arm__)
 985                                 if((entry & ARM_VM_PTE_MASK) !=
 986                                         (maskedentry & ARM_VM_PTE_MASK)) {
 987 #endif
 988                                         printf("pt_writemap: physaddr mismatch (0x%lx, 0x%lx); ",
 989                                                 (long)entry, (long)maskedentry);
 990                                 } else printf("phys ok; ");
 991                                 printf(" flags: found %s; ",
 992                                         ptestr(pt->pt_pt[pde][pte]));
 993                                 printf(" masked %s; ",
 994                                         ptestr(maskedentry));
 995                                 printf(" expected %s\n", ptestr(entry));
 996                                 ret = EFAULT;
 997                                 goto resume_exit;
 998                         }
 999                 } else {
1000                         /* Write pagetable entry. */
1001                         pt->pt_pt[pde][pte] = entry;
1002                 }
1003
1004                 physaddr += VM_PAGE_SIZE;
1005                 v += VM_PAGE_SIZE;
1006         }
1007
1008 resume_exit:
1009
1010 #ifdef CONFIG_SMP
1011         if (vminhibit_clear) {
1012                 assert(vmp && vmp->vm_endpoint != NONE && vmp->vm_endpoint != VM_PROC_NR &&
1013                         !(vmp->vm_flags & VMF_EXITING));
1014                 sys_vmctl(vmp->vm_endpoint, VMCTL_VMINHIBIT_CLEAR, 0);
1015         }
1016 #endif
1017
1018         return ret;
1019 }
1020
1021 /*===========================================================================*
1022  *                              pt_checkrange                                *
1023  *===========================================================================*/
1024 int pt_checkrange(pt_t *pt, vir_bytes v,  size_t bytes,
1025         int write)
1026 {
1027         int p, pages;
1028
1029         assert(!(bytes % VM_PAGE_SIZE));
1030
1031         pages = bytes / VM_PAGE_SIZE;
1032
1033         for(p = 0; p < pages; p++) {
1034 #if defined(__i386__)
1035                 int pde = I386_VM_PDE(v);
1036                 int pte = I386_VM_PTE(v);
1037 #elif defined(__arm__)
1038                 int pde = ARM_VM_PDE(v);
1039                 int pte = ARM_VM_PTE(v);
1040 #endif
1041
1042                 assert(!(v % VM_PAGE_SIZE));
1043                 assert(pte >= 0 && pte < ARCH_VM_PT_ENTRIES);
1044                 assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES);
1045
1046                 /* Page table has to be there. */
1047                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT))
1048                         return EFAULT;
1049
1050                 /* Make sure page directory entry for this page table
1051                  * is marked present and page table entry is available.
1052                  */
1053                 assert((pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT) && pt->pt_pt[pde]);
1054
1055                 if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) {
1056                         return EFAULT;
1057                 }
1058
1059 #if defined(__i386__)
1060                 if(write && !(pt->pt_pt[pde][pte] & ARCH_VM_PTE_RW)) {
1061 #elif defined(__arm__)
1062                 if(write && (pt->pt_pt[pde][pte] & ARCH_VM_PTE_RO)) {
1063 #endif
1064                         return EFAULT;
1065                 }
1066
1067                 v += VM_PAGE_SIZE;
1068         }
1069
1070         return OK;
1071 }
1072
1073 /*===========================================================================*
1074  *                              pt_new                                       *
1075  *===========================================================================*/
1076 int pt_new(pt_t *pt)
1077 {
1078 /* Allocate a pagetable root. Allocate a page-aligned page directory
1079  * and set them to 0 (indicating no page tables are allocated). Lookup
1080  * its physical address as we'll need that in the future. Verify it's
1081  * page-aligned.
1082  */
1083         int i, r;
1084
1085         /* Don't ever re-allocate/re-move a certain process slot's
1086          * page directory once it's been created. This is a fraction
1087          * faster, but also avoids having to invalidate the page
1088          * mappings from in-kernel page tables pointing to
1089          * the page directories (the page_directories data).
1090          */
1091         if(!pt->pt_dir &&
1092           !(pt->pt_dir = vm_allocpage((phys_bytes *)&pt->pt_dir_phys, VMP_PAGEDIR))) {
1093                 return ENOMEM;
1094         }
1095 #if defined(__arm__)
1096         assert(!((u32_t)pt->pt_dir_phys % ARCH_PAGEDIR_SIZE));
1097 #endif
1098
1099         for(i = 0; i < ARCH_VM_DIR_ENTRIES; i++) {
1100                 pt->pt_dir[i] = 0; /* invalid entry (PRESENT bit = 0) */
1101                 pt->pt_pt[i] = NULL;
1102         }
1103
1104         /* Where to start looking for free virtual address space? */
1105         pt->pt_virtop = 0;
1106
1107         /* Map in kernel. */
1108         if((r=pt_mapkernel(pt)) != OK)
1109                 return r;
1110
1111         return OK;
1112 }
1113
1114 static int freepde(void)
1115 {
1116         int p = kernel_boot_info.freepde_start++;
1117         assert(kernel_boot_info.freepde_start < ARCH_VM_DIR_ENTRIES);
1118         return p;
1119 }
1120
1121 /*===========================================================================*
1122  *                              pt_init                                      *
1123  *===========================================================================*/
1124 void pt_init(void)
1125 {
1126         pt_t *newpt;
1127         int s, r, p;
1128         int global_bit_ok = 0;
1129         vir_bytes sparepages_mem;
1130 #if defined(__arm__)
1131         vir_bytes sparepagedirs_mem;
1132 #endif
1133         static u32_t currentpagedir[ARCH_VM_DIR_ENTRIES];
1134         int m = kernel_boot_info.kern_mod;
1135 #if defined(__i386__)
1136         u32_t mypdbr; /* Page Directory Base Register (cr3) value */
1137 #elif defined(__arm__)
1138         u32_t myttbr;
1139 #endif
1140
1141         /* Find what the physical location of the kernel is. */
1142         assert(m >= 0);
1143         assert(m < kernel_boot_info.mods_with_kernel);
1144         assert(kernel_boot_info.mods_with_kernel < MULTIBOOT_MAX_MODS);
1145         kern_mb_mod = &kernel_boot_info.module_list[m];
1146         kern_size = kern_mb_mod->mod_end - kern_mb_mod->mod_start;
1147         assert(!(kern_mb_mod->mod_start % ARCH_BIG_PAGE_SIZE));
1148         assert(!(kernel_boot_info.vir_kern_start % ARCH_BIG_PAGE_SIZE));
1149         kern_start_pde = kernel_boot_info.vir_kern_start / ARCH_BIG_PAGE_SIZE;
1150
1151         /* Get ourselves spare pages. */
1152         sparepages_mem = (vir_bytes) static_sparepages;
1153         assert(!(sparepages_mem % VM_PAGE_SIZE));
1154
1155 #if defined(__arm__)
1156         /* Get ourselves spare pagedirs. */
1157         sparepagedirs_mem = (vir_bytes) static_sparepagedirs;
1158         assert(!(sparepagedirs_mem % ARCH_PAGEDIR_SIZE));
1159 #endif
1160
1161         /* Spare pages are used to allocate memory before VM has its own page
1162          * table that things (i.e. arbitrary physical memory) can be mapped into.
1163          * We get it by pre-allocating it in our bss (allocated and mapped in by
1164          * the kernel) in static_sparepages. We also need the physical addresses
1165          * though; we look them up now so they are ready for use.
1166          */
1167 #if defined(__arm__)
1168         missing_sparedirs = 0;
1169         assert(STATIC_SPAREPAGEDIRS < SPAREPAGEDIRS);
1170         for(s = 0; s < SPAREPAGEDIRS; s++) {
1171                 vir_bytes v = (sparepagedirs_mem + s*ARCH_PAGEDIR_SIZE);;
1172                 phys_bytes ph;
1173                 if((r=sys_umap(SELF, VM_D, (vir_bytes) v,
1174                         ARCH_PAGEDIR_SIZE, &ph)) != OK)
1175                                 panic("pt_init: sys_umap failed: %d", r);
1176                 if(s >= STATIC_SPAREPAGEDIRS) {
1177                         sparepagedirs[s].pagedir = NULL;
1178                         missing_sparedirs++;
1179                         continue;
1180                 }
1181                 sparepagedirs[s].pagedir = (void *) v;
1182                 sparepagedirs[s].phys = ph;
1183         }
1184 #endif
1185
1186         missing_spares = 0;
1187         assert(STATIC_SPAREPAGES < SPAREPAGES);
1188         for(s = 0; s < SPAREPAGES; s++) {
1189                 vir_bytes v = (sparepages_mem + s*VM_PAGE_SIZE);;
1190                 phys_bytes ph;
1191                 if((r=sys_umap(SELF, VM_D, (vir_bytes) v,
1192                         VM_PAGE_SIZE*SPAREPAGES, &ph)) != OK)
1193                                 panic("pt_init: sys_umap failed: %d", r);
1194                 if(s >= STATIC_SPAREPAGES) {
1195                         sparepages[s].page = NULL;
1196                         missing_spares++;
1197                         continue;
1198                 }
1199                 sparepages[s].page = (void *) v;
1200                 sparepages[s].phys = ph;
1201         }
1202
1203 #if defined(__i386__)
1204         /* global bit and 4MB pages available? */
1205         global_bit_ok = _cpufeature(_CPUF_I386_PGE);
1206         bigpage_ok = _cpufeature(_CPUF_I386_PSE);
1207
1208         /* Set bit for PTE's and PDE's if available. */
1209         if(global_bit_ok)
1210                 global_bit = I386_VM_GLOBAL;
1211 #endif
1212
1213         /* Allocate us a page table in which to remember page directory
1214          * pointers.
1215          */
1216         if(!(page_directories = vm_allocpage(&page_directories_phys,
1217                 VMP_PAGETABLE)))
1218                 panic("no virt addr for vm mappings");
1219
1220         memset(page_directories, 0, VM_PAGE_SIZE);
1221
1222         /* Now reserve another pde for kernel's own mappings. */
1223         {
1224                 int kernmap_pde;
1225                 phys_bytes addr, len;
1226                 int flags, index = 0;
1227                 u32_t offset = 0;
1228
1229                 kernmap_pde = freepde();
1230                 offset = kernmap_pde * ARCH_BIG_PAGE_SIZE;
1231
1232                 while(sys_vmctl_get_mapping(index, &addr, &len,
1233                         &flags) == OK)  {
1234                         vir_bytes vir;
1235                         if(index >= MAX_KERNMAPPINGS)
1236                                 panic("VM: too many kernel mappings: %d", index);
1237                         kern_mappings[index].phys_addr = addr;
1238                         kern_mappings[index].len = len;
1239                         kern_mappings[index].flags = flags;
1240 #if defined(__i386__)
1241                         kern_mappings[index].vir_addr = offset;
1242 #elif defined(__arm__)
1243                         kern_mappings[index].vir_addr = addr;
1244 #endif
1245                         kern_mappings[index].flags =
1246                                 ARCH_VM_PTE_PRESENT;
1247                         if(flags & VMMF_UNCACHED)
1248 #if defined(__i386__)
1249                                 kern_mappings[index].flags |= PTF_NOCACHE;
1250 #elif defined(__arm__)
1251                                 kern_mappings[index].flags |= ARM_VM_PTE_DEVICE;
1252                         else
1253                                 kern_mappings[index].flags |=
1254                                     ARM_VM_PTE_WB | ARM_VM_PTE_SHAREABLE;
1255 #endif
1256                         if(flags & VMMF_USER)
1257                                 kern_mappings[index].flags |= ARCH_VM_PTE_USER;
1258 #if defined(__arm__)
1259                         else
1260                                 kern_mappings[index].flags |= ARM_VM_PTE_SUPER;
1261 #endif
1262                         if(flags & VMMF_WRITE)
1263                                 kern_mappings[index].flags |= ARCH_VM_PTE_RW;
1264 #if defined(__i386__)
1265                         if(flags & VMMF_GLO)
1266                                 kern_mappings[index].flags |= I386_VM_GLOBAL;
1267 #elif defined(__arm__)
1268                         else
1269                                 kern_mappings[index].flags |= ARCH_VM_PTE_RO;
1270 #endif
1271                         if(addr % VM_PAGE_SIZE)
1272                                 panic("VM: addr unaligned: %d", addr);
1273                         if(len % VM_PAGE_SIZE)
1274                                 panic("VM: len unaligned: %d", len);
1275                         vir = offset;
1276                         if(sys_vmctl_reply_mapping(index, vir) != OK)
1277                                 panic("VM: reply failed");
1278                         offset += len;
1279                         index++;
1280                         kernmappings++;
1281                 }
1282         }
1283
1284         /* Find a PDE below processes available for mapping in the
1285          * page directories.
1286          */
1287         pagedir_pde = freepde();
1288 #if defined(__i386__)
1289         pagedir_pde_val = (page_directories_phys & ARCH_VM_ADDR_MASK) |
1290                         ARCH_VM_PDE_PRESENT | ARCH_VM_PTE_RW;
1291 #elif defined(__arm__)
1292         pagedir_pde_val = (page_directories_phys & ARCH_VM_PDE_MASK) |
1293                         ARCH_VM_PDE_PRESENT | ARM_VM_PDE_DOMAIN;
1294 #endif
1295
1296         /* Allright. Now. We have to make our own page directory and page tables,
1297          * that the kernel has already set up, accessible to us. It's easier to
1298          * understand if we just copy all the required pages (i.e. page directory
1299          * and page tables), and set up the pointers as if VM had done it itself.
1300          *
1301          * This allocation will happen without using any page table, and just
1302          * uses spare pages.
1303          */
1304         newpt = &vmprocess->vm_pt;
1305         if(pt_new(newpt) != OK)
1306                 panic("vm pt_new failed");
1307
1308         /* Get our current pagedir so we can see it. */
1309 #if defined(__i386__)
1310         if(sys_vmctl_get_pdbr(SELF, &mypdbr) != OK)
1311 #elif defined(__arm__)
1312         if(sys_vmctl_get_pdbr(SELF, &myttbr) != OK)
1313 #endif
1314                 panic("VM: sys_vmctl_get_pdbr failed");
1315 #if defined(__i386__)
1316         if(sys_vircopy(NONE, mypdbr, SELF,
1317                 (vir_bytes) currentpagedir, VM_PAGE_SIZE) != OK)
1318 #elif defined(__arm__)
1319         if(sys_vircopy(NONE, myttbr, SELF,
1320                 (vir_bytes) currentpagedir, ARCH_PAGEDIR_SIZE) != OK)
1321 #endif
1322                 panic("VM: sys_vircopy failed");
1323
1324         /* We have mapped in kernel ourselves; now copy mappings for VM
1325          * that kernel made, including allocations for BSS. Skip identity
1326          * mapping bits; just map in VM.
1327          */
1328         for(p = 0; p < ARCH_VM_DIR_ENTRIES; p++) {
1329                 u32_t entry = currentpagedir[p];
1330                 phys_bytes ptaddr_kern, ptaddr_us;
1331
1332                 /* BIGPAGEs are kernel mapping (do ourselves) or boot
1333                  * identity mapping (don't want).
1334                  */
1335                 if(!(entry & ARCH_VM_PDE_PRESENT)) continue;
1336                 if((entry & ARCH_VM_BIGPAGE)) continue;
1337
1338                 if(pt_ptalloc(newpt, p, 0) != OK)
1339                         panic("pt_ptalloc failed");
1340                 assert(newpt->pt_dir[p] & ARCH_VM_PDE_PRESENT);
1341
1342 #if defined(__i386__)
1343                 ptaddr_kern = entry & ARCH_VM_ADDR_MASK;
1344                 ptaddr_us = newpt->pt_dir[p] & ARCH_VM_ADDR_MASK;
1345 #elif defined(__arm__)
1346                 ptaddr_kern = entry & ARCH_VM_PDE_MASK;
1347                 ptaddr_us = newpt->pt_dir[p] & ARCH_VM_PDE_MASK;
1348 #endif
1349
1350                 /* Copy kernel-initialized pagetable contents into our
1351                  * normally accessible pagetable.
1352                  */
1353                 if(sys_abscopy(ptaddr_kern, ptaddr_us, VM_PAGE_SIZE) != OK)
1354                         panic("pt_init: abscopy failed");
1355         }
1356
1357         /* Inform kernel vm has a newly built page table. */
1358         assert(vmproc[VM_PROC_NR].vm_endpoint == VM_PROC_NR);
1359         pt_bind(newpt, &vmproc[VM_PROC_NR]);
1360
1361         pt_init_done = 1;
1362
1363         /* All OK. */
1364         return;
1365 }
1366
1367 /*===========================================================================*
1368  *                              pt_bind                                      *
1369  *===========================================================================*/
1370 int pt_bind(pt_t *pt, struct vmproc *who)
1371 {
1372         int slot;
1373         u32_t phys;
1374         void *pdes;
1375         int pages_per_pagedir = ARCH_PAGEDIR_SIZE/VM_PAGE_SIZE;
1376
1377         /* Basic sanity checks. */
1378         assert(who);
1379         assert(who->vm_flags & VMF_INUSE);
1380         assert(pt);
1381
1382         assert(pagedir_pde >= 0);
1383
1384         slot = who->vm_slot;
1385         assert(slot >= 0);
1386         assert(slot < ELEMENTS(vmproc));
1387         assert(slot < ARCH_VM_PT_ENTRIES / pages_per_pagedir);
1388
1389 #if defined(__i386__)
1390         phys = pt->pt_dir_phys & ARCH_VM_ADDR_MASK;
1391 #elif defined(__arm__)
1392         phys = pt->pt_dir_phys & ARM_VM_PTE_MASK;
1393 #endif
1394         assert(pt->pt_dir_phys == phys);
1395         assert(!(pt->pt_dir_phys % ARCH_PAGEDIR_SIZE));
1396
1397         /* Update "page directory pagetable." */
1398 #if defined(__i386__)
1399         page_directories[slot] = phys | ARCH_VM_PDE_PRESENT|ARCH_VM_PTE_RW;
1400 #elif defined(__arm__)
1401         {
1402         int i;
1403         for (i = 0; i < pages_per_pagedir; i++)
1404             page_directories[slot*pages_per_pagedir+i] =
1405                 (phys+i*VM_PAGE_SIZE) |
1406                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_RW |
1407                 ARCH_VM_PTE_USER;
1408         }
1409 #endif
1410
1411         /* This is where the PDE's will be visible to the kernel
1412          * in its address space.
1413          */
1414         pdes = (void *) (pagedir_pde*ARCH_BIG_PAGE_SIZE +
1415 #if defined(__i386__)
1416                         slot * VM_PAGE_SIZE);
1417 #elif defined(__arm__)
1418                         slot * ARCH_PAGEDIR_SIZE);
1419 #endif
1420
1421 #if 0
1422         printf("VM: slot %d endpoint %d has pde val 0x%lx at kernel address 0x%lx\n",
1423                 slot, who->vm_endpoint, page_directories[slot], pdes);
1424 #endif
1425         /* Tell kernel about new page table root. */
1426         return sys_vmctl_set_addrspace(who->vm_endpoint, pt->pt_dir_phys, pdes);
1427 }
1428
1429 /*===========================================================================*
1430  *                              pt_free                                      *
1431  *===========================================================================*/
1432 void pt_free(pt_t *pt)
1433 {
1434 /* Free memory associated with this pagetable. */
1435         int i;
1436
1437         for(i = 0; i < ARCH_VM_DIR_ENTRIES; i++)
1438                 if(pt->pt_pt[i])
1439                         vm_freepages((vir_bytes) pt->pt_pt[i], 1);
1440
1441         return;
1442 }
1443
1444 /*===========================================================================*
1445  *                              pt_mapkernel                                 *
1446  *===========================================================================*/
1447 int pt_mapkernel(pt_t *pt)
1448 {
1449         int i;
1450         int kern_pde = kern_start_pde;
1451         phys_bytes addr, mapped = 0;
1452
1453         /* Any page table needs to map in the kernel address space. */
1454         assert(bigpage_ok);
1455         assert(pagedir_pde >= 0);
1456         assert(kern_pde >= 0);
1457
1458         /* pt_init() has made sure this is ok. */
1459         addr = kern_mb_mod->mod_start;
1460
1461         /* Actually mapping in kernel */
1462         while(mapped < kern_size) {
1463 #if defined(__i386__)
1464                 pt->pt_dir[kern_pde] = addr | ARCH_VM_PDE_PRESENT |
1465                         ARCH_VM_BIGPAGE | ARCH_VM_PTE_RW | global_bit;
1466 #elif defined(__arm__)
1467                 pt->pt_dir[kern_pde] = (addr & ARCH_VM_PDE_MASK) |
1468                         ARM_VM_SECTION |
1469                         ARM_VM_SECTION_DOMAIN | ARM_VM_SECTION_WB |
1470                         ARM_VM_SECTION_SHAREABLE | ARM_VM_SECTION_SUPER;
1471 #endif
1472                 kern_pde++;
1473                 mapped += ARCH_BIG_PAGE_SIZE;
1474                 addr += ARCH_BIG_PAGE_SIZE;
1475         }
1476
1477         /* Kernel also wants to know about all page directories. */
1478         assert(pagedir_pde > kern_pde);
1479         pt->pt_dir[pagedir_pde] = pagedir_pde_val;
1480
1481         /* Kernel also wants various mappings of its own. */
1482         for(i = 0; i < kernmappings; i++) {
1483                 int r;
1484                 if((r=pt_writemap(NULL, pt,
1485                         kern_mappings[i].vir_addr,
1486                         kern_mappings[i].phys_addr,
1487                         kern_mappings[i].len,
1488                         kern_mappings[i].flags, 0)) != OK) {
1489                         return r;
1490                 }
1491         }
1492
1493         return OK;
1494 }
1495
1496 /*===========================================================================*
1497  *                              pt_cycle                                     *
1498  *===========================================================================*/
1499 void pt_cycle(void)
1500 {
1501         vm_checkspares();
1502 #if defined(__arm__)
1503         vm_checksparedirs();
1504 #endif
1505 }
1506
1507 int get_vm_self_pages(void) { return vm_self_pages; }