servers/vm/pagetable.c

   1
   2 #define _SYSTEM 1
   3
   4 #include <minix/callnr.h>
   5 #include <minix/com.h>
   6 #include <minix/config.h>
   7 #include <minix/const.h>
   8 #include <minix/ds.h>
   9 #include <minix/endpoint.h>
  10 #include <minix/minlib.h>
  11 #include <minix/type.h>
  12 #include <minix/ipc.h>
  13 #include <minix/sysutil.h>
  14 #include <minix/syslib.h>
  15 #include <minix/safecopies.h>
  16 #include <minix/cpufeature.h>
  17 #include <minix/bitmap.h>
  18 #include <minix/debug.h>
  19
  20 #include <errno.h>
  21 #include <stdlib.h>
  22 #include <assert.h>
  23 #include <string.h>
  24 #include <env.h>
  25 #include <stdio.h>
  26 #include <fcntl.h>
  27 #include <stdlib.h>
  28
  29 #include "proto.h"
  30 #include "glo.h"
  31 #include "util.h"
  32 #include "vm.h"
  33 #include "sanitycheck.h"
  34
  35 static int vm_self_pages;
  36
  37 /* PDE used to map in kernel, kernel physical address. */
  38 #define MAX_PAGEDIR_PDES 5
  39 static struct pdm {
  40         int             pdeno;
  41         u32_t           val;
  42         phys_bytes      phys;
  43         u32_t           *page_directories;
  44 } pagedir_mappings[MAX_PAGEDIR_PDES];
  45
  46 static multiboot_module_t *kern_mb_mod = NULL;
  47 static size_t kern_size = 0;
  48 static int kern_start_pde = -1;
  49
  50 /* big page size available in hardware? */
  51 static int bigpage_ok = 1;
  52
  53 /* Our process table entry. */
  54 struct vmproc *vmprocess = &vmproc[VM_PROC_NR];
  55
  56 /* Spare memory, ready to go after initialization, to avoid a
  57  * circular dependency on allocating memory and writing it into VM's
  58  * page table.
  59  */
  60 #if SANITYCHECKS
  61 #define SPAREPAGES 200
  62 #define STATIC_SPAREPAGES 190
  63 #else
  64 #ifdef __arm__
  65 # define SPAREPAGES 150
  66 # define STATIC_SPAREPAGES 140
  67 #else
  68 static u32_t global_bit = 0;
  69 # define SPAREPAGES 20
  70 # define STATIC_SPAREPAGES 15
  71 #endif /* __arm__ */
  72 #endif
  73
  74 #define SPAREPAGEDIRS 1
  75 #define STATIC_SPAREPAGEDIRS 1
  76
  77 int missing_sparedirs = SPAREPAGEDIRS;
  78 static struct {
  79         void *pagedir;
  80         phys_bytes phys;
  81 } sparepagedirs[SPAREPAGEDIRS];
  82
  83 extern char _end;
  84 #define is_staticaddr(v) ((vir_bytes) (v) < (vir_bytes) &_end)
  85
  86 #define MAX_KERNMAPPINGS 10
  87 static struct {
  88         phys_bytes      phys_addr;      /* Physical addr. */
  89         phys_bytes      len;            /* Length in bytes. */
  90         vir_bytes       vir_addr;       /* Offset in page table. */
  91         int             flags;
  92 } kern_mappings[MAX_KERNMAPPINGS];
  93 int kernmappings = 0;
  94
  95 /* Clicks must be pages, as
  96  *  - they must be page aligned to map them
  97  *  - they must be a multiple of the page size
  98  *  - it's inconvenient to have them bigger than pages, because we often want
  99  *    just one page
 100  * May as well require them to be equal then.
 101  */
 102 #if CLICK_SIZE != VM_PAGE_SIZE
 103 #error CLICK_SIZE must be page size.
 104 #endif
 105
 106 static void *spare_pagequeue;
 107 static char static_sparepages[VM_PAGE_SIZE*STATIC_SPAREPAGES]
 108         __aligned(VM_PAGE_SIZE);
 109
 110 #if defined(__arm__)
 111 static char static_sparepagedirs[ARCH_PAGEDIR_SIZE*STATIC_SPAREPAGEDIRS + ARCH_PAGEDIR_SIZE] __aligned(ARCH_PAGEDIR_SIZE);
 112 #endif
 113
 114 #if SANITYCHECKS
 115 /*===========================================================================*
 116  *                              pt_sanitycheck                               *
 117  *===========================================================================*/
 118 void pt_sanitycheck(pt_t *pt, char *file, int line)
 119 {
 120 /* Basic pt sanity check. */
 121         int slot;
 122
 123         MYASSERT(pt);
 124         MYASSERT(pt->pt_dir);
 125         MYASSERT(pt->pt_dir_phys);
 126
 127         for(slot = 0; slot < ELEMENTS(vmproc); slot++) {
 128                 if(pt == &vmproc[slot].vm_pt)
 129                         break;
 130         }
 131
 132         if(slot >= ELEMENTS(vmproc)) {
 133                 panic("pt_sanitycheck: passed pt not in any proc");
 134         }
 135
 136         MYASSERT(usedpages_add(pt->pt_dir_phys, VM_PAGE_SIZE) == OK);
 137 }
 138 #endif
 139
 140 /*===========================================================================*
 141  *                              findhole                                     *
 142  *===========================================================================*/
 143 static u32_t findhole(int pages)
 144 {
 145 /* Find a space in the virtual address space of VM. */
 146         u32_t curv;
 147         int pde = 0, try_restart;
 148         static u32_t lastv = 0;
 149         pt_t *pt = &vmprocess->vm_pt;
 150         vir_bytes vmin, vmax;
 151         u32_t holev = NO_MEM;
 152         int holesize = -1;
 153
 154         vmin = (vir_bytes) (&_end); /* marks end of VM BSS */
 155         vmin += 1024*1024*1024; /* reserve 1GB virtual address space for VM heap */
 156         vmin &= ARCH_VM_ADDR_MASK;
 157         vmax = vmin + 100 * 1024 * 1024; /* allow 100MB of address space for VM */
 158
 159         /* Input sanity check. */
 160         assert(vmin + VM_PAGE_SIZE >= vmin);
 161         assert(vmax >= vmin + VM_PAGE_SIZE);
 162         assert((vmin % VM_PAGE_SIZE) == 0);
 163         assert((vmax % VM_PAGE_SIZE) == 0);
 164         assert(pages > 0);
 165
 166         curv = lastv;
 167         if(curv < vmin || curv >= vmax)
 168                 curv = vmin;
 169
 170         try_restart = 1;
 171
 172         /* Start looking for a free page starting at vmin. */
 173         while(curv < vmax) {
 174                 int pte;
 175
 176                 assert(curv >= vmin);
 177                 assert(curv < vmax);
 178
 179                 pde = ARCH_VM_PDE(curv);
 180                 pte = ARCH_VM_PTE(curv);
 181
 182                 if((pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT) &&
 183                    (pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) {
 184                         /* there is a page here - so keep looking for holes */
 185                         holev = NO_MEM;
 186                         holesize = 0;
 187                 } else {
 188                         /* there is no page here - so we have a hole, a bigger
 189                          * one if we already had one
 190                          */
 191                         if(holev == NO_MEM) {
 192                                 holev = curv;
 193                                 holesize = 1;
 194                         } else holesize++;
 195
 196                         assert(holesize > 0);
 197                         assert(holesize <= pages);
 198
 199                         /* if it's big enough, return it */
 200                         if(holesize == pages) {
 201                                 lastv = curv + VM_PAGE_SIZE;
 202                                 return holev;
 203                         }
 204                 }
 205
 206                 curv+=VM_PAGE_SIZE;
 207
 208                 /* if we reached the limit, start scanning from the beginning if
 209                  * we haven't looked there yet
 210                  */
 211                 if(curv >= vmax && try_restart) {
 212                         try_restart = 0;
 213                         curv = vmin;
 214                 }
 215         }
 216
 217         printf("VM: out of virtual address space in vm\n");
 218
 219         return NO_MEM;
 220 }
 221
 222 /*===========================================================================*
 223  *                              vm_freepages                                 *
 224  *===========================================================================*/
 225 void vm_freepages(vir_bytes vir, int pages)
 226 {
 227         assert(!(vir % VM_PAGE_SIZE));
 228
 229         if(is_staticaddr(vir)) {
 230                 printf("VM: not freeing static page\n");
 231                 return;
 232         }
 233
 234         if(pt_writemap(vmprocess, &vmprocess->vm_pt, vir,
 235                 MAP_NONE, pages*VM_PAGE_SIZE, 0,
 236                 WMF_OVERWRITE | WMF_FREE) != OK)
 237                 panic("vm_freepages: pt_writemap failed");
 238
 239         vm_self_pages--;
 240
 241 #if SANITYCHECKS
 242         /* If SANITYCHECKS are on, flush tlb so accessing freed pages is
 243          * always trapped, also if not in tlb.
 244          */
 245         if((sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 246                 panic("VMCTL_FLUSHTLB failed");
 247         }
 248 #endif
 249 }
 250
 251 /*===========================================================================*
 252  *                              vm_getsparepage                              *
 253  *===========================================================================*/
 254 static void *vm_getsparepage(phys_bytes *phys)
 255 {
 256         void *ptr;
 257         if(reservedqueue_alloc(spare_pagequeue, phys, &ptr) != OK) {
 258                 printf("vm_getsparepage: no spare found\n");
 259                 return NULL;
 260         }
 261         assert(ptr);
 262         return ptr;
 263 }
 264
 265 /*===========================================================================*
 266  *                              vm_getsparepagedir                           *
 267  *===========================================================================*/
 268 static void *vm_getsparepagedir(phys_bytes *phys)
 269 {
 270         int s;
 271         assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
 272         for(s = 0; s < SPAREPAGEDIRS; s++) {
 273                 if(sparepagedirs[s].pagedir) {
 274                         void *sp;
 275                         sp = sparepagedirs[s].pagedir;
 276                         *phys = sparepagedirs[s].phys;
 277                         sparepagedirs[s].pagedir = NULL;
 278                         missing_sparedirs++;
 279                         assert(missing_sparedirs >= 0 && missing_sparedirs <= SPAREPAGEDIRS);
 280                         return sp;
 281                 }
 282         }
 283         return NULL;
 284 }
 285
 286 void *vm_mappages(phys_bytes p, int pages)
 287 {
 288         vir_bytes loc;
 289         int r;
 290         pt_t *pt = &vmprocess->vm_pt;
 291
 292         /* Where in our virtual address space can we put it? */
 293         loc = findhole(pages);
 294         if(loc == NO_MEM) {
 295                 printf("vm_mappages: findhole failed\n");
 296                 return NULL;
 297         }
 298
 299         /* Map this page into our address space. */
 300         if((r=pt_writemap(vmprocess, pt, loc, p, VM_PAGE_SIZE*pages,
 301                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW
 302 #if defined(__arm__)
 303                 | ARM_VM_PTE_CACHED
 304 #endif
 305                 , 0)) != OK) {
 306                 printf("vm_mappages writemap failed\n");
 307                 return NULL;
 308         }
 309
 310         if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 311                 panic("VMCTL_FLUSHTLB failed: %d", r);
 312         }
 313
 314         assert(loc);
 315
 316         return (void *) loc;
 317 }
 318
 319 static int pt_init_done;
 320
 321 /*===========================================================================*
 322  *                              vm_allocpage                                 *
 323  *===========================================================================*/
 324 void *vm_allocpages(phys_bytes *phys, int reason, int pages)
 325 {
 326 /* Allocate a page for use by VM itself. */
 327         phys_bytes newpage;
 328         static int level = 0;
 329         void *ret;
 330         u32_t mem_flags = 0;
 331
 332         assert(reason >= 0 && reason < VMP_CATEGORIES);
 333
 334         assert(pages > 0);
 335
 336         level++;
 337
 338         assert(level >= 1);
 339         assert(level <= 2);
 340
 341         if((level > 1) || !pt_init_done) {
 342                 void *s;
 343
 344                 if(pages == 1) s=vm_getsparepage(phys);
 345                 else if(pages == 4) s=vm_getsparepagedir(phys);
 346                 else panic("%d pages", pages);
 347
 348                 level--;
 349                 if(!s) {
 350                         util_stacktrace();
 351                         printf("VM: warning: out of spare pages\n");
 352                 }
 353                 if(!is_staticaddr(s)) vm_self_pages++;
 354                 return s;
 355         }
 356
 357 #if defined(__arm__)
 358         if (reason == VMP_PAGEDIR) {
 359                 mem_flags |= PAF_ALIGN16K;
 360         }
 361 #endif
 362
 363         /* Allocate page of memory for use by VM. As VM
 364          * is trusted, we don't have to pre-clear it.
 365          */
 366         if((newpage = alloc_mem(pages, mem_flags)) == NO_MEM) {
 367                 level--;
 368                 printf("VM: vm_allocpage: alloc_mem failed\n");
 369                 return NULL;
 370         }
 371
 372         *phys = CLICK2ABS(newpage);
 373
 374         if(!(ret = vm_mappages(*phys, pages))) {
 375                 level--;
 376                 printf("VM: vm_allocpage: vm_mappages failed\n");
 377                 return NULL;
 378         }
 379
 380         level--;
 381         vm_self_pages++;
 382
 383         return ret;
 384 }
 385
 386 void *vm_allocpage(phys_bytes *phys, int reason)
 387 {
 388         return vm_allocpages(phys, reason, 1);
 389 }
 390
 391 /*===========================================================================*
 392  *                              vm_pagelock                                  *
 393  *===========================================================================*/
 394 void vm_pagelock(void *vir, int lockflag)
 395 {
 396 /* Mark a page allocated by vm_allocpage() unwritable, i.e. only for VM. */
 397         vir_bytes m = (vir_bytes) vir;
 398         int r;
 399         u32_t flags = ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER;
 400         pt_t *pt;
 401
 402         pt = &vmprocess->vm_pt;
 403
 404         assert(!(m % VM_PAGE_SIZE));
 405
 406         if(!lockflag)
 407                 flags |= ARCH_VM_PTE_RW;
 408 #if defined(__arm__)
 409         else
 410                 flags |= ARCH_VM_PTE_RO;
 411
 412         flags |= ARM_VM_PTE_CACHED ;
 413 #endif
 414
 415         /* Update flags. */
 416         if((r=pt_writemap(vmprocess, pt, m, 0, VM_PAGE_SIZE,
 417                 flags, WMF_OVERWRITE | WMF_WRITEFLAGSONLY)) != OK) {
 418                 panic("vm_lockpage: pt_writemap failed");
 419         }
 420
 421         if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 422                 panic("VMCTL_FLUSHTLB failed: %d", r);
 423         }
 424
 425         return;
 426 }
 427
 428 /*===========================================================================*
 429  *                              vm_addrok                                    *
 430  *===========================================================================*/
 431 int vm_addrok(void *vir, int writeflag)
 432 {
 433         pt_t *pt = &vmprocess->vm_pt;
 434         int pde, pte;
 435         vir_bytes v = (vir_bytes) vir;
 436
 437         pde = ARCH_VM_PDE(v);
 438         pte = ARCH_VM_PTE(v);
 439
 440         if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 441                 printf("addr not ok: missing pde %d\n", pde);
 442                 return 0;
 443         }
 444
 445 #if defined(__i386__)
 446         if(writeflag &&
 447                 !(pt->pt_dir[pde] & ARCH_VM_PTE_RW)) {
 448                 printf("addr not ok: pde %d present but pde unwritable\n", pde);
 449                 return 0;
 450         }
 451 #elif defined(__arm__)
 452         if(writeflag &&
 453                  (pt->pt_dir[pde] & ARCH_VM_PTE_RO)) {
 454                 printf("addr not ok: pde %d present but pde unwritable\n", pde);
 455                 return 0;
 456         }
 457
 458 #endif
 459         if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) {
 460                 printf("addr not ok: missing pde %d / pte %d\n",
 461                         pde, pte);
 462                 return 0;
 463         }
 464
 465 #if defined(__i386__)
 466         if(writeflag &&
 467                 !(pt->pt_pt[pde][pte] & ARCH_VM_PTE_RW)) {
 468                 printf("addr not ok: pde %d / pte %d present but unwritable\n",
 469                         pde, pte);
 470 #elif defined(__arm__)
 471         if(writeflag &&
 472                  (pt->pt_pt[pde][pte] & ARCH_VM_PTE_RO)) {
 473                 printf("addr not ok: pde %d / pte %d present but unwritable\n",
 474                         pde, pte);
 475 #endif
 476                 return 0;
 477         }
 478
 479         return 1;
 480 }
 481
 482 /*===========================================================================*
 483  *                              pt_ptalloc                                   *
 484  *===========================================================================*/
 485 static int pt_ptalloc(pt_t *pt, int pde, u32_t flags)
 486 {
 487 /* Allocate a page table and write its address into the page directory. */
 488         int i;
 489         phys_bytes pt_phys;
 490
 491         /* Argument must make sense. */
 492         assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES);
 493         assert(!(flags & ~(PTF_ALLFLAGS)));
 494
 495         /* We don't expect to overwrite page directory entry, nor
 496          * storage for the page table.
 497          */
 498         assert(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT));
 499         assert(!pt->pt_pt[pde]);
 500
 501         /* Get storage for the page table. */
 502         if(!(pt->pt_pt[pde] = vm_allocpage(&pt_phys, VMP_PAGETABLE)))
 503                 return ENOMEM;
 504
 505         for(i = 0; i < ARCH_VM_PT_ENTRIES; i++)
 506                 pt->pt_pt[pde][i] = 0;  /* Empty entry. */
 507
 508         /* Make page directory entry.
 509          * The PDE is always 'present,' 'writable,' and 'user accessible,'
 510          * relying on the PTE for protection.
 511          */
 512 #if defined(__i386__)
 513         pt->pt_dir[pde] = (pt_phys & ARCH_VM_ADDR_MASK) | flags
 514                 | ARCH_VM_PDE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW;
 515 #elif defined(__arm__)
 516         pt->pt_dir[pde] = (pt_phys & ARCH_VM_PDE_MASK)
 517                 | ARCH_VM_PDE_PRESENT | ARM_VM_PDE_DOMAIN; //LSC FIXME
 518 #endif
 519
 520         return OK;
 521 }
 522
 523 /*===========================================================================*
 524  *                          pt_ptalloc_in_range                              *
 525  *===========================================================================*/
 526 int pt_ptalloc_in_range(pt_t *pt, vir_bytes start, vir_bytes end,
 527         u32_t flags, int verify)
 528 {
 529 /* Allocate all the page tables in the range specified. */
 530         int pde, first_pde, last_pde;
 531
 532         first_pde = ARCH_VM_PDE(start);
 533         last_pde = ARCH_VM_PDE(end-1);
 534
 535         assert(first_pde >= 0);
 536         assert(last_pde < ARCH_VM_DIR_ENTRIES);
 537
 538         /* Scan all page-directory entries in the range. */
 539         for(pde = first_pde; pde <= last_pde; pde++) {
 540                 assert(!(pt->pt_dir[pde] & ARCH_VM_BIGPAGE));
 541                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 542                         int r;
 543                         if(verify) {
 544                                 printf("pt_ptalloc_in_range: no pde %d\n", pde);
 545                                 return EFAULT;
 546                         }
 547                         assert(!pt->pt_dir[pde]);
 548                         if((r=pt_ptalloc(pt, pde, flags)) != OK) {
 549                                 /* Couldn't do (complete) mapping.
 550                                  * Don't bother freeing any previously
 551                                  * allocated page tables, they're
 552                                  * still writable, don't point to nonsense,
 553                                  * and pt_ptalloc leaves the directory
 554                                  * and other data in a consistent state.
 555                                  */
 556                                 return r;
 557                         }
 558                         assert(pt->pt_pt[pde]);
 559                 }
 560                 assert(pt->pt_pt[pde]);
 561                 assert(pt->pt_dir[pde]);
 562                 assert(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT);
 563         }
 564
 565         return OK;
 566 }
 567
 568 static const char *ptestr(u32_t pte)
 569 {
 570 #define FLAG(constant, name) {                                          \
 571         if(pte & (constant)) { strcat(str, name); strcat(str, " "); }   \
 572 }
 573
 574         static char str[30];
 575         if(!(pte & ARCH_VM_PTE_PRESENT)) {
 576                 return "not present";
 577         }
 578         str[0] = '\0';
 579 #if defined(__i386__)
 580         FLAG(ARCH_VM_PTE_RW, "W");
 581 #elif defined(__arm__)
 582         if(pte & ARCH_VM_PTE_RO) {
 583             strcat(str, "R ");
 584         } else {
 585             strcat(str, "W ");
 586         }
 587 #endif
 588         FLAG(ARCH_VM_PTE_USER, "U");
 589 #if defined(__i386__)
 590         FLAG(I386_VM_PWT, "PWT");
 591         FLAG(I386_VM_PCD, "PCD");
 592         FLAG(I386_VM_ACC, "ACC");
 593         FLAG(I386_VM_DIRTY, "DIRTY");
 594         FLAG(I386_VM_PS, "PS");
 595         FLAG(I386_VM_GLOBAL, "G");
 596         FLAG(I386_VM_PTAVAIL1, "AV1");
 597         FLAG(I386_VM_PTAVAIL2, "AV2");
 598         FLAG(I386_VM_PTAVAIL3, "AV3");
 599 #elif defined(__arm__)
 600         FLAG(ARM_VM_PTE_SUPER, "S");
 601         FLAG(ARM_VM_PTE_S, "SH");
 602         FLAG(ARM_VM_PTE_WB, "WB");
 603         FLAG(ARM_VM_PTE_WT, "WT");
 604 #endif
 605
 606         return str;
 607 }
 608
 609 /*===========================================================================*
 610  *                           pt_map_in_range                                 *
 611  *===========================================================================*/
 612 int pt_map_in_range(struct vmproc *src_vmp, struct vmproc *dst_vmp,
 613         vir_bytes start, vir_bytes end)
 614 {
 615 /* Transfer all the mappings from the pt of the source process to the pt of
 616  * the destination process in the range specified.
 617  */
 618         int pde, pte;
 619         vir_bytes viraddr;
 620         pt_t *pt, *dst_pt;
 621
 622         pt = &src_vmp->vm_pt;
 623         dst_pt = &dst_vmp->vm_pt;
 624
 625         end = end ? end : VM_DATATOP;
 626         assert(start % VM_PAGE_SIZE == 0);
 627         assert(end % VM_PAGE_SIZE == 0);
 628
 629         assert( /* ARCH_VM_PDE(start) >= 0 && */ start <= end);
 630         assert(ARCH_VM_PDE(end) < ARCH_VM_DIR_ENTRIES);
 631
 632 #if LU_DEBUG
 633         printf("VM: pt_map_in_range: src = %d, dst = %d\n",
 634                 src_vmp->vm_endpoint, dst_vmp->vm_endpoint);
 635         printf("VM: pt_map_in_range: transferring from 0x%08x (pde %d pte %d) to 0x%08x (pde %d pte %d)\n",
 636                 start, ARCH_VM_PDE(start), ARCH_VM_PTE(start),
 637                 end, ARCH_VM_PDE(end), ARCH_VM_PTE(end));
 638 #endif
 639
 640         /* Scan all page-table entries in the range. */
 641         for(viraddr = start; viraddr <= end; viraddr += VM_PAGE_SIZE) {
 642                 pde = ARCH_VM_PDE(viraddr);
 643                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 644                         if(viraddr == VM_DATATOP) break;
 645                         continue;
 646                 }
 647                 pte = ARCH_VM_PTE(viraddr);
 648                 if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) {
 649                         if(viraddr == VM_DATATOP) break;
 650                         continue;
 651                 }
 652
 653                 /* Transfer the mapping. */
 654                 dst_pt->pt_pt[pde][pte] = pt->pt_pt[pde][pte];
 655
 656                 if(viraddr == VM_DATATOP) break;
 657         }
 658
 659         return OK;
 660 }
 661
 662 /*===========================================================================*
 663  *                              pt_ptmap                                     *
 664  *===========================================================================*/
 665 int pt_ptmap(struct vmproc *src_vmp, struct vmproc *dst_vmp)
 666 {
 667 /* Transfer mappings to page dir and page tables from source process and
 668  * destination process. Make sure all the mappings are above the stack, not
 669  * to corrupt valid mappings in the data segment of the destination process.
 670  */
 671         int pde, r;
 672         phys_bytes physaddr;
 673         vir_bytes viraddr;
 674         pt_t *pt;
 675
 676         pt = &src_vmp->vm_pt;
 677
 678 #if LU_DEBUG
 679         printf("VM: pt_ptmap: src = %d, dst = %d\n",
 680                 src_vmp->vm_endpoint, dst_vmp->vm_endpoint);
 681 #endif
 682
 683         /* Transfer mapping to the page directory. */
 684         viraddr = (vir_bytes) pt->pt_dir;
 685         physaddr = pt->pt_dir_phys & ARCH_VM_ADDR_MASK;
 686 #if defined(__i386__)
 687         if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, VM_PAGE_SIZE,
 688                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW,
 689 #elif defined(__arm__)
 690         if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, ARCH_PAGEDIR_SIZE,
 691                 ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER |
 692                 ARM_VM_PTE_CACHED ,
 693 #endif
 694                 WMF_OVERWRITE)) != OK) {
 695                 return r;
 696         }
 697 #if LU_DEBUG
 698         printf("VM: pt_ptmap: transferred mapping to page dir: 0x%08x (0x%08x)\n",
 699                 viraddr, physaddr);
 700 #endif
 701
 702         /* Scan all non-reserved page-directory entries. */
 703         for(pde=0; pde < ARCH_VM_DIR_ENTRIES; pde++) {
 704                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT)) {
 705                         continue;
 706                 }
 707
 708                 /* Transfer mapping to the page table. */
 709                 viraddr = (vir_bytes) pt->pt_pt[pde];
 710 #if defined(__i386__)
 711                 physaddr = pt->pt_dir[pde] & ARCH_VM_ADDR_MASK;
 712 #elif defined(__arm__)
 713                 physaddr = pt->pt_dir[pde] & ARCH_VM_PDE_MASK;
 714 #endif
 715                 if((r=pt_writemap(dst_vmp, &dst_vmp->vm_pt, viraddr, physaddr, VM_PAGE_SIZE,
 716                         ARCH_VM_PTE_PRESENT | ARCH_VM_PTE_USER | ARCH_VM_PTE_RW
 717 #ifdef __arm__
 718                         | ARM_VM_PTE_CACHED
 719 #endif
 720                         ,
 721                         WMF_OVERWRITE)) != OK) {
 722                         return r;
 723                 }
 724         }
 725
 726         return OK;
 727 }
 728
 729 void pt_clearmapcache(void)
 730 {
 731         /* Make sure kernel will invalidate tlb when using current
 732          * pagetable (i.e. vm's) to make new mappings before new cr3
 733          * is loaded.
 734          */
 735         if(sys_vmctl(SELF, VMCTL_CLEARMAPCACHE, 0) != OK)
 736                 panic("VMCTL_CLEARMAPCACHE failed");
 737 }
 738
 739 int pt_writable(struct vmproc *vmp, vir_bytes v)
 740 {
 741         u32_t entry;
 742         pt_t *pt = &vmp->vm_pt;
 743         assert(!(v % VM_PAGE_SIZE));
 744         int pde = ARCH_VM_PDE(v);
 745         int pte = ARCH_VM_PTE(v);
 746
 747         assert(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT);
 748         assert(pt->pt_pt[pde]);
 749
 750         entry = pt->pt_pt[pde][pte];
 751
 752 #if defined(__i386__)
 753         return((entry & PTF_WRITE) ? 1 : 0);
 754 #elif defined(__arm__)
 755         return((entry & ARCH_VM_PTE_RO) ? 0 : 1);
 756 #endif
 757 }
 758
 759 /*===========================================================================*
 760  *                              pt_writemap                                  *
 761  *===========================================================================*/
 762 int pt_writemap(struct vmproc * vmp,
 763                         pt_t *pt,
 764                         vir_bytes v,
 765                         phys_bytes physaddr,
 766                         size_t bytes,
 767                         u32_t flags,
 768                         u32_t writemapflags)
 769 {
 770 /* Write mapping into page table. Allocate a new page table if necessary. */
 771 /* Page directory and table entries for this virtual address. */
 772         int p, pages;
 773         int verify = 0;
 774         int ret = OK;
 775
 776 #ifdef CONFIG_SMP
 777         int vminhibit_clear = 0;
 778         /* FIXME
 779          * don't do it everytime, stop the process only on the first change and
 780          * resume the execution on the last change. Do in a wrapper of this
 781          * function
 782          */
 783         if (vmp && vmp->vm_endpoint != NONE && vmp->vm_endpoint != VM_PROC_NR &&
 784                         !(vmp->vm_flags & VMF_EXITING)) {
 785                 sys_vmctl(vmp->vm_endpoint, VMCTL_VMINHIBIT_SET, 0);
 786                 vminhibit_clear = 1;
 787         }
 788 #endif
 789
 790         if(writemapflags & WMF_VERIFY)
 791                 verify = 1;
 792
 793         assert(!(bytes % VM_PAGE_SIZE));
 794         assert(!(flags & ~(PTF_ALLFLAGS)));
 795
 796         pages = bytes / VM_PAGE_SIZE;
 797
 798         /* MAP_NONE means to clear the mapping. It doesn't matter
 799          * what's actually written into the PTE if PRESENT
 800          * isn't on, so we can just write MAP_NONE into it.
 801          */
 802         assert(physaddr == MAP_NONE || (flags & ARCH_VM_PTE_PRESENT));
 803         assert(physaddr != MAP_NONE || !flags);
 804
 805         /* First make sure all the necessary page tables are allocated,
 806          * before we start writing in any of them, because it's a pain
 807          * to undo our work properly.
 808          */
 809         ret = pt_ptalloc_in_range(pt, v, v + VM_PAGE_SIZE*pages, flags, verify);
 810         if(ret != OK) {
 811                 printf("VM: writemap: pt_ptalloc_in_range failed\n");
 812                 goto resume_exit;
 813         }
 814
 815         /* Now write in them. */
 816         for(p = 0; p < pages; p++) {
 817                 u32_t entry;
 818                 int pde = ARCH_VM_PDE(v);
 819                 int pte = ARCH_VM_PTE(v);
 820
 821                 assert(!(v % VM_PAGE_SIZE));
 822                 assert(pte >= 0 && pte < ARCH_VM_PT_ENTRIES);
 823                 assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES);
 824
 825                 /* Page table has to be there. */
 826                 assert(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT);
 827
 828                 /* We do not expect it to be a bigpage. */
 829                 assert(!(pt->pt_dir[pde] & ARCH_VM_BIGPAGE));
 830
 831                 /* Make sure page directory entry for this page table
 832                  * is marked present and page table entry is available.
 833                  */
 834                 assert(pt->pt_pt[pde]);
 835
 836 #if SANITYCHECKS
 837                 /* We don't expect to overwrite a page. */
 838                 if(!(writemapflags & (WMF_OVERWRITE|WMF_VERIFY)))
 839                         assert(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT));
 840 #endif
 841                 if(writemapflags & (WMF_WRITEFLAGSONLY|WMF_FREE)) {
 842 #if defined(__i386__)
 843                         physaddr = pt->pt_pt[pde][pte] & ARCH_VM_ADDR_MASK;
 844 #elif defined(__arm__)
 845                         physaddr = pt->pt_pt[pde][pte] & ARM_VM_PTE_MASK;
 846 #endif
 847                 }
 848
 849                 if(writemapflags & WMF_FREE) {
 850                         free_mem(ABS2CLICK(physaddr), 1);
 851                 }
 852
 853                 /* Entry we will write. */
 854 #if defined(__i386__)
 855                 entry = (physaddr & ARCH_VM_ADDR_MASK) | flags;
 856 #elif defined(__arm__)
 857                 entry = (physaddr & ARM_VM_PTE_MASK) | flags;
 858 #endif
 859
 860                 if(verify) {
 861                         u32_t maskedentry;
 862                         maskedentry = pt->pt_pt[pde][pte];
 863 #if defined(__i386__)
 864                         maskedentry &= ~(I386_VM_ACC|I386_VM_DIRTY);
 865 #endif
 866                         /* Verify pagetable entry. */
 867 #if defined(__i386__)
 868                         if(entry & ARCH_VM_PTE_RW) {
 869                                 /* If we expect a writable page, allow a readonly page. */
 870                                 maskedentry |= ARCH_VM_PTE_RW;
 871                         }
 872 #elif defined(__arm__)
 873                         if(!(entry & ARCH_VM_PTE_RO)) {
 874                                 /* If we expect a writable page, allow a readonly page. */
 875                                 maskedentry &= ~ARCH_VM_PTE_RO;
 876                         }
 877                         maskedentry &= ~(ARM_VM_PTE_WB|ARM_VM_PTE_WT);
 878 #endif
 879                         if(maskedentry != entry) {
 880                                 printf("pt_writemap: mismatch: ");
 881 #if defined(__i386__)
 882                                 if((entry & ARCH_VM_ADDR_MASK) !=
 883                                         (maskedentry & ARCH_VM_ADDR_MASK)) {
 884 #elif defined(__arm__)
 885                                 if((entry & ARM_VM_PTE_MASK) !=
 886                                         (maskedentry & ARM_VM_PTE_MASK)) {
 887 #endif
 888                                         printf("pt_writemap: physaddr mismatch (0x%lx, 0x%lx); ",
 889                                                 (long)entry, (long)maskedentry);
 890                                 } else printf("phys ok; ");
 891                                 printf(" flags: found %s; ",
 892                                         ptestr(pt->pt_pt[pde][pte]));
 893                                 printf(" masked %s; ",
 894                                         ptestr(maskedentry));
 895                                 printf(" expected %s\n", ptestr(entry));
 896                                 printf("found 0x%x, wanted 0x%x\n",
 897                                         pt->pt_pt[pde][pte], entry);
 898                                 ret = EFAULT;
 899                                 goto resume_exit;
 900                         }
 901                 } else {
 902                         /* Write pagetable entry. */
 903                         pt->pt_pt[pde][pte] = entry;
 904                 }
 905
 906                 physaddr += VM_PAGE_SIZE;
 907                 v += VM_PAGE_SIZE;
 908         }
 909
 910 resume_exit:
 911
 912 #ifdef CONFIG_SMP
 913         if (vminhibit_clear) {
 914                 assert(vmp && vmp->vm_endpoint != NONE && vmp->vm_endpoint != VM_PROC_NR &&
 915                         !(vmp->vm_flags & VMF_EXITING));
 916                 sys_vmctl(vmp->vm_endpoint, VMCTL_VMINHIBIT_CLEAR, 0);
 917         }
 918 #endif
 919
 920         return ret;
 921 }
 922
 923 /*===========================================================================*
 924  *                              pt_checkrange                                *
 925  *===========================================================================*/
 926 int pt_checkrange(pt_t *pt, vir_bytes v,  size_t bytes,
 927         int write)
 928 {
 929         int p, pages;
 930
 931         assert(!(bytes % VM_PAGE_SIZE));
 932
 933         pages = bytes / VM_PAGE_SIZE;
 934
 935         for(p = 0; p < pages; p++) {
 936                 int pde = ARCH_VM_PDE(v);
 937                 int pte = ARCH_VM_PTE(v);
 938
 939                 assert(!(v % VM_PAGE_SIZE));
 940                 assert(pte >= 0 && pte < ARCH_VM_PT_ENTRIES);
 941                 assert(pde >= 0 && pde < ARCH_VM_DIR_ENTRIES);
 942
 943                 /* Page table has to be there. */
 944                 if(!(pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT))
 945                         return EFAULT;
 946
 947                 /* Make sure page directory entry for this page table
 948                  * is marked present and page table entry is available.
 949                  */
 950                 assert((pt->pt_dir[pde] & ARCH_VM_PDE_PRESENT) && pt->pt_pt[pde]);
 951
 952                 if(!(pt->pt_pt[pde][pte] & ARCH_VM_PTE_PRESENT)) {
 953                         return EFAULT;
 954                 }
 955
 956 #if defined(__i386__)
 957                 if(write && !(pt->pt_pt[pde][pte] & ARCH_VM_PTE_RW)) {
 958 #elif defined(__arm__)
 959                 if(write && (pt->pt_pt[pde][pte] & ARCH_VM_PTE_RO)) {
 960 #endif
 961                         return EFAULT;
 962                 }
 963
 964                 v += VM_PAGE_SIZE;
 965         }
 966
 967         return OK;
 968 }
 969
 970 /*===========================================================================*
 971  *                              pt_new                                       *
 972  *===========================================================================*/
 973 int pt_new(pt_t *pt)
 974 {
 975 /* Allocate a pagetable root. Allocate a page-aligned page directory
 976  * and set them to 0 (indicating no page tables are allocated). Lookup
 977  * its physical address as we'll need that in the future. Verify it's
 978  * page-aligned.
 979  */
 980         int i, r;
 981
 982         /* Don't ever re-allocate/re-move a certain process slot's
 983          * page directory once it's been created. This is a fraction
 984          * faster, but also avoids having to invalidate the page
 985          * mappings from in-kernel page tables pointing to
 986          * the page directories (the page_directories data).
 987          */
 988         if(!pt->pt_dir &&
 989           !(pt->pt_dir = vm_allocpages((phys_bytes *)&pt->pt_dir_phys,
 990                 VMP_PAGEDIR, ARCH_PAGEDIR_SIZE/VM_PAGE_SIZE))) {
 991                 return ENOMEM;
 992         }
 993
 994         assert(!((u32_t)pt->pt_dir_phys % ARCH_PAGEDIR_SIZE));
 995
 996         for(i = 0; i < ARCH_VM_DIR_ENTRIES; i++) {
 997                 pt->pt_dir[i] = 0; /* invalid entry (PRESENT bit = 0) */
 998                 pt->pt_pt[i] = NULL;
 999         }
1000
1001         /* Where to start looking for free virtual address space? */
1002         pt->pt_virtop = 0;
1003
1004         /* Map in kernel. */
1005         if((r=pt_mapkernel(pt)) != OK)
1006                 return r;
1007
1008         return OK;
1009 }
1010
1011 static int freepde(void)
1012 {
1013         int p = kernel_boot_info.freepde_start++;
1014         assert(kernel_boot_info.freepde_start < ARCH_VM_DIR_ENTRIES);
1015         return p;
1016 }
1017
1018 /*===========================================================================*
1019  *                              pt_init                                      *
1020  *===========================================================================*/
1021 void pt_init(void)
1022 {
1023         pt_t *newpt;
1024         int s, r, p;
1025         vir_bytes sparepages_mem;
1026 #if defined(__arm__)
1027         vir_bytes sparepagedirs_mem;
1028 #endif
1029         static u32_t currentpagedir[ARCH_VM_DIR_ENTRIES];
1030         int m = kernel_boot_info.kern_mod;
1031 #if defined(__i386__)
1032         int global_bit_ok = 0;
1033         u32_t mypdbr; /* Page Directory Base Register (cr3) value */
1034 #elif defined(__arm__)
1035         u32_t myttbr;
1036 #endif
1037
1038         /* Find what the physical location of the kernel is. */
1039         assert(m >= 0);
1040         assert(m < kernel_boot_info.mods_with_kernel);
1041         assert(kernel_boot_info.mods_with_kernel < MULTIBOOT_MAX_MODS);
1042         kern_mb_mod = &kernel_boot_info.module_list[m];
1043         kern_size = kern_mb_mod->mod_end - kern_mb_mod->mod_start;
1044         assert(!(kern_mb_mod->mod_start % ARCH_BIG_PAGE_SIZE));
1045         assert(!(kernel_boot_info.vir_kern_start % ARCH_BIG_PAGE_SIZE));
1046         kern_start_pde = kernel_boot_info.vir_kern_start / ARCH_BIG_PAGE_SIZE;
1047
1048         /* Get ourselves spare pages. */
1049         sparepages_mem = (vir_bytes) static_sparepages;
1050         assert(!(sparepages_mem % VM_PAGE_SIZE));
1051
1052 #if defined(__arm__)
1053         /* Get ourselves spare pagedirs. */
1054         sparepagedirs_mem = (vir_bytes) static_sparepagedirs;
1055         assert(!(sparepagedirs_mem % ARCH_PAGEDIR_SIZE));
1056 #endif
1057
1058         /* Spare pages are used to allocate memory before VM has its own page
1059          * table that things (i.e. arbitrary physical memory) can be mapped into.
1060          * We get it by pre-allocating it in our bss (allocated and mapped in by
1061          * the kernel) in static_sparepages. We also need the physical addresses
1062          * though; we look them up now so they are ready for use.
1063          */
1064 #if defined(__arm__)
1065         missing_sparedirs = 0;
1066         assert(STATIC_SPAREPAGEDIRS <= SPAREPAGEDIRS);
1067         for(s = 0; s < SPAREPAGEDIRS; s++) {
1068                 vir_bytes v = (sparepagedirs_mem + s*ARCH_PAGEDIR_SIZE);;
1069                 phys_bytes ph;
1070                 if((r=sys_umap(SELF, VM_D, (vir_bytes) v,
1071                         ARCH_PAGEDIR_SIZE, &ph)) != OK)
1072                                 panic("pt_init: sys_umap failed: %d", r);
1073                 if(s >= STATIC_SPAREPAGEDIRS) {
1074                         sparepagedirs[s].pagedir = NULL;
1075                         missing_sparedirs++;
1076                         continue;
1077                 }
1078                 sparepagedirs[s].pagedir = (void *) v;
1079                 sparepagedirs[s].phys = ph;
1080         }
1081 #endif
1082
1083         if(!(spare_pagequeue = reservedqueue_new(SPAREPAGES, 1, 1, 0)))
1084                 panic("reservedqueue_new for single pages failed");
1085
1086         assert(STATIC_SPAREPAGES < SPAREPAGES);
1087         for(s = 0; s < STATIC_SPAREPAGES; s++) {
1088                 void *v = (void *) (sparepages_mem + s*VM_PAGE_SIZE);
1089                 phys_bytes ph;
1090                 if((r=sys_umap(SELF, VM_D, (vir_bytes) v,
1091                         VM_PAGE_SIZE*SPAREPAGES, &ph)) != OK)
1092                                 panic("pt_init: sys_umap failed: %d", r);
1093                 reservedqueue_add(spare_pagequeue, v, ph);
1094         }
1095
1096 #if defined(__i386__)
1097         /* global bit and 4MB pages available? */
1098         global_bit_ok = _cpufeature(_CPUF_I386_PGE);
1099         bigpage_ok = _cpufeature(_CPUF_I386_PSE);
1100
1101         /* Set bit for PTE's and PDE's if available. */
1102         if(global_bit_ok)
1103                 global_bit = I386_VM_GLOBAL;
1104 #endif
1105
1106         /* Now reserve another pde for kernel's own mappings. */
1107         {
1108                 int kernmap_pde;
1109                 phys_bytes addr, len;
1110                 int flags, pindex = 0;
1111                 u32_t offset = 0;
1112
1113                 kernmap_pde = freepde();
1114                 offset = kernmap_pde * ARCH_BIG_PAGE_SIZE;
1115
1116                 while(sys_vmctl_get_mapping(pindex, &addr, &len,
1117                         &flags) == OK)  {
1118                         int usedpde;
1119                         vir_bytes vir;
1120                         if(pindex >= MAX_KERNMAPPINGS)
1121                                 panic("VM: too many kernel mappings: %d", pindex);
1122                         kern_mappings[pindex].phys_addr = addr;
1123                         kern_mappings[pindex].len = len;
1124                         kern_mappings[pindex].flags = flags;
1125                         kern_mappings[pindex].vir_addr = offset;
1126                         kern_mappings[pindex].flags =
1127                                 ARCH_VM_PTE_PRESENT;
1128                         if(flags & VMMF_UNCACHED)
1129 #if defined(__i386__)
1130                                 kern_mappings[pindex].flags |= PTF_NOCACHE;
1131 #elif defined(__arm__)
1132                                 kern_mappings[pindex].flags |= ARM_VM_PTE_DEVICE;
1133                         else {
1134                                 kern_mappings[pindex].flags |= ARM_VM_PTE_CACHED;
1135                         }
1136 #endif
1137                         if(flags & VMMF_USER)
1138                                 kern_mappings[pindex].flags |= ARCH_VM_PTE_USER;
1139 #if defined(__arm__)
1140                         else
1141                                 kern_mappings[pindex].flags |= ARM_VM_PTE_SUPER;
1142 #endif
1143                         if(flags & VMMF_WRITE)
1144                                 kern_mappings[pindex].flags |= ARCH_VM_PTE_RW;
1145 #if defined(__arm__)
1146                         else
1147                                 kern_mappings[pindex].flags |= ARCH_VM_PTE_RO;
1148 #endif
1149
1150 #if defined(__i386__)
1151                         if(flags & VMMF_GLO)
1152                                 kern_mappings[pindex].flags |= I386_VM_GLOBAL;
1153 #endif
1154
1155                         if(addr % VM_PAGE_SIZE)
1156                                 panic("VM: addr unaligned: %lu", addr);
1157                         if(len % VM_PAGE_SIZE)
1158                                 panic("VM: len unaligned: %lu", len);
1159                         vir = offset;
1160                         if(sys_vmctl_reply_mapping(pindex, vir) != OK)
1161                                 panic("VM: reply failed");
1162                         offset += len;
1163                         pindex++;
1164                         kernmappings++;
1165
1166                         usedpde = ARCH_VM_PDE(offset);
1167                         while(usedpde > kernmap_pde) {
1168                                 int newpde = freepde();
1169                                 assert(newpde == kernmap_pde+1);
1170                                 kernmap_pde = newpde;
1171                         }
1172                 }
1173         }
1174
1175         /* Reserve PDEs available for mapping in the page directories. */
1176         {
1177                 int pd;
1178                 for(pd = 0; pd < MAX_PAGEDIR_PDES; pd++) {
1179                         struct pdm *pdm = &pagedir_mappings[pd];
1180                         pdm->pdeno = freepde();
1181                         phys_bytes ph;
1182
1183                         /* Allocate us a page table in which to
1184                          * remember page directory pointers.
1185                          */
1186                         if(!(pdm->page_directories =
1187                                 vm_allocpage(&ph, VMP_PAGETABLE))) {
1188                                 panic("no virt addr for vm mappings");
1189                         }
1190                         memset(pdm->page_directories, 0, VM_PAGE_SIZE);
1191                         pdm->phys = ph;
1192
1193 #if defined(__i386__)
1194                         pdm->val = (ph & ARCH_VM_ADDR_MASK) |
1195                                 ARCH_VM_PDE_PRESENT | ARCH_VM_PTE_RW;
1196 #elif defined(__arm__)
1197                         pdm->val = (ph & ARCH_VM_PDE_MASK)
1198                                 | ARCH_VM_PDE_PRESENT
1199                                 | ARM_VM_PTE_CACHED
1200                                 | ARM_VM_PDE_DOMAIN; //LSC FIXME
1201 #endif
1202                 }
1203         }
1204
1205         /* Allright. Now. We have to make our own page directory and page tables,
1206          * that the kernel has already set up, accessible to us. It's easier to
1207          * understand if we just copy all the required pages (i.e. page directory
1208          * and page tables), and set up the pointers as if VM had done it itself.
1209          *
1210          * This allocation will happen without using any page table, and just
1211          * uses spare pages.
1212          */
1213         newpt = &vmprocess->vm_pt;
1214         if(pt_new(newpt) != OK)
1215                 panic("vm pt_new failed");
1216
1217         /* Get our current pagedir so we can see it. */
1218 #if defined(__i386__)
1219         if(sys_vmctl_get_pdbr(SELF, &mypdbr) != OK)
1220 #elif defined(__arm__)
1221         if(sys_vmctl_get_pdbr(SELF, &myttbr) != OK)
1222 #endif
1223
1224                 panic("VM: sys_vmctl_get_pdbr failed");
1225 #if defined(__i386__)
1226         if(sys_vircopy(NONE, mypdbr, SELF,
1227                 (vir_bytes) currentpagedir, VM_PAGE_SIZE) != OK)
1228 #elif defined(__arm__)
1229         if(sys_vircopy(NONE, myttbr, SELF,
1230                 (vir_bytes) currentpagedir, ARCH_PAGEDIR_SIZE) != OK)
1231 #endif
1232                 panic("VM: sys_vircopy failed");
1233
1234         /* We have mapped in kernel ourselves; now copy mappings for VM
1235          * that kernel made, including allocations for BSS. Skip identity
1236          * mapping bits; just map in VM.
1237          */
1238         for(p = 0; p < ARCH_VM_DIR_ENTRIES; p++) {
1239                 u32_t entry = currentpagedir[p];
1240                 phys_bytes ptaddr_kern, ptaddr_us;
1241
1242                 /* BIGPAGEs are kernel mapping (do ourselves) or boot
1243                  * identity mapping (don't want).
1244                  */
1245                 if(!(entry & ARCH_VM_PDE_PRESENT)) continue;
1246                 if((entry & ARCH_VM_BIGPAGE)) continue;
1247
1248                 if(pt_ptalloc(newpt, p, 0) != OK)
1249                         panic("pt_ptalloc failed");
1250                 assert(newpt->pt_dir[p] & ARCH_VM_PDE_PRESENT);
1251
1252 #if defined(__i386__)
1253                 ptaddr_kern = entry & ARCH_VM_ADDR_MASK;
1254                 ptaddr_us = newpt->pt_dir[p] & ARCH_VM_ADDR_MASK;
1255 #elif defined(__arm__)
1256                 ptaddr_kern = entry & ARCH_VM_PDE_MASK;
1257                 ptaddr_us = newpt->pt_dir[p] & ARCH_VM_PDE_MASK;
1258 #endif
1259
1260                 /* Copy kernel-initialized pagetable contents into our
1261                  * normally accessible pagetable.
1262                  */
1263                 if(sys_abscopy(ptaddr_kern, ptaddr_us, VM_PAGE_SIZE) != OK)
1264                         panic("pt_init: abscopy failed");
1265         }
1266
1267         /* Inform kernel vm has a newly built page table. */
1268         assert(vmproc[VM_PROC_NR].vm_endpoint == VM_PROC_NR);
1269         pt_bind(newpt, &vmproc[VM_PROC_NR]);
1270
1271         pt_init_done = 1;
1272
1273         /* All OK. */
1274         return;
1275 }
1276
1277 /*===========================================================================*
1278  *                              pt_bind                                      *
1279  *===========================================================================*/
1280 int pt_bind(pt_t *pt, struct vmproc *who)
1281 {
1282         int procslot, pdeslot;
1283         u32_t phys;
1284         void *pdes;
1285         int pagedir_pde;
1286         int slots_per_pde;
1287         int pages_per_pagedir = ARCH_PAGEDIR_SIZE/VM_PAGE_SIZE;
1288         struct pdm *pdm;
1289
1290         slots_per_pde = ARCH_VM_PT_ENTRIES / pages_per_pagedir;
1291
1292         /* Basic sanity checks. */
1293         assert(who);
1294         assert(who->vm_flags & VMF_INUSE);
1295         assert(pt);
1296
1297         procslot = who->vm_slot;
1298         pdm = &pagedir_mappings[procslot/slots_per_pde];
1299         pdeslot = procslot%slots_per_pde;
1300         pagedir_pde = pdm->pdeno;
1301         assert(pdeslot >= 0);
1302         assert(procslot < ELEMENTS(vmproc));
1303         assert(pdeslot < ARCH_VM_PT_ENTRIES / pages_per_pagedir);
1304         assert(pagedir_pde >= 0);
1305
1306 #if defined(__i386__)
1307         phys = pt->pt_dir_phys & ARCH_VM_ADDR_MASK;
1308 #elif defined(__arm__)
1309         phys = pt->pt_dir_phys & ARM_VM_PTE_MASK;
1310 #endif
1311         assert(pt->pt_dir_phys == phys);
1312         assert(!(pt->pt_dir_phys % ARCH_PAGEDIR_SIZE));
1313
1314         /* Update "page directory pagetable." */
1315 #if defined(__i386__)
1316         pdm->page_directories[pdeslot] =
1317                 phys | ARCH_VM_PDE_PRESENT|ARCH_VM_PTE_RW;
1318 #elif defined(__arm__)
1319 {
1320         int i;
1321         for (i = 0; i < pages_per_pagedir; i++) {
1322                 pdm->page_directories[pdeslot*pages_per_pagedir+i] =
1323                         (phys+i*VM_PAGE_SIZE)
1324                         | ARCH_VM_PTE_PRESENT
1325                         | ARCH_VM_PTE_RW
1326                         | ARM_VM_PTE_CACHED
1327                         | ARCH_VM_PTE_USER; //LSC FIXME
1328         }
1329 }
1330 #endif
1331
1332         /* This is where the PDE's will be visible to the kernel
1333          * in its address space.
1334          */
1335         pdes = (void *) (pagedir_pde*ARCH_BIG_PAGE_SIZE +
1336 #if defined(__i386__)
1337                         pdeslot * VM_PAGE_SIZE);
1338 #elif defined(__arm__)
1339                         pdeslot * ARCH_PAGEDIR_SIZE);
1340 #endif
1341
1342         /* Tell kernel about new page table root. */
1343         return sys_vmctl_set_addrspace(who->vm_endpoint, pt->pt_dir_phys , pdes);
1344 }
1345
1346 /*===========================================================================*
1347  *                              pt_free                                      *
1348  *===========================================================================*/
1349 void pt_free(pt_t *pt)
1350 {
1351 /* Free memory associated with this pagetable. */
1352         int i;
1353
1354         for(i = 0; i < ARCH_VM_DIR_ENTRIES; i++)
1355                 if(pt->pt_pt[i])
1356                         vm_freepages((vir_bytes) pt->pt_pt[i], 1);
1357
1358         return;
1359 }
1360
1361 /*===========================================================================*
1362  *                              pt_mapkernel                                 *
1363  *===========================================================================*/
1364 int pt_mapkernel(pt_t *pt)
1365 {
1366         int i;
1367         int kern_pde = kern_start_pde;
1368         phys_bytes addr, mapped = 0;
1369
1370         /* Any page table needs to map in the kernel address space. */
1371         assert(bigpage_ok);
1372         assert(kern_pde >= 0);
1373
1374         /* pt_init() has made sure this is ok. */
1375         addr = kern_mb_mod->mod_start;
1376
1377         /* Actually mapping in kernel */
1378         while(mapped < kern_size) {
1379 #if defined(__i386__)
1380                 pt->pt_dir[kern_pde] = addr | ARCH_VM_PDE_PRESENT |
1381                         ARCH_VM_BIGPAGE | ARCH_VM_PTE_RW | global_bit;
1382 #elif defined(__arm__)
1383                 pt->pt_dir[kern_pde] = (addr & ARM_VM_SECTION_MASK)
1384                         | ARM_VM_SECTION
1385                         | ARM_VM_SECTION_DOMAIN
1386                         | ARM_VM_SECTION_CACHED
1387                         | ARM_VM_SECTION_SUPER;
1388 #endif
1389                 kern_pde++;
1390                 mapped += ARCH_BIG_PAGE_SIZE;
1391                 addr += ARCH_BIG_PAGE_SIZE;
1392         }
1393
1394         /* Kernel also wants to know about all page directories. */
1395         {
1396                 int pd;
1397                 for(pd = 0; pd < MAX_PAGEDIR_PDES; pd++) {
1398                         struct pdm *pdm = &pagedir_mappings[pd];
1399
1400                         assert(pdm->pdeno > 0);
1401                         assert(pdm->pdeno > kern_pde);
1402                         pt->pt_dir[pdm->pdeno] = pdm->val;
1403                 }
1404         }
1405
1406         /* Kernel also wants various mappings of its own. */
1407         for(i = 0; i < kernmappings; i++) {
1408                 int r;
1409                 if((r=pt_writemap(NULL, pt,
1410                         kern_mappings[i].vir_addr,
1411                         kern_mappings[i].phys_addr,
1412                         kern_mappings[i].len,
1413                         kern_mappings[i].flags, 0)) != OK) {
1414                         return r;
1415                 }
1416
1417         }
1418
1419         return OK;
1420 }
1421
1422 int get_vm_self_pages(void) { return vm_self_pages; }