servers/vm/i386/pagetable.c

   1
   2 #define _SYSTEM 1
   3 #define _POSIX_SOURCE 1
   4
   5 #include <minix/callnr.h>
   6 #include <minix/com.h>
   7 #include <minix/config.h>
   8 #include <minix/const.h>
   9 #include <minix/ds.h>
  10 #include <minix/endpoint.h>
  11 #include <minix/keymap.h>
  12 #include <minix/minlib.h>
  13 #include <minix/type.h>
  14 #include <minix/ipc.h>
  15 #include <minix/sysutil.h>
  16 #include <minix/syslib.h>
  17 #include <minix/safecopies.h>
  18 #include <minix/cpufeature.h>
  19 #include <minix/bitmap.h>
  20
  21 #include <errno.h>
  22 #include <stdlib.h>
  23 #include <assert.h>
  24 #include <string.h>
  25 #include <env.h>
  26 #include <stdio.h>
  27 #include <fcntl.h>
  28 #include <stdlib.h>
  29
  30 #include "../proto.h"
  31 #include "../glo.h"
  32 #include "../util.h"
  33 #include "../vm.h"
  34 #include "../sanitycheck.h"
  35
  36 #include "memory.h"
  37
  38 /* PDE used to map in kernel, kernel physical address. */
  39 PRIVATE int id_map_high_pde = -1, pagedir_pde = -1;
  40 PRIVATE u32_t global_bit = 0, pagedir_pde_val;
  41
  42 PRIVATE int proc_pde = 0;
  43
  44 /* 4MB page size available in hardware? */
  45 PRIVATE int bigpage_ok = 0;
  46
  47 /* Our process table entry. */
  48 struct vmproc *vmp = &vmproc[VM_PROC_NR];
  49
  50 /* Spare memory, ready to go after initialization, to avoid a
  51  * circular dependency on allocating memory and writing it into VM's
  52  * page table.
  53  */
  54 #define SPAREPAGES 25
  55 int missing_spares = SPAREPAGES;
  56 PRIVATE struct {
  57         void *page;
  58         u32_t phys;
  59 } sparepages[SPAREPAGES];
  60
  61 #define MAX_KERNMAPPINGS 10
  62 PRIVATE struct {
  63         phys_bytes      phys_addr;      /* Physical addr. */
  64         phys_bytes      len;            /* Length in bytes. */
  65         vir_bytes       lin_addr;       /* Offset in page table. */
  66         int             flags;
  67 } kern_mappings[MAX_KERNMAPPINGS];
  68 int kernmappings = 0;
  69
  70 /* Clicks must be pages, as
  71  *  - they must be page aligned to map them
  72  *  - they must be a multiple of the page size
  73  *  - it's inconvenient to have them bigger than pages, because we often want
  74  *    just one page
  75  * May as well require them to be equal then.
  76  */
  77 #if CLICK_SIZE != I386_PAGE_SIZE
  78 #error CLICK_SIZE must be page size.
  79 #endif
  80
  81 /* Bytes of virtual address space one pde controls. */
  82 #define BYTESPERPDE (I386_VM_PT_ENTRIES * I386_PAGE_SIZE)
  83
  84 /* Nevertheless, introduce these macros to make the code readable. */
  85 #define CLICK2PAGE(c) ((c) / CLICKSPERPAGE)
  86
  87 /* Page table that contains pointers to all page directories. */
  88 u32_t page_directories_phys, *page_directories = NULL;
  89
  90 #if SANITYCHECKS
  91 /*===========================================================================*
  92  *                              pt_sanitycheck                               *
  93  *===========================================================================*/
  94 PUBLIC void pt_sanitycheck(pt_t *pt, char *file, int line)
  95 {
  96 /* Basic pt sanity check. */
  97         int i;
  98         int slot;
  99
 100         MYASSERT(pt);
 101         MYASSERT(pt->pt_dir);
 102         MYASSERT(pt->pt_dir_phys);
 103
 104         for(slot = 0; slot < ELEMENTS(vmproc); slot++) {
 105                 if(pt == &vmproc[slot].vm_pt)
 106                         break;
 107         }
 108
 109         if(slot >= ELEMENTS(vmproc)) {
 110                 vm_panic("pt_sanitycheck: passed pt not in any proc", NO_NUM);
 111         }
 112
 113         MYASSERT(usedpages_add(pt->pt_dir_phys, I386_PAGE_SIZE) == OK);
 114
 115         for(i = proc_pde; i < I386_VM_DIR_ENTRIES; i++) {
 116                 if(pt->pt_pt[i]) {
 117                         if(!(pt->pt_dir[i] & I386_VM_PRESENT)) {
 118                                 printf("slot %d: pt->pt_pt[%d] = 0x%lx, but pt_dir entry 0x%lx\n",
 119                                         slot, i, pt->pt_pt[i], pt->pt_dir[i]);
 120                         }
 121                         MYASSERT(pt->pt_dir[i] & I386_VM_PRESENT);
 122                         MYASSERT(usedpages_add(I386_VM_PFA(pt->pt_dir[i]),
 123                                 I386_PAGE_SIZE) == OK);
 124                 } else {
 125                         MYASSERT(!(pt->pt_dir[i] & I386_VM_PRESENT));
 126                 }
 127         }
 128 }
 129 #endif
 130
 131 /*===========================================================================*
 132  *                              aalloc                                       *
 133  *===========================================================================*/
 134 PRIVATE void *aalloc(size_t bytes)
 135 {
 136 /* Page-aligned malloc(). only used if vm_allocpage can't be used.  */
 137         u32_t b;
 138
 139         b = (u32_t) malloc(I386_PAGE_SIZE + bytes);
 140         if(!b) vm_panic("aalloc: out of memory", bytes);
 141         b += I386_PAGE_SIZE - (b % I386_PAGE_SIZE);
 142
 143         return (void *) b;
 144 }
 145
 146 /*===========================================================================*
 147  *                              findhole                                     *
 148  *===========================================================================*/
 149 PRIVATE u32_t findhole(pt_t *pt, u32_t vmin, u32_t vmax)
 150 {
 151 /* Find a space in the virtual address space of pageteble 'pt',
 152  * between page-aligned BYTE offsets vmin and vmax, to fit
 153  * a page in. Return byte offset.
 154  */
 155         u32_t freefound = 0, curv;
 156         int pde = 0, try_restart;
 157         static u32_t lastv = 0;
 158
 159         /* Input sanity check. */
 160         vm_assert(vmin + I386_PAGE_SIZE >= vmin);
 161         vm_assert(vmax >= vmin + I386_PAGE_SIZE);
 162         vm_assert((vmin % I386_PAGE_SIZE) == 0);
 163         vm_assert((vmax % I386_PAGE_SIZE) == 0);
 164
 165 #if SANITYCHECKS
 166         curv = ((u32_t) random()) % ((vmax - vmin)/I386_PAGE_SIZE);
 167         curv *= I386_PAGE_SIZE;
 168         curv += vmin;
 169 #else
 170         curv = lastv;
 171         if(curv < vmin || curv >= vmax)
 172                 curv = vmin;
 173 #endif
 174         try_restart = 1;
 175
 176         /* Start looking for a free page starting at vmin. */
 177         while(curv < vmax) {
 178                 int pte;
 179
 180                 vm_assert(curv >= vmin);
 181                 vm_assert(curv < vmax);
 182
 183                 pde = I386_VM_PDE(curv);
 184                 pte = I386_VM_PTE(curv);
 185
 186                 if(!(pt->pt_dir[pde] & I386_VM_PRESENT) ||
 187                    !(pt->pt_pt[pde][pte] & I386_VM_PRESENT)) {
 188                         lastv = curv;
 189                         return curv;
 190                 }
 191
 192                 curv+=I386_PAGE_SIZE;
 193
 194                 if(curv >= vmax && try_restart) {
 195                         curv = vmin;
 196                         try_restart = 0;
 197                 }
 198         }
 199
 200         printf("VM: out of virtual address space in vm\n");
 201
 202         return NO_MEM;
 203 }
 204
 205 /*===========================================================================*
 206  *                              vm_freepages                                 *
 207  *===========================================================================*/
 208 PRIVATE void vm_freepages(vir_bytes vir, vir_bytes phys, int pages, int reason)
 209 {
 210         vm_assert(reason >= 0 && reason < VMP_CATEGORIES);
 211         if(vir >= vmp->vm_stacktop) {
 212                 vm_assert(!(vir % I386_PAGE_SIZE));
 213                 vm_assert(!(phys % I386_PAGE_SIZE));
 214                 FREE_MEM(ABS2CLICK(phys), pages);
 215                 if(pt_writemap(&vmp->vm_pt, arch_vir2map(vmp, vir),
 216                         MAP_NONE, pages*I386_PAGE_SIZE, 0, WMF_OVERWRITE) != OK)
 217                                 vm_panic("vm_freepages: pt_writemap failed",
 218                                         NO_NUM);
 219         } else {
 220                 printf("VM: vm_freepages not freeing VM heap pages (%d)\n",
 221                         pages);
 222         }
 223 }
 224
 225 /*===========================================================================*
 226  *                              vm_getsparepage                              *
 227  *===========================================================================*/
 228 PRIVATE void *vm_getsparepage(u32_t *phys)
 229 {
 230         int s;
 231         vm_assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
 232         for(s = 0; s < SPAREPAGES; s++) {
 233                 if(sparepages[s].page) {
 234                         void *sp;
 235                         sp = sparepages[s].page;
 236                         *phys = sparepages[s].phys;
 237                         sparepages[s].page = NULL;
 238                         missing_spares++;
 239                         vm_assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
 240                         return sp;
 241                 }
 242         }
 243         return NULL;
 244 }
 245
 246 /*===========================================================================*
 247  *                              vm_checkspares                               *
 248  *===========================================================================*/
 249 PRIVATE void *vm_checkspares(void)
 250 {
 251         int s, n = 0;
 252         static int total = 0, worst = 0;
 253         vm_assert(missing_spares >= 0 && missing_spares <= SPAREPAGES);
 254         for(s = 0; s < SPAREPAGES && missing_spares > 0; s++)
 255             if(!sparepages[s].page) {
 256                 n++;
 257                 if((sparepages[s].page = vm_allocpage(&sparepages[s].phys,
 258                         VMP_SPARE))) {
 259                         missing_spares--;
 260                         vm_assert(missing_spares >= 0);
 261                         vm_assert(missing_spares <= SPAREPAGES);
 262                 } else {
 263                         printf("VM: warning: couldn't get new spare page\n");
 264                 }
 265         }
 266         if(worst < n) worst = n;
 267         total += n;
 268
 269         return NULL;
 270 }
 271
 272 /*===========================================================================*
 273  *                              vm_allocpage                                 *
 274  *===========================================================================*/
 275 PUBLIC void *vm_allocpage(phys_bytes *phys, int reason)
 276 {
 277 /* Allocate a page for use by VM itself. */
 278         phys_bytes newpage;
 279         vir_bytes loc;
 280         pt_t *pt;
 281         int r;
 282         static int level = 0;
 283         void *ret;
 284
 285         pt = &vmp->vm_pt;
 286         vm_assert(reason >= 0 && reason < VMP_CATEGORIES);
 287
 288         level++;
 289
 290         vm_assert(level >= 1);
 291         vm_assert(level <= 2);
 292
 293         if(level > 1 || !(vmp->vm_flags & VMF_HASPT) || !meminit_done) {
 294                 int r;
 295                 void *s;
 296                 s=vm_getsparepage(phys);
 297                 level--;
 298                 if(!s) {
 299                         util_stacktrace();
 300                         printf("VM: warning: out of spare pages\n");
 301                 }
 302                 return s;
 303         }
 304
 305         /* VM does have a pagetable, so get a page and map it in there.
 306          * Where in our virtual address space can we put it?
 307          */
 308         loc = findhole(pt,  arch_vir2map(vmp, vmp->vm_stacktop),
 309                 vmp->vm_arch.vm_data_top);
 310         if(loc == NO_MEM) {
 311                 level--;
 312                 printf("VM: vm_allocpage: findhole failed\n");
 313                 return NULL;
 314         }
 315
 316         /* Allocate page of memory for use by VM. As VM
 317          * is trusted, we don't have to pre-clear it.
 318          */
 319         if((newpage = ALLOC_MEM(CLICKSPERPAGE, 0)) == NO_MEM) {
 320                 level--;
 321                 printf("VM: vm_allocpage: ALLOC_MEM failed\n");
 322                 return NULL;
 323         }
 324
 325         *phys = CLICK2ABS(newpage);
 326
 327         /* Map this page into our address space. */
 328         if((r=pt_writemap(pt, loc, *phys, I386_PAGE_SIZE,
 329                 I386_VM_PRESENT | I386_VM_USER | I386_VM_WRITE, 0)) != OK) {
 330                 FREE_MEM(newpage, CLICKSPERPAGE);
 331                 printf("vm_allocpage writemap failed\n", ret);
 332                 level--;
 333                 return NULL;
 334         }
 335
 336         if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 337                 vm_panic("VMCTL_FLUSHTLB failed", r);
 338         }
 339
 340         level--;
 341
 342         /* Return user-space-ready pointer to it. */
 343         ret = (void *) arch_map2vir(vmp, loc);
 344
 345         return ret;
 346 }
 347
 348 /*===========================================================================*
 349  *                              vm_pagelock                                  *
 350  *===========================================================================*/
 351 PUBLIC void vm_pagelock(void *vir, int lockflag)
 352 {
 353 /* Mark a page allocated by vm_allocpage() unwritable, i.e. only for VM. */
 354         vir_bytes m;
 355         int r;
 356         u32_t flags = I386_VM_PRESENT | I386_VM_USER;
 357         pt_t *pt;
 358
 359         pt = &vmp->vm_pt;
 360         m = arch_vir2map(vmp, (vir_bytes) vir);
 361
 362         vm_assert(!(m % I386_PAGE_SIZE));
 363
 364         if(!lockflag)
 365                 flags |= I386_VM_WRITE;
 366
 367         /* Update flags. */
 368         if((r=pt_writemap(pt, m, 0, I386_PAGE_SIZE,
 369                 flags, WMF_OVERWRITE | WMF_WRITEFLAGSONLY)) != OK) {
 370                 vm_panic("vm_lockpage: pt_writemap failed\n", NO_NUM);
 371         }
 372
 373         if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) {
 374                 vm_panic("VMCTL_FLUSHTLB failed", r);
 375         }
 376
 377         return;
 378 }
 379
 380 /*===========================================================================*
 381  *                              pt_ptalloc                                   *
 382  *===========================================================================*/
 383 PRIVATE int pt_ptalloc(pt_t *pt, int pde, u32_t flags)
 384 {
 385 /* Allocate a page table and write its address into the page directory. */
 386         int i;
 387         u32_t pt_phys;
 388
 389         /* Argument must make sense. */
 390         vm_assert(pde >= 0 && pde < I386_VM_DIR_ENTRIES);
 391         vm_assert(!(flags & ~(PTF_ALLFLAGS)));
 392
 393         /* We don't expect to overwrite page directory entry, nor
 394          * storage for the page table.
 395          */
 396         vm_assert(!(pt->pt_dir[pde] & I386_VM_PRESENT));
 397         vm_assert(!pt->pt_pt[pde]);
 398
 399         /* Get storage for the page table. */
 400         if(!(pt->pt_pt[pde] = vm_allocpage(&pt_phys, VMP_PAGETABLE)))
 401                 return ENOMEM;
 402
 403         for(i = 0; i < I386_VM_PT_ENTRIES; i++)
 404                 pt->pt_pt[pde][i] = 0;  /* Empty entry. */
 405
 406         /* Make page directory entry.
 407          * The PDE is always 'present,' 'writable,' and 'user accessible,'
 408          * relying on the PTE for protection.
 409          */
 410         pt->pt_dir[pde] = (pt_phys & I386_VM_ADDR_MASK) | flags
 411                 | I386_VM_PRESENT | I386_VM_USER | I386_VM_WRITE;
 412
 413         return OK;
 414 }
 415
 416 /*===========================================================================*
 417  *                              pt_writemap                                  *
 418  *===========================================================================*/
 419 PUBLIC int pt_writemap(pt_t *pt, vir_bytes v, phys_bytes physaddr,
 420         size_t bytes, u32_t flags, u32_t writemapflags)
 421 {
 422 /* Write mapping into page table. Allocate a new page table if necessary. */
 423 /* Page directory and table entries for this virtual address. */
 424         int p, pages, pdecheck;
 425         int finalpde;
 426         int verify = 0;
 427
 428         if(writemapflags & WMF_VERIFY)
 429                 verify = 1;
 430
 431         vm_assert(!(bytes % I386_PAGE_SIZE));
 432         vm_assert(!(flags & ~(PTF_ALLFLAGS)));
 433
 434         pages = bytes / I386_PAGE_SIZE;
 435
 436         /* MAP_NONE means to clear the mapping. It doesn't matter
 437          * what's actually written into the PTE if I386_VM_PRESENT
 438          * isn't on, so we can just write MAP_NONE into it.
 439          */
 440 #if SANITYCHECKS
 441         if(physaddr != MAP_NONE && !(flags & I386_VM_PRESENT)) {
 442                 vm_panic("pt_writemap: writing dir with !P\n", NO_NUM);
 443         }
 444         if(physaddr == MAP_NONE && flags) {
 445                 vm_panic("pt_writemap: writing 0 with flags\n", NO_NUM);
 446         }
 447 #endif
 448
 449         finalpde = I386_VM_PDE(v + I386_PAGE_SIZE * pages);
 450
 451         /* First make sure all the necessary page tables are allocated,
 452          * before we start writing in any of them, because it's a pain
 453          * to undo our work properly. Walk the range in page-directory-entry
 454          * sized leaps.
 455          */
 456         for(pdecheck = I386_VM_PDE(v); pdecheck <= finalpde; pdecheck++) {
 457                 vm_assert(pdecheck >= 0 && pdecheck < I386_VM_DIR_ENTRIES);
 458                 if(pt->pt_dir[pdecheck] & I386_VM_BIGPAGE) {
 459                         printf("pt_writemap: trying to write 0x%lx into 0x%lx\n",
 460                                 physaddr, v);
 461                         vm_panic("pt_writemap: BIGPAGE found", NO_NUM);
 462                 }
 463                 if(!(pt->pt_dir[pdecheck] & I386_VM_PRESENT)) {
 464                         int r;
 465                         if(verify) {
 466                                 printf("pt_writemap verify: no pde %d\n", pdecheck);
 467                                 return EFAULT;
 468                         }
 469                         vm_assert(!pt->pt_dir[pdecheck]);
 470                         if((r=pt_ptalloc(pt, pdecheck, flags)) != OK) {
 471                                 /* Couldn't do (complete) mapping.
 472                                  * Don't bother freeing any previously
 473                                  * allocated page tables, they're
 474                                  * still writable, don't point to nonsense,
 475                                  * and pt_ptalloc leaves the directory
 476                                  * and other data in a consistent state.
 477                                  */
 478                                 printf("pt_writemap: pt_ptalloc failed\n", pdecheck);
 479                                 return r;
 480                         }
 481                 }
 482                 vm_assert(pt->pt_dir[pdecheck] & I386_VM_PRESENT);
 483         }
 484
 485         /* Now write in them. */
 486         for(p = 0; p < pages; p++) {
 487                 u32_t entry;
 488                 int pde = I386_VM_PDE(v);
 489                 int pte = I386_VM_PTE(v);
 490
 491                 vm_assert(!(v % I386_PAGE_SIZE));
 492                 vm_assert(pte >= 0 && pte < I386_VM_PT_ENTRIES);
 493                 vm_assert(pde >= 0 && pde < I386_VM_DIR_ENTRIES);
 494
 495                 /* Page table has to be there. */
 496                 vm_assert(pt->pt_dir[pde] & I386_VM_PRESENT);
 497
 498                 /* Make sure page directory entry for this page table
 499                  * is marked present and page table entry is available.
 500                  */
 501                 vm_assert((pt->pt_dir[pde] & I386_VM_PRESENT) && pt->pt_pt[pde]);
 502
 503 #if SANITYCHECKS
 504                 /* We don't expect to overwrite a page. */
 505                 if(!(writemapflags & (WMF_OVERWRITE|WMF_VERIFY)))
 506                         vm_assert(!(pt->pt_pt[pde][pte] & I386_VM_PRESENT));
 507 #endif
 508                 if(writemapflags & (WMF_WRITEFLAGSONLY|WMF_FREE)) {
 509                         physaddr = pt->pt_pt[pde][pte] & I386_VM_ADDR_MASK;
 510                 }
 511
 512                 if(writemapflags & WMF_FREE) {
 513                         FREE_MEM(ABS2CLICK(physaddr), 1);
 514                 }
 515
 516                 /* Entry we will write. */
 517                 entry = (physaddr & I386_VM_ADDR_MASK) | flags;
 518
 519                 if(verify) {
 520                         u32_t maskedentry;
 521                         maskedentry = pt->pt_pt[pde][pte];
 522                         maskedentry &= ~(I386_VM_ACC|I386_VM_DIRTY);
 523                         /* Verify pagetable entry. */
 524                         if(maskedentry != entry) {
 525                                 printf("pt_writemap: 0x%lx found, masked 0x%lx, 0x%lx expected\n",
 526                                         pt->pt_pt[pde][pte], maskedentry, entry);
 527                                 return EFAULT;
 528                         }
 529                 } else {
 530                         /* Write pagetable entry. */
 531                         pt->pt_pt[pde][pte] = entry;
 532                 }
 533
 534                 physaddr += I386_PAGE_SIZE;
 535                 v += I386_PAGE_SIZE;
 536         }
 537
 538         return OK;
 539 }
 540
 541 /*===========================================================================*
 542  *                              pt_checkrange                                *
 543  *===========================================================================*/
 544 PUBLIC int pt_checkrange(pt_t *pt, vir_bytes v,  size_t bytes,
 545         int write)
 546 {
 547         int p, pages, pde;
 548
 549         vm_assert(!(bytes % I386_PAGE_SIZE));
 550
 551         pages = bytes / I386_PAGE_SIZE;
 552
 553         for(p = 0; p < pages; p++) {
 554                 u32_t entry;
 555                 int pde = I386_VM_PDE(v);
 556                 int pte = I386_VM_PTE(v);
 557
 558                 vm_assert(!(v % I386_PAGE_SIZE));
 559                 vm_assert(pte >= 0 && pte < I386_VM_PT_ENTRIES);
 560                 vm_assert(pde >= 0 && pde < I386_VM_DIR_ENTRIES);
 561
 562                 /* Page table has to be there. */
 563                 if(!(pt->pt_dir[pde] & I386_VM_PRESENT))
 564                         return EFAULT;
 565
 566                 /* Make sure page directory entry for this page table
 567                  * is marked present and page table entry is available.
 568                  */
 569                 vm_assert((pt->pt_dir[pde] & I386_VM_PRESENT) && pt->pt_pt[pde]);
 570
 571                 if(!(pt->pt_pt[pde][pte] & I386_VM_PRESENT)) {
 572                         return EFAULT;
 573                 }
 574
 575                 if(write && !(pt->pt_pt[pde][pte] & I386_VM_WRITE)) {
 576                         return EFAULT;
 577                 }
 578
 579                 v += I386_PAGE_SIZE;
 580         }
 581
 582         return OK;
 583 }
 584
 585 /*===========================================================================*
 586  *                              pt_new                                       *
 587  *===========================================================================*/
 588 PUBLIC int pt_new(pt_t *pt)
 589 {
 590 /* Allocate a pagetable root. On i386, allocate a page-aligned page directory
 591  * and set them to 0 (indicating no page tables are allocated). Lookup
 592  * its physical address as we'll need that in the future. Verify it's
 593  * page-aligned.
 594  */
 595         int i;
 596
 597         /* Don't ever re-allocate/re-move a certain process slot's
 598          * page directory once it's been created. This is a fraction
 599          * faster, but also avoids having to invalidate the page
 600          * mappings from in-kernel page tables pointing to
 601          * the page directories (the page_directories data).
 602          */
 603         if(!pt->pt_dir &&
 604           !(pt->pt_dir = vm_allocpage(&pt->pt_dir_phys, VMP_PAGEDIR))) {
 605                 return ENOMEM;
 606         }
 607
 608         for(i = 0; i < I386_VM_DIR_ENTRIES; i++) {
 609                 pt->pt_dir[i] = 0; /* invalid entry (I386_VM_PRESENT bit = 0) */
 610                 pt->pt_pt[i] = NULL;
 611         }
 612
 613         /* Where to start looking for free virtual address space? */
 614         pt->pt_virtop = 0;
 615
 616         /* Map in kernel. */
 617         if(pt_mapkernel(pt) != OK)
 618                 vm_panic("pt_new: pt_mapkernel failed", NO_NUM);
 619
 620         return OK;
 621 }
 622
 623 /*===========================================================================*
 624  *                              pt_identity                                  *
 625  *===========================================================================*/
 626 PUBLIC int pt_identity(pt_t *pt)
 627 {
 628 /* Allocate a pagetable that does a 1:1 mapping. */
 629         int i;
 630
 631         /* Allocate page directory. */
 632         if(!pt->pt_dir &&
 633           !(pt->pt_dir = vm_allocpage(&pt->pt_dir_phys, VMP_PAGEDIR))) {
 634                 return ENOMEM;
 635         }
 636
 637         for(i = 0; i < I386_VM_DIR_ENTRIES; i++) {
 638                 phys_bytes addr;
 639                 addr = I386_BIG_PAGE_SIZE*i;
 640                 pt->pt_dir[i] = (addr & I386_VM_ADDR_MASK_4MB) |
 641                                 I386_VM_BIGPAGE|
 642                                 I386_VM_USER|
 643                                 I386_VM_PRESENT|I386_VM_WRITE;
 644                 pt->pt_pt[i] = NULL;
 645         }
 646
 647         /* Where to start looking for free virtual address space? */
 648         pt->pt_virtop = 0;
 649
 650         return OK;
 651 }
 652
 653 /*===========================================================================*
 654  *                              pt_init                                      *
 655  *===========================================================================*/
 656 PUBLIC void pt_init(phys_bytes usedlimit)
 657 {
 658 /* By default, the kernel gives us a data segment with pre-allocated
 659  * memory that then can't grow. We want to be able to allocate memory
 660  * dynamically, however. So here we copy the part of the page table
 661  * that's ours, so we get a private page table. Then we increase the
 662  * hardware segment size so we can allocate memory above our stack.
 663  */
 664         pt_t *newpt;
 665         int s, r;
 666         vir_bytes v, kpagedir;
 667         phys_bytes lo, hi;
 668         vir_bytes extra_clicks;
 669         u32_t moveup = 0;
 670         int global_bit_ok = 0;
 671         int free_pde;
 672         int p;
 673         vir_bytes kernlimit;
 674         vir_bytes sparepages_mem;
 675         phys_bytes sparepages_ph;
 676
 677         /* Shorthand. */
 678         newpt = &vmp->vm_pt;
 679
 680
 681         /* Get ourselves spare pages. */
 682         if(!(sparepages_mem = (vir_bytes) aalloc(I386_PAGE_SIZE*SPAREPAGES)))
 683                 vm_panic("pt_init: aalloc for spare failed", NO_NUM);
 684         if((r=sys_umap(SELF, VM_D, (vir_bytes) sparepages_mem,
 685                 I386_PAGE_SIZE*SPAREPAGES, &sparepages_ph)) != OK)
 686                 vm_panic("pt_init: sys_umap failed", r);
 687
 688         for(s = 0; s < SPAREPAGES; s++) {
 689                 sparepages[s].page = (void *) (sparepages_mem + s*I386_PAGE_SIZE);
 690                 sparepages[s].phys = sparepages_ph + s*I386_PAGE_SIZE;
 691         }
 692
 693         missing_spares = 0;
 694
 695         /* global bit and 4MB pages available? */
 696         global_bit_ok = _cpufeature(_CPUF_I386_PGE);
 697         bigpage_ok = _cpufeature(_CPUF_I386_PSE);
 698
 699         /* Set bit for PTE's and PDE's if available. */
 700         if(global_bit_ok)
 701                 global_bit = I386_VM_GLOBAL;
 702
 703         /* The kernel and boot time processes need an identity mapping.
 704          * We use full PDE's for this without separate page tables.
 705          * Figure out which pde we can start using for other purposes.
 706          */
 707         id_map_high_pde = usedlimit / I386_BIG_PAGE_SIZE;
 708
 709         /* We have to make mappings up till here. */
 710         free_pde = id_map_high_pde+1;
 711
 712         /* Initial (current) range of our virtual address space. */
 713         lo = CLICK2ABS(vmp->vm_arch.vm_seg[T].mem_phys);
 714         hi = CLICK2ABS(vmp->vm_arch.vm_seg[S].mem_phys +
 715                 vmp->vm_arch.vm_seg[S].mem_len);
 716
 717         vm_assert(!(lo % I386_PAGE_SIZE));
 718         vm_assert(!(hi % I386_PAGE_SIZE));
 719
 720         if(lo < VM_PROCSTART) {
 721                 moveup = VM_PROCSTART - lo;
 722                 vm_assert(!(VM_PROCSTART % I386_PAGE_SIZE));
 723                 vm_assert(!(lo % I386_PAGE_SIZE));
 724                 vm_assert(!(moveup % I386_PAGE_SIZE));
 725         }
 726
 727         /* Make new page table for ourselves, partly copied
 728          * from the current one.
 729          */
 730         if(pt_new(newpt) != OK)
 731                 vm_panic("pt_init: pt_new failed", NO_NUM);
 732
 733         /* Set up mappings for VM process. */
 734         for(v = lo; v < hi; v += I386_PAGE_SIZE)  {
 735                 phys_bytes addr;
 736                 u32_t flags;
 737
 738                 /* We have to write the new position in the PT,
 739                  * so we can move our segments.
 740                  */
 741                 if(pt_writemap(newpt, v+moveup, v, I386_PAGE_SIZE,
 742                         I386_VM_PRESENT|I386_VM_WRITE|I386_VM_USER, 0) != OK)
 743                         vm_panic("pt_init: pt_writemap failed", NO_NUM);
 744         }
 745
 746         /* Move segments up too. */
 747         vmp->vm_arch.vm_seg[T].mem_phys += ABS2CLICK(moveup);
 748         vmp->vm_arch.vm_seg[D].mem_phys += ABS2CLICK(moveup);
 749         vmp->vm_arch.vm_seg[S].mem_phys += ABS2CLICK(moveup);
 750
 751         /* Allocate us a page table in which to remember page directory
 752          * pointers.
 753          */
 754         if(!(page_directories = vm_allocpage(&page_directories_phys,
 755                 VMP_PAGETABLE)))
 756                 vm_panic("no virt addr for vm mappings", NO_NUM);
 757
 758         memset(page_directories, 0, I386_PAGE_SIZE);
 759
 760         /* Increase our hardware data segment to create virtual address
 761          * space above our stack. We want to increase it to VM_DATATOP,
 762          * like regular processes have.
 763          */
 764         extra_clicks = ABS2CLICK(VM_DATATOP - hi);
 765         vmp->vm_arch.vm_seg[S].mem_len += extra_clicks;
 766
 767         /* We pretend to the kernel we have a huge stack segment to
 768          * increase our data segment.
 769          */
 770         vmp->vm_arch.vm_data_top =
 771                 (vmp->vm_arch.vm_seg[S].mem_vir +
 772                 vmp->vm_arch.vm_seg[S].mem_len) << CLICK_SHIFT;
 773
 774         /* Where our free virtual address space starts.
 775          * This is only a hint to the VM system.
 776          */
 777         newpt->pt_virtop = 0;
 778
 779         /* Let other functions know VM now has a private page table. */
 780         vmp->vm_flags |= VMF_HASPT;
 781
 782         /* Now reserve another pde for kernel's own mappings. */
 783         {
 784                 int kernmap_pde;
 785                 phys_bytes addr, len;
 786                 int flags, index = 0;
 787                 u32_t offset = 0;
 788
 789                 kernmap_pde = free_pde++;
 790                 offset = kernmap_pde * I386_BIG_PAGE_SIZE;
 791
 792                 while(sys_vmctl_get_mapping(index, &addr, &len,
 793                         &flags) == OK)  {
 794                         vir_bytes vir;
 795                         if(index >= MAX_KERNMAPPINGS)
 796                                 vm_panic("VM: too many kernel mappings", index);
 797                         kern_mappings[index].phys_addr = addr;
 798                         kern_mappings[index].len = len;
 799                         kern_mappings[index].flags = flags;
 800                         kern_mappings[index].lin_addr = offset;
 801                         kern_mappings[index].flags =
 802                                 I386_VM_PRESENT | I386_VM_USER | I386_VM_WRITE |
 803                                 global_bit;
 804                         if(flags & VMMF_UNCACHED)
 805                                 kern_mappings[index].flags |=
 806                                         I386_VM_PWT | I386_VM_PCD;
 807                         if(addr % I386_PAGE_SIZE)
 808                                 vm_panic("VM: addr unaligned", addr);
 809                         if(len % I386_PAGE_SIZE)
 810                                 vm_panic("VM: len unaligned", len);
 811                         vir = arch_map2vir(&vmproc[VMP_SYSTEM], offset);
 812                         if(sys_vmctl_reply_mapping(index, vir) != OK)
 813                                 vm_panic("VM: reply failed", NO_NUM);
 814                         offset += len;
 815                         index++;
 816                         kernmappings++;
 817                 }
 818         }
 819
 820         /* Find a PDE below processes available for mapping in the
 821          * page directories (readonly).
 822          */
 823         pagedir_pde = free_pde++;
 824         pagedir_pde_val = (page_directories_phys & I386_VM_ADDR_MASK) |
 825                         I386_VM_PRESENT | I386_VM_USER | I386_VM_WRITE;
 826
 827         /* Tell kernel about free pde's. */
 828         while(free_pde*I386_BIG_PAGE_SIZE < VM_PROCSTART) {
 829                 if((r=sys_vmctl(SELF, VMCTL_I386_FREEPDE, free_pde++)) != OK) {
 830                         vm_panic("VMCTL_I386_FREEPDE failed", r);
 831                 }
 832         }
 833
 834         /* first pde in use by process. */
 835         proc_pde = free_pde;
 836
 837         kernlimit = free_pde*I386_BIG_PAGE_SIZE;
 838
 839         /* Increase kernel segment to address this memory. */
 840         if((r=sys_vmctl(SELF, VMCTL_I386_KERNELLIMIT, kernlimit)) != OK) {
 841                 vm_panic("VMCTL_I386_KERNELLIMIT failed", r);
 842         }
 843
 844         kpagedir = arch_map2vir(&vmproc[VMP_SYSTEM],
 845                 pagedir_pde*I386_BIG_PAGE_SIZE);
 846
 847         /* Tell kernel how to get at the page directories. */
 848         if((r=sys_vmctl(SELF, VMCTL_I386_PAGEDIRS, kpagedir)) != OK) {
 849                 vm_panic("VMCTL_I386_KERNELLIMIT failed", r);
 850         }
 851
 852         /* Give our process the new, copied, private page table. */
 853         pt_mapkernel(newpt);    /* didn't know about vm_dir pages earlier */
 854         pt_bind(newpt, vmp);
 855
 856         /* Now actually enable paging. */
 857         if(sys_vmctl_enable_paging(vmp->vm_arch.vm_seg) != OK)
 858                 vm_panic("pt_init: enable paging failed", NO_NUM);
 859
 860         /* Back to reality - this is where the stack actually is. */
 861         vmp->vm_arch.vm_seg[S].mem_len -= extra_clicks;
 862
 863         /* All OK. */
 864         return;
 865 }
 866
 867
 868 /*===========================================================================*
 869  *                              pt_bind                                      *
 870  *===========================================================================*/
 871 PUBLIC int pt_bind(pt_t *pt, struct vmproc *who)
 872 {
 873         int slot, ispt;
 874         u32_t phys;
 875
 876         /* Basic sanity checks. */
 877         vm_assert(who);
 878         vm_assert(who->vm_flags & VMF_INUSE);
 879         vm_assert(pt);
 880
 881         slot = who->vm_slot;
 882         vm_assert(slot >= 0);
 883         vm_assert(slot < ELEMENTS(vmproc));
 884         vm_assert(slot < I386_VM_PT_ENTRIES);
 885
 886         phys = pt->pt_dir_phys & I386_VM_ADDR_MASK;
 887         vm_assert(pt->pt_dir_phys == phys);
 888
 889         /* Update "page directory pagetable." */
 890         page_directories[slot] = phys | I386_VM_PRESENT|I386_VM_WRITE;
 891
 892 #if 0
 893         printf("VM: slot %d has pde val 0x%lx\n", slot, page_directories[slot]);
 894 #endif
 895         /* Tell kernel about new page table root. */
 896         return sys_vmctl(who->vm_endpoint, VMCTL_I386_SETCR3,
 897                 pt ? pt->pt_dir_phys : 0);
 898 }
 899
 900 /*===========================================================================*
 901  *                              pt_free                                      *
 902  *===========================================================================*/
 903 PUBLIC void pt_free(pt_t *pt)
 904 {
 905 /* Free memory associated with this pagetable. */
 906         int i;
 907
 908         for(i = 0; i < I386_VM_DIR_ENTRIES; i++)
 909                 if(pt->pt_pt[i])
 910                         vm_freepages((vir_bytes) pt->pt_pt[i],
 911                                 I386_VM_PFA(pt->pt_dir[i]), 1, VMP_PAGETABLE);
 912
 913         return;
 914 }
 915
 916 /*===========================================================================*
 917  *                              pt_mapkernel                                 *
 918  *===========================================================================*/
 919 PUBLIC int pt_mapkernel(pt_t *pt)
 920 {
 921         int r, i;
 922
 923         /* Any i386 page table needs to map in the kernel address space. */
 924         vm_assert(vmproc[VMP_SYSTEM].vm_flags & VMF_INUSE);
 925
 926         if(bigpage_ok) {
 927                 int pde;
 928                 for(pde = 0; pde <= id_map_high_pde; pde++) {
 929                         phys_bytes addr;
 930                         addr = pde * I386_BIG_PAGE_SIZE;
 931                         vm_assert((addr & I386_VM_ADDR_MASK) == addr);
 932                         pt->pt_dir[pde] = addr | I386_VM_PRESENT |
 933                                 I386_VM_BIGPAGE | I386_VM_USER |
 934                                 I386_VM_WRITE | global_bit;
 935                 }
 936         } else {
 937                 vm_panic("VM: pt_mapkernel: no bigpage", NO_NUM);
 938         }
 939
 940         if(pagedir_pde >= 0) {
 941                 /* Kernel also wants to know about all page directories. */
 942                 pt->pt_dir[pagedir_pde] = pagedir_pde_val;
 943         }
 944
 945         for(i = 0; i < kernmappings; i++) {
 946                 if(pt_writemap(pt,
 947                         kern_mappings[i].lin_addr,
 948                         kern_mappings[i].phys_addr,
 949                         kern_mappings[i].len,
 950                         kern_mappings[i].flags, 0) != OK) {
 951                         vm_panic("pt_mapkernel: pt_writemap failed", NO_NUM);
 952                 }
 953         }
 954
 955         return OK;
 956 }
 957
 958 /*===========================================================================*
 959  *                              pt_cycle                                     *
 960  *===========================================================================*/
 961 PUBLIC void pt_cycle(void)
 962 {
 963         vm_checkspares();
 964 }
 965