kernel/arch/i386/memory.c

   1
   2 #include "kernel/kernel.h"
   3 #include "kernel/proc.h"
   4 #include "kernel/vm.h"
   5
   6 #include <machine/vm.h>
   7
   8 #include <minix/type.h>
   9 #include <minix/syslib.h>
  10 #include <minix/cpufeature.h>
  11 #include <string.h>
  12 #include <assert.h>
  13 #include <signal.h>
  14 #include <stdlib.h>
  15
  16 #include <machine/vm.h>
  17
  18 #include "oxpcie.h"
  19 #include "arch_proto.h"
  20 #include "kernel/proto.h"
  21 #include "kernel/debug.h"
  22
  23 #ifdef USE_APIC
  24 #include "apic.h"
  25 #ifdef USE_WATCHDOG
  26 #include "kernel/watchdog.h"
  27 #endif
  28 #endif
  29
  30 phys_bytes video_mem_vaddr = 0;
  31
  32 #define HASPT(procptr) ((procptr)->p_seg.p_cr3 != 0)
  33 static int nfreepdes = 0;
  34 #define MAXFREEPDES     2
  35 static int freepdes[MAXFREEPDES];
  36
  37 static u32_t phys_get32(phys_bytes v);
  38
  39 void mem_clear_mapcache(void)
  40 {
  41         int i;
  42         for(i = 0; i < nfreepdes; i++) {
  43                 struct proc *ptproc = get_cpulocal_var(ptproc);
  44                 int pde = freepdes[i];
  45                 u32_t *ptv;
  46                 assert(ptproc);
  47                 ptv = ptproc->p_seg.p_cr3_v;
  48                 assert(ptv);
  49                 ptv[pde] = 0;
  50         }
  51 }
  52
  53 /* This function sets up a mapping from within the kernel's address
  54  * space to any other area of memory, either straight physical
  55  * memory (pr == NULL) or a process view of memory, in 4MB windows.
  56  * I.e., it maps in 4MB chunks of virtual (or physical) address space
  57  * to 4MB chunks of kernel virtual address space.
  58  *
  59  * It recognizes pr already being in memory as a special case (no
  60  * mapping required).
  61  *
  62  * The target (i.e. in-kernel) mapping area is one of the freepdes[]
  63  * VM has earlier already told the kernel about that is available. It is
  64  * identified as the 'pde' parameter. This value can be chosen freely
  65  * by the caller, as long as it is in range (i.e. 0 or higher and corresonds
  66  * to a known freepde slot). It is up to the caller to keep track of which
  67  * freepde's are in use, and to determine which ones are free to use.
  68  *
  69  * The logical number supplied by the caller is translated into an actual
  70  * pde number to be used, and a pointer to it (linear address) is returned
  71  * for actual use by phys_copy or memset.
  72  */
  73 static phys_bytes createpde(
  74         const struct proc *pr,  /* Requested process, NULL for physical. */
  75         const phys_bytes linaddr,/* Address after segment translation. */
  76         phys_bytes *bytes,      /* Size of chunk, function may truncate it. */
  77         int free_pde_idx,       /* index of the free slot to use */
  78         int *changed            /* If mapping is made, this is set to 1. */
  79         )
  80 {
  81         u32_t pdeval;
  82         phys_bytes offset;
  83         int pde;
  84
  85         assert(free_pde_idx >= 0 && free_pde_idx < nfreepdes);
  86         pde = freepdes[free_pde_idx];
  87         assert(pde >= 0 && pde < 1024);
  88
  89         if(pr && ((pr == get_cpulocal_var(ptproc)) || iskernelp(pr))) {
  90                 /* Process memory is requested, and
  91                  * it's a process that is already in current page table, or
  92                  * the kernel, which is always there.
  93                  * Therefore linaddr is valid directly, with the requested
  94                  * size.
  95                  */
  96                 return linaddr;
  97         }
  98
  99         if(pr) {
 100                 /* Requested address is in a process that is not currently
 101                  * accessible directly. Grab the PDE entry of that process'
 102                  * page table that corresponds to the requested address.
 103                  */
 104                 assert(pr->p_seg.p_cr3_v);
 105                 pdeval = pr->p_seg.p_cr3_v[I386_VM_PDE(linaddr)];
 106         } else {
 107                 /* Requested address is physical. Make up the PDE entry. */
 108                 pdeval = (linaddr & I386_VM_ADDR_MASK_4MB) |
 109                         I386_VM_BIGPAGE | I386_VM_PRESENT |
 110                         I386_VM_WRITE | I386_VM_USER;
 111         }
 112
 113         /* Write the pde value that we need into a pde that the kernel
 114          * can access, into the currently loaded page table so it becomes
 115          * visible.
 116          */
 117         assert(get_cpulocal_var(ptproc)->p_seg.p_cr3_v);
 118         if(get_cpulocal_var(ptproc)->p_seg.p_cr3_v[pde] != pdeval) {
 119                 get_cpulocal_var(ptproc)->p_seg.p_cr3_v[pde] = pdeval;
 120                 *changed = 1;
 121         }
 122
 123         /* Memory is now available, but only the 4MB window of virtual
 124          * address space that we have mapped; calculate how much of
 125          * the requested range is visible and return that in *bytes,
 126          * if that is less than the requested range.
 127          */
 128         offset = linaddr & I386_VM_OFFSET_MASK_4MB; /* Offset in 4MB window. */
 129         *bytes = MIN(*bytes, I386_BIG_PAGE_SIZE - offset);
 130
 131         /* Return the linear address of the start of the new mapping. */
 132         return I386_BIG_PAGE_SIZE*pde + offset;
 133 }
 134
 135 /*===========================================================================*
 136  *                              lin_lin_copy                                 *
 137  *===========================================================================*/
 138 static int lin_lin_copy(struct proc *srcproc, vir_bytes srclinaddr,
 139         struct proc *dstproc, vir_bytes dstlinaddr, vir_bytes bytes)
 140 {
 141         u32_t addr;
 142         proc_nr_t procslot;
 143
 144         assert(get_cpulocal_var(ptproc));
 145         assert(get_cpulocal_var(proc_ptr));
 146         assert(read_cr3() == get_cpulocal_var(ptproc)->p_seg.p_cr3);
 147
 148         procslot = get_cpulocal_var(ptproc)->p_nr;
 149
 150         assert(procslot >= 0 && procslot < I386_VM_DIR_ENTRIES);
 151
 152         if(srcproc) assert(!RTS_ISSET(srcproc, RTS_SLOT_FREE));
 153         if(dstproc) assert(!RTS_ISSET(dstproc, RTS_SLOT_FREE));
 154         assert(!RTS_ISSET(get_cpulocal_var(ptproc), RTS_SLOT_FREE));
 155         assert(get_cpulocal_var(ptproc)->p_seg.p_cr3_v);
 156         if(srcproc) assert(!RTS_ISSET(srcproc, RTS_VMINHIBIT));
 157         if(dstproc) assert(!RTS_ISSET(dstproc, RTS_VMINHIBIT));
 158
 159         while(bytes > 0) {
 160                 phys_bytes srcptr, dstptr;
 161                 vir_bytes chunk = bytes;
 162                 int changed = 0;
 163
 164 #ifdef CONFIG_SMP
 165                 unsigned cpu = cpuid;
 166
 167                 if (srcproc && GET_BIT(srcproc->p_stale_tlb, cpu)) {
 168                         changed = 1;
 169                         UNSET_BIT(srcproc->p_stale_tlb, cpu);
 170                 }
 171                 if (dstproc && GET_BIT(dstproc->p_stale_tlb, cpu)) {
 172                         changed = 1;
 173                         UNSET_BIT(dstproc->p_stale_tlb, cpu);
 174                 }
 175 #endif
 176
 177                 /* Set up 4MB ranges. */
 178                 srcptr = createpde(srcproc, srclinaddr, &chunk, 0, &changed);
 179                 dstptr = createpde(dstproc, dstlinaddr, &chunk, 1, &changed);
 180                 if(changed)
 181                         reload_cr3();
 182
 183                 /* Copy pages. */
 184                 PHYS_COPY_CATCH(srcptr, dstptr, chunk, addr);
 185
 186                 if(addr) {
 187                         /* If addr is nonzero, a page fault was caught. */
 188
 189                         if(addr >= srcptr && addr < (srcptr + chunk)) {
 190                                 return EFAULT_SRC;
 191                         }
 192                         if(addr >= dstptr && addr < (dstptr + chunk)) {
 193                                 return EFAULT_DST;
 194                         }
 195
 196                         panic("lin_lin_copy fault out of range");
 197
 198                         /* Not reached. */
 199                         return EFAULT;
 200                 }
 201
 202                 /* Update counter and addresses for next iteration, if any. */
 203                 bytes -= chunk;
 204                 srclinaddr += chunk;
 205                 dstlinaddr += chunk;
 206         }
 207
 208         if(srcproc) assert(!RTS_ISSET(srcproc, RTS_SLOT_FREE));
 209         if(dstproc) assert(!RTS_ISSET(dstproc, RTS_SLOT_FREE));
 210         assert(!RTS_ISSET(get_cpulocal_var(ptproc), RTS_SLOT_FREE));
 211         assert(get_cpulocal_var(ptproc)->p_seg.p_cr3_v);
 212
 213         return OK;
 214 }
 215
 216
 217 static u32_t phys_get32(phys_bytes addr)
 218 {
 219         const u32_t v;
 220         int r;
 221
 222         if((r=lin_lin_copy(NULL, addr,
 223                 proc_addr(SYSTEM), (phys_bytes) &v, sizeof(v))) != OK) {
 224                 panic("lin_lin_copy for phys_get32 failed: %d",  r);
 225         }
 226
 227         return v;
 228 }
 229
 230 #if 0
 231 static char *cr0_str(u32_t e)
 232 {
 233         static char str[80];
 234         strcpy(str, "");
 235 #define FLAG(v) do { if(e & (v)) { strcat(str, #v " "); e &= ~v; } } while(0)
 236         FLAG(I386_CR0_PE);
 237         FLAG(I386_CR0_MP);
 238         FLAG(I386_CR0_EM);
 239         FLAG(I386_CR0_TS);
 240         FLAG(I386_CR0_ET);
 241         FLAG(I386_CR0_PG);
 242         FLAG(I386_CR0_WP);
 243         if(e) { strcat(str, " (++)"); }
 244         return str;
 245 }
 246
 247 static char *cr4_str(u32_t e)
 248 {
 249         static char str[80];
 250         strcpy(str, "");
 251         FLAG(I386_CR4_VME);
 252         FLAG(I386_CR4_PVI);
 253         FLAG(I386_CR4_TSD);
 254         FLAG(I386_CR4_DE);
 255         FLAG(I386_CR4_PSE);
 256         FLAG(I386_CR4_PAE);
 257         FLAG(I386_CR4_MCE);
 258         FLAG(I386_CR4_PGE);
 259         if(e) { strcat(str, " (++)"); }
 260         return str;
 261 }
 262 #endif
 263
 264 /*===========================================================================*
 265  *                              umap_virtual                                 *
 266  *===========================================================================*/
 267 phys_bytes umap_virtual(rp, seg, vir_addr, bytes)
 268 register struct proc *rp;       /* pointer to proc table entry for process */
 269 int seg;                        /* T, D, or S segment */
 270 vir_bytes vir_addr;             /* virtual address in bytes within the seg */
 271 vir_bytes bytes;                /* # of bytes to be copied */
 272 {
 273         phys_bytes phys = 0;
 274
 275         if(vm_lookup(rp, vir_addr, &phys, NULL) != OK) {
 276                 printf("SYSTEM:umap_virtual: vm_lookup of %s: seg 0x%x: 0x%lx failed\n", rp->p_name, seg, vir_addr);
 277                 phys = 0;
 278         } else {
 279                 if(phys == 0)
 280                         panic("vm_lookup returned phys: %d",  phys);
 281         }
 282
 283         if(phys == 0) {
 284                 printf("SYSTEM:umap_virtual: lookup failed\n");
 285                 return 0;
 286         }
 287
 288         /* Now make sure addresses are contiguous in physical memory
 289          * so that the umap makes sense.
 290          */
 291         if(bytes > 0 && vm_lookup_range(rp, vir_addr, NULL, bytes) != bytes) {
 292                 printf("umap_virtual: %s: %lu at 0x%lx (vir 0x%lx) not contiguous\n",
 293                         rp->p_name, bytes, vir_addr, vir_addr);
 294                 return 0;
 295         }
 296
 297         /* phys must be larger than 0 (or the caller will think the call
 298          * failed), and address must not cross a page boundary.
 299          */
 300         assert(phys);
 301
 302         return phys;
 303 }
 304
 305
 306 /*===========================================================================*
 307  *                              vm_lookup                                    *
 308  *===========================================================================*/
 309 int vm_lookup(const struct proc *proc, const vir_bytes virtual,
 310  phys_bytes *physical, u32_t *ptent)
 311 {
 312         u32_t *root, *pt;
 313         int pde, pte;
 314         u32_t pde_v, pte_v;
 315
 316         assert(proc);
 317         assert(physical);
 318         assert(!isemptyp(proc));
 319         assert(HASPT(proc));
 320
 321         /* Retrieve page directory entry. */
 322         root = (u32_t *) proc->p_seg.p_cr3;
 323         assert(!((u32_t) root % I386_PAGE_SIZE));
 324         pde = I386_VM_PDE(virtual);
 325         assert(pde >= 0 && pde < I386_VM_DIR_ENTRIES);
 326         pde_v = phys_get32((u32_t) (root + pde));
 327
 328         if(!(pde_v & I386_VM_PRESENT)) {
 329                 return EFAULT;
 330         }
 331
 332         /* We don't expect to ever see this. */
 333         if(pde_v & I386_VM_BIGPAGE) {
 334                 *physical = pde_v & I386_VM_ADDR_MASK_4MB;
 335                 if(ptent) *ptent = pde_v;
 336                 *physical += virtual & I386_VM_OFFSET_MASK_4MB;
 337         } else {
 338                 /* Retrieve page table entry. */
 339                 pt = (u32_t *) I386_VM_PFA(pde_v);
 340                 assert(!((u32_t) pt % I386_PAGE_SIZE));
 341                 pte = I386_VM_PTE(virtual);
 342                 assert(pte >= 0 && pte < I386_VM_PT_ENTRIES);
 343                 pte_v = phys_get32((u32_t) (pt + pte));
 344                 if(!(pte_v & I386_VM_PRESENT)) {
 345                         return EFAULT;
 346                 }
 347
 348                 if(ptent) *ptent = pte_v;
 349
 350                 /* Actual address now known; retrieve it and add page offset. */
 351                 *physical = I386_VM_PFA(pte_v);
 352                 *physical += virtual % I386_PAGE_SIZE;
 353         }
 354
 355         return OK;
 356 }
 357
 358 /*===========================================================================*
 359  *                              vm_lookup_range                              *
 360  *===========================================================================*/
 361 size_t vm_lookup_range(const struct proc *proc, vir_bytes vir_addr,
 362         phys_bytes *phys_addr, size_t bytes)
 363 {
 364         /* Look up the physical address corresponding to linear virtual address
 365          * 'vir_addr' for process 'proc'. Return the size of the range covered
 366          * by contiguous physical memory starting from that address; this may
 367          * be anywhere between 0 and 'bytes' inclusive. If the return value is
 368          * nonzero, and 'phys_addr' is non-NULL, 'phys_addr' will be set to the
 369          * base physical address of the range. 'vir_addr' and 'bytes' need not
 370          * be page-aligned, but the caller must have verified that the given
 371          * linear range is valid for the given process at all.
 372          */
 373         phys_bytes phys, next_phys;
 374         size_t len;
 375
 376         assert(proc);
 377         assert(bytes > 0);
 378         assert(HASPT(proc));
 379
 380         /* Look up the first page. */
 381         if (vm_lookup(proc, vir_addr, &phys, NULL) != OK)
 382                 return 0;
 383
 384         if (phys_addr != NULL)
 385                 *phys_addr = phys;
 386
 387         len = I386_PAGE_SIZE - (vir_addr % I386_PAGE_SIZE);
 388         vir_addr += len;
 389         next_phys = phys + len;
 390
 391         /* Look up any next pages and test physical contiguity. */
 392         while (len < bytes) {
 393                 if (vm_lookup(proc, vir_addr, &phys, NULL) != OK)
 394                         break;
 395
 396                 if (next_phys != phys)
 397                         break;
 398
 399                 len += I386_PAGE_SIZE;
 400                 vir_addr += I386_PAGE_SIZE;
 401                 next_phys += I386_PAGE_SIZE;
 402         }
 403
 404         /* We might now have overshot the requested length somewhat. */
 405         return MIN(bytes, len);
 406 }
 407
 408 /*===========================================================================*
 409  *                              vm_suspend                                *
 410  *===========================================================================*/
 411 static void vm_suspend(struct proc *caller, const struct proc *target,
 412         const vir_bytes linaddr, const vir_bytes len, const int type)
 413 {
 414         /* This range is not OK for this process. Set parameters
 415          * of the request and notify VM about the pending request.
 416          */
 417         assert(!RTS_ISSET(caller, RTS_VMREQUEST));
 418         assert(!RTS_ISSET(target, RTS_VMREQUEST));
 419
 420         RTS_SET(caller, RTS_VMREQUEST);
 421
 422         caller->p_vmrequest.req_type = VMPTYPE_CHECK;
 423         caller->p_vmrequest.target = target->p_endpoint;
 424         caller->p_vmrequest.params.check.start = linaddr;
 425         caller->p_vmrequest.params.check.length = len;
 426         caller->p_vmrequest.params.check.writeflag = 1;
 427         caller->p_vmrequest.type = type;
 428
 429         /* Connect caller on vmrequest wait queue. */
 430         if(!(caller->p_vmrequest.nextrequestor = vmrequest))
 431                 if(OK != send_sig(VM_PROC_NR, SIGKMEM))
 432                         panic("send_sig failed");
 433         vmrequest = caller;
 434 }
 435
 436 /*===========================================================================*
 437  *                              vm_check_range                               *
 438  *===========================================================================*/
 439 int vm_check_range(struct proc *caller, struct proc *target,
 440         vir_bytes vir_addr, size_t bytes)
 441 {
 442         /* Public interface to vm_suspend(), for use by kernel calls. On behalf
 443          * of 'caller', call into VM to check linear virtual address range of
 444          * process 'target', starting at 'vir_addr', for 'bytes' bytes. This
 445          * function assumes that it will called twice if VM returned an error
 446          * the first time (since nothing has changed in that case), and will
 447          * then return the error code resulting from the first call. Upon the
 448          * first call, a non-success error code is returned as well.
 449          */
 450         int r;
 451
 452         if ((caller->p_misc_flags & MF_KCALL_RESUME) &&
 453                         (r = caller->p_vmrequest.vmresult) != OK)
 454                 return r;
 455
 456         vm_suspend(caller, target, vir_addr, bytes, VMSTYPE_KERNELCALL);
 457
 458         return VMSUSPEND;
 459 }
 460
 461 /*===========================================================================*
 462  *                              delivermsg                                *
 463  *===========================================================================*/
 464 void delivermsg(struct proc *rp)
 465 {
 466         int r = OK;
 467
 468         assert(rp->p_misc_flags & MF_DELIVERMSG);
 469         assert(rp->p_delivermsg.m_source != NONE);
 470
 471         if (copy_msg_to_user(&rp->p_delivermsg,
 472                                 (message *) rp->p_delivermsg_vir)) {
 473                 printf("WARNING wrong user pointer 0x%08lx from "
 474                                 "process %s / %d\n",
 475                                 rp->p_delivermsg_vir,
 476                                 rp->p_name,
 477                                 rp->p_endpoint);
 478                 r = EFAULT;
 479         }
 480
 481         /* Indicate message has been delivered; address is 'used'. */
 482         rp->p_delivermsg.m_source = NONE;
 483         rp->p_misc_flags &= ~MF_DELIVERMSG;
 484
 485         if(!(rp->p_misc_flags & MF_CONTEXT_SET)) {
 486                 rp->p_reg.retreg = r;
 487         }
 488 }
 489
 490 #if 0
 491 static char *flagstr(u32_t e, const int dir)
 492 {
 493         static char str[80];
 494         strcpy(str, "");
 495         FLAG(I386_VM_PRESENT);
 496         FLAG(I386_VM_WRITE);
 497         FLAG(I386_VM_USER);
 498         FLAG(I386_VM_PWT);
 499         FLAG(I386_VM_PCD);
 500         FLAG(I386_VM_GLOBAL);
 501         if(dir)
 502                 FLAG(I386_VM_BIGPAGE);  /* Page directory entry only */
 503         else
 504                 FLAG(I386_VM_DIRTY);    /* Page table entry only */
 505         return str;
 506 }
 507
 508 static void vm_pt_print(u32_t *pagetable, const u32_t v)
 509 {
 510         int pte;
 511         int col = 0;
 512
 513         assert(!((u32_t) pagetable % I386_PAGE_SIZE));
 514
 515         for(pte = 0; pte < I386_VM_PT_ENTRIES; pte++) {
 516                 u32_t pte_v, pfa;
 517                 pte_v = phys_get32((u32_t) (pagetable + pte));
 518                 if(!(pte_v & I386_VM_PRESENT))
 519                         continue;
 520                 pfa = I386_VM_PFA(pte_v);
 521                 printf("%4d:%08lx:%08lx %2s ",
 522                         pte, v + I386_PAGE_SIZE*pte, pfa,
 523                         (pte_v & I386_VM_WRITE) ? "rw":"RO");
 524                 col++;
 525                 if(col == 3) { printf("\n"); col = 0; }
 526         }
 527         if(col > 0) printf("\n");
 528
 529         return;
 530 }
 531
 532 static void vm_print(u32_t *root)
 533 {
 534         int pde;
 535
 536         assert(!((u32_t) root % I386_PAGE_SIZE));
 537
 538         printf("page table 0x%lx:\n", root);
 539
 540         for(pde = 0; pde < I386_VM_DIR_ENTRIES; pde++) {
 541                 u32_t pde_v;
 542                 u32_t *pte_a;
 543                 pde_v = phys_get32((u32_t) (root + pde));
 544                 if(!(pde_v & I386_VM_PRESENT))
 545                         continue;
 546                 if(pde_v & I386_VM_BIGPAGE) {
 547                         printf("%4d: 0x%lx, flags %s\n",
 548                                 pde, I386_VM_PFA(pde_v), flagstr(pde_v, 1));
 549                 } else {
 550                         pte_a = (u32_t *) I386_VM_PFA(pde_v);
 551                         printf("%4d: pt %08lx %s\n",
 552                                 pde, pte_a, flagstr(pde_v, 1));
 553                         vm_pt_print(pte_a, pde * I386_VM_PT_ENTRIES * I386_PAGE_SIZE);
 554                         printf("\n");
 555                 }
 556         }
 557
 558
 559         return;
 560 }
 561 #endif
 562
 563 int vm_memset(endpoint_t who, phys_bytes ph, const u8_t c, phys_bytes bytes)
 564 {
 565         u32_t p;
 566         int r = OK;
 567         struct proc *whoptr = NULL;
 568
 569         /* NONE for physical, otherwise virtual */
 570         if(who != NONE) {
 571                 int n;
 572                 if(!isokendpt(who, &n)) return ESRCH;
 573                 whoptr = proc_addr(n);
 574         }
 575
 576         p = c | (c << 8) | (c << 16) | (c << 24);
 577
 578         assert(get_cpulocal_var(ptproc)->p_seg.p_cr3_v);
 579
 580         assert(!catch_pagefaults);
 581         catch_pagefaults=1;
 582
 583         /* With VM, we have to map in the memory (virtual or physical).
 584          * We can do this 4MB at a time.
 585          */
 586         while(bytes > 0) {
 587                 int changed = 0;
 588                 phys_bytes chunk = bytes, ptr, pfa;
 589                 ptr = createpde(whoptr, ph, &chunk, 0, &changed);
 590                 if(changed)
 591                         reload_cr3();
 592
 593                 /* We can memset as many bytes as we have remaining,
 594                  * or as many as remain in the 4MB chunk we mapped in.
 595                  */
 596                 if((pfa=phys_memset(ptr, p, chunk))) {
 597                         printf("kernel memset pagefault\n");
 598                         r = EFAULT;
 599                         break;
 600                 }
 601                 bytes -= chunk;
 602                 ph += chunk;
 603         }
 604
 605         assert(catch_pagefaults);
 606         catch_pagefaults=0;
 607
 608         assert(get_cpulocal_var(ptproc)->p_seg.p_cr3_v);
 609
 610         return OK;
 611 }
 612
 613 /*===========================================================================*
 614  *                              virtual_copy_f                               *
 615  *===========================================================================*/
 616 int virtual_copy_f(caller, src_addr, dst_addr, bytes, vmcheck)
 617 struct proc * caller;
 618 struct vir_addr *src_addr;      /* source virtual address */
 619 struct vir_addr *dst_addr;      /* destination virtual address */
 620 vir_bytes bytes;                /* # of bytes to copy  */
 621 int vmcheck;                    /* if nonzero, can return VMSUSPEND */
 622 {
 623 /* Copy bytes from virtual address src_addr to virtual address dst_addr. */
 624   struct vir_addr *vir_addr[2]; /* virtual source and destination address */
 625   int i, r;
 626   struct proc *procs[2];
 627
 628   assert((vmcheck && caller) || (!vmcheck && !caller));
 629
 630   /* Check copy count. */
 631   if (bytes <= 0) return(EDOM);
 632
 633   /* Do some more checks and map virtual addresses to physical addresses. */
 634   vir_addr[_SRC_] = src_addr;
 635   vir_addr[_DST_] = dst_addr;
 636
 637   for (i=_SRC_; i<=_DST_; i++) {
 638         endpoint_t proc_e = vir_addr[i]->proc_nr_e;
 639         int proc_nr;
 640         struct proc *p;
 641
 642         if(proc_e == NONE) {
 643                 p = NULL;
 644         } else {
 645                 if(!isokendpt(proc_e, &proc_nr)) {
 646                         printf("virtual_copy: no reasonable endpoint\n");
 647                         return ESRCH;
 648                 }
 649                 p = proc_addr(proc_nr);
 650         }
 651
 652         procs[i] = p;
 653   }
 654
 655   if(caller && (caller->p_misc_flags & MF_KCALL_RESUME)) {
 656         assert(caller->p_vmrequest.vmresult != VMSUSPEND);
 657         if(caller->p_vmrequest.vmresult != OK) {
 658                 return caller->p_vmrequest.vmresult;
 659         }
 660   }
 661
 662   if((r=lin_lin_copy(procs[_SRC_], vir_addr[_SRC_]->offset,
 663         procs[_DST_], vir_addr[_DST_]->offset, bytes)) != OK) {
 664         struct proc *target = NULL;
 665         phys_bytes lin;
 666         if(r != EFAULT_SRC && r != EFAULT_DST)
 667                 panic("lin_lin_copy failed: %d",  r);
 668         if(!vmcheck || !caller) {
 669                 return r;
 670         }
 671
 672         if(r == EFAULT_SRC) {
 673                 lin = vir_addr[_SRC_]->offset;
 674                 target = procs[_SRC_];
 675         } else if(r == EFAULT_DST) {
 676                 lin = vir_addr[_DST_]->offset;
 677                 target = procs[_DST_];
 678         } else {
 679                 panic("r strange: %d",  r);
 680         }
 681
 682         assert(caller);
 683         assert(target);
 684
 685         vm_suspend(caller, target, lin, bytes, VMSTYPE_KERNELCALL);
 686         return VMSUSPEND;
 687   }
 688
 689   return OK;
 690 }
 691
 692 /*===========================================================================*
 693  *                              data_copy                                    *
 694  *===========================================================================*/
 695 int data_copy(const endpoint_t from_proc, const vir_bytes from_addr,
 696         const endpoint_t to_proc, const vir_bytes to_addr,
 697         size_t bytes)
 698 {
 699   struct vir_addr src, dst;
 700
 701   src.offset = from_addr;
 702   dst.offset = to_addr;
 703   src.proc_nr_e = from_proc;
 704   dst.proc_nr_e = to_proc;
 705   assert(src.proc_nr_e != NONE);
 706   assert(dst.proc_nr_e != NONE);
 707
 708   return virtual_copy(&src, &dst, bytes);
 709 }
 710
 711 /*===========================================================================*
 712  *                              data_copy_vmcheck                            *
 713  *===========================================================================*/
 714 int data_copy_vmcheck(struct proc * caller,
 715         const endpoint_t from_proc, const vir_bytes from_addr,
 716         const endpoint_t to_proc, const vir_bytes to_addr,
 717         size_t bytes)
 718 {
 719   struct vir_addr src, dst;
 720
 721   src.offset = from_addr;
 722   dst.offset = to_addr;
 723   src.proc_nr_e = from_proc;
 724   dst.proc_nr_e = to_proc;
 725   assert(src.proc_nr_e != NONE);
 726   assert(dst.proc_nr_e != NONE);
 727
 728   return virtual_copy_vmcheck(caller, &src, &dst, bytes);
 729 }
 730
 731 void memory_init(void)
 732 {
 733         assert(nfreepdes == 0);
 734
 735         freepdes[nfreepdes++] = kinfo.freepde_start++;
 736         freepdes[nfreepdes++] = kinfo.freepde_start++;
 737
 738         assert(kinfo.freepde_start < I386_VM_DIR_ENTRIES);
 739         assert(nfreepdes == 2);
 740         assert(nfreepdes <= MAXFREEPDES);
 741 }
 742
 743 /*===========================================================================*
 744  *                              arch_proc_init                               *
 745  *===========================================================================*/
 746 void arch_proc_init(struct proc *pr, const u32_t ip, const u32_t sp, char *name)
 747 {
 748         arch_proc_reset(pr);
 749         strlcpy(pr->p_name, name, sizeof(pr->p_name));
 750
 751         /* set custom state we know */
 752         pr->p_reg.pc = ip;
 753         pr->p_reg.sp = sp;
 754 }
 755
 756 static int oxpcie_mapping_index = -1,
 757         lapic_mapping_index = -1,
 758         ioapic_first_index = -1,
 759         ioapic_last_index = -1,
 760         video_mem_mapping_index = -1;
 761
 762 extern char *video_mem;
 763
 764 int arch_phys_map(const int index,
 765                         phys_bytes *addr,
 766                         phys_bytes *len,
 767                         int *flags)
 768 {
 769         static int first = 1;
 770         int freeidx = 0;
 771         static char *ser_var = NULL;
 772
 773         if(first) {
 774                 video_mem_mapping_index = freeidx++;
 775
 776 #ifdef USE_APIC
 777                 if(lapic_addr)
 778                         lapic_mapping_index = freeidx++;
 779                 if (ioapic_enabled) {
 780                         ioapic_first_index = freeidx;
 781                         assert(nioapics > 0);
 782                         freeidx += nioapics;
 783                         ioapic_last_index = freeidx-1;
 784                 }
 785 #endif
 786
 787 #ifdef CONFIG_OXPCIE
 788                 if((ser_var = env_get("oxpcie"))) {
 789                         if(ser_var[0] != '0' || ser_var[1] != 'x') {
 790                                 printf("oxpcie address in hex please\n");
 791                         } else {
 792                                 printf("oxpcie address is %s\n", ser_var);
 793                                 oxpcie_mapping_index = freeidx++;
 794                         }
 795                 }
 796 #endif
 797
 798                 first = 0;
 799         }
 800
 801 #ifdef USE_APIC
 802         if (index == video_mem_mapping_index) {
 803                 /* map video memory in so we can print panic messages */
 804                 *addr = MULTIBOOT_VIDEO_BUFFER;
 805                 *len = I386_PAGE_SIZE;
 806                 *flags = 0;
 807                 return OK;
 808         }
 809         else if (index == lapic_mapping_index) {
 810                 /* map the local APIC if enabled */
 811                 if (!lapic_addr)
 812                         return EINVAL;
 813                 *addr = lapic_addr;
 814                 *len = 4 << 10 /* 4kB */;
 815                 *flags = VMMF_UNCACHED;
 816                 return OK;
 817         }
 818         else if (ioapic_enabled && index >= ioapic_first_index && index <= ioapic_last_index) {
 819                 int ioapic_idx = index - ioapic_first_index;
 820                 *addr = io_apic[ioapic_idx].paddr;
 821                 assert(*addr);
 822                 *len = 4 << 10 /* 4kB */;
 823                 *flags = VMMF_UNCACHED;
 824                 printf("ioapic map: addr 0x%lx\n", *addr);
 825                 return OK;
 826         }
 827 #endif
 828
 829 #if CONFIG_OXPCIE
 830         if(index == oxpcie_mapping_index) {
 831                 *addr = strtoul(ser_var+2, NULL, 16);
 832                 *len = 0x4000;
 833                 *flags = VMMF_UNCACHED;
 834                 return OK;
 835         }
 836 #endif
 837
 838         return EINVAL;
 839 }
 840
 841 int arch_phys_map_reply(const int index, const vir_bytes addr)
 842 {
 843 #ifdef USE_APIC
 844         /* if local APIC is enabled */
 845         if (index == lapic_mapping_index && lapic_addr) {
 846                 lapic_addr_vaddr = addr;
 847                 return OK;
 848         }
 849         else if (ioapic_enabled && index >= ioapic_first_index &&
 850                 index <= ioapic_last_index) {
 851                 int i = index - ioapic_first_index;
 852                 io_apic[i].vaddr = addr;
 853                 return OK;
 854         }
 855 #endif
 856
 857 #if CONFIG_OXPCIE
 858         if (index == oxpcie_mapping_index) {
 859                 oxpcie_set_vaddr((unsigned char *) addr);
 860                 return OK;
 861         }
 862 #endif
 863         if (index == video_mem_mapping_index) {
 864                 video_mem_vaddr =  addr;
 865                 return OK;
 866         }
 867
 868         return EINVAL;
 869 }
 870
 871 int arch_enable_paging(struct proc * caller)
 872 {
 873         assert(caller->p_seg.p_cr3);
 874
 875         /* load caller's page table */
 876         switch_address_space(caller);
 877
 878         video_mem = (char *) video_mem_vaddr;
 879
 880 #ifdef USE_APIC
 881         /* start using the virtual addresses */
 882
 883         /* if local APIC is enabled */
 884         if (lapic_addr) {
 885                 lapic_addr = lapic_addr_vaddr;
 886                 lapic_eoi_addr = LAPIC_EOI;
 887         }
 888         /* if IO apics are enabled */
 889         if (ioapic_enabled) {
 890                 int i;
 891
 892                 for (i = 0; i < nioapics; i++) {
 893                         io_apic[i].addr = io_apic[i].vaddr;
 894                 }
 895         }
 896 #if CONFIG_SMP
 897         barrier();
 898
 899         wait_for_APs_to_finish_booting();
 900 #endif
 901 #endif
 902
 903 #ifdef USE_WATCHDOG
 904         /*
 905          * We make sure that we don't enable the watchdog until paging is turned
 906          * on as we might get an NMI while switching and we might still use wrong
 907          * lapic address. Bad things would happen. It is unfortunate but such is
 908          * life
 909          */
 910         if (watchdog_enabled)
 911                 i386_watchdog_start();
 912 #endif
 913
 914         return OK;
 915 }
 916
 917 void release_address_space(struct proc *pr)
 918 {
 919         pr->p_seg.p_cr3_v = NULL;
 920 }
 921
 922 /* computes a checksum of a buffer of a given length. The byte sum must be zero */
 923 int platform_tbl_checksum_ok(void *ptr, unsigned int length)
 924 {
 925         u8_t total = 0;
 926         unsigned int i;
 927         for (i = 0; i < length; i++)
 928                 total += ((unsigned char *)ptr)[i];
 929         return !total;
 930 }
 931
 932 int platform_tbl_ptr(phys_bytes start,
 933                                         phys_bytes end,
 934                                         unsigned increment,
 935                                         void * buff,
 936                                         unsigned size,
 937                                         phys_bytes * phys_addr,
 938                                         int ((* cmp_f)(void *)))
 939 {
 940         phys_bytes addr;
 941
 942         for (addr = start; addr < end; addr += increment) {
 943                 phys_copy (addr, (phys_bytes) buff, size);
 944                 if (cmp_f(buff)) {
 945                         if (phys_addr)
 946                                 *phys_addr = addr;
 947                         return 1;
 948                 }
 949         }
 950         return 0;
 951 }