sys/arch/powerpc/ibm4xx/trap.c

   1 /*      $NetBSD: trap.c,v 1.52 2009/10/21 21:12:01 rmind Exp $  */
   2
   3 /*
   4  * Copyright 2001 Wasabi Systems, Inc.
   5  * All rights reserved.
   6  *
   7  * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  * 3. All advertising materials mentioning features or use of this software
  18  *    must display the following acknowledgement:
  19  *      This product includes software developed for the NetBSD Project by
  20  *      Wasabi Systems, Inc.
  21  * 4. The name of Wasabi Systems, Inc. may not be used to endorse
  22  *    or promote products derived from this software without specific prior
  23  *    written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
  26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  27  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  28  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
  29  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  30  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  31  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  32  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  33  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  35  * POSSIBILITY OF SUCH DAMAGE.
  36  */
  37
  38 /*
  39  * Copyright (C) 1995, 1996 Wolfgang Solfrank.
  40  * Copyright (C) 1995, 1996 TooLs GmbH.
  41  * All rights reserved.
  42  *
  43  * Redistribution and use in source and binary forms, with or without
  44  * modification, are permitted provided that the following conditions
  45  * are met:
  46  * 1. Redistributions of source code must retain the above copyright
  47  *    notice, this list of conditions and the following disclaimer.
  48  * 2. Redistributions in binary form must reproduce the above copyright
  49  *    notice, this list of conditions and the following disclaimer in the
  50  *    documentation and/or other materials provided with the distribution.
  51  * 3. All advertising materials mentioning features or use of this software
  52  *    must display the following acknowledgement:
  53  *      This product includes software developed by TooLs GmbH.
  54  * 4. The name of TooLs GmbH may not be used to endorse or promote products
  55  *    derived from this software without specific prior written permission.
  56  *
  57  * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
  58  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  59  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  60  * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  61  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  62  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  63  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  64  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  65  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  66  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  67  */
  68
  69 #include <sys/cdefs.h>
  70 __KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.52 2009/10/21 21:12:01 rmind Exp $");
  71
  72 #include "opt_altivec.h"
  73 #include "opt_ddb.h"
  74 #include "opt_kgdb.h"
  75
  76 #include <sys/param.h>
  77 #include <sys/proc.h>
  78 #include <sys/reboot.h>
  79 #include <sys/syscall.h>
  80 #include <sys/systm.h>
  81 #include <sys/pool.h>
  82 #include <sys/sa.h>
  83 #include <sys/savar.h>
  84 #include <sys/userret.h>
  85 #include <sys/kauth.h>
  86
  87 #if defined(KGDB)
  88 #include <sys/kgdb.h>
  89 #endif
  90
  91 #include <uvm/uvm_extern.h>
  92
  93 #include <dev/cons.h>
  94
  95 #include <machine/cpu.h>
  96 #include <machine/db_machdep.h>
  97 #include <machine/fpu.h>
  98 #include <machine/frame.h>
  99 #include <machine/pcb.h>
 100 #include <machine/psl.h>
 101 #include <machine/trap.h>
 102
 103 #include <powerpc/spr.h>
 104 #include <powerpc/ibm4xx/pmap.h>
 105 #include <powerpc/ibm4xx/tlb.h>
 106 #include <powerpc/fpu/fpu_extern.h>
 107
 108 /* These definitions should probably be somewhere else                  XXX */
 109 #define FIRSTARG        3               /* first argument is in reg 3 */
 110 #define NARGREG         8               /* 8 args are in registers */
 111 #define MOREARGS(sp)    ((void *)((int)(sp) + 8)) /* more args go here */
 112
 113 static int fix_unaligned(struct lwp *l, struct trapframe *frame);
 114
 115 void trap(struct trapframe *);  /* Called from locore / trap_subr */
 116 /* Why are these not defined in a header? */
 117 int badaddr(void *, size_t);
 118 int badaddr_read(void *, size_t, int *);
 119 int ctx_setup(int, int);
 120
 121 #ifdef DEBUG
 122 #define TDB_ALL 0x1
 123 int trapdebug = /* TDB_ALL */ 0;
 124 #define DBPRINTF(x, y)  if (trapdebug & (x)) printf y
 125 #else
 126 #define DBPRINTF(x, y)
 127 #endif
 128
 129 void
 130 trap(struct trapframe *frame)
 131 {
 132         struct lwp *l = curlwp;
 133         struct proc *p = l ? l->l_proc : NULL;
 134         struct pcb *pcb;
 135         int type = frame->exc;
 136         int ftype, rv;
 137         ksiginfo_t ksi;
 138
 139         KASSERT(l == NULL || (l->l_stat == LSONPROC));
 140
 141         if (frame->srr1 & PSL_PR) {
 142                 LWP_CACHE_CREDS(l, p);
 143                 type |= EXC_USER;
 144         }
 145
 146         ftype = VM_PROT_READ;
 147
 148         DBPRINTF(TDB_ALL, ("trap(%x) at %lx from frame %p &frame %p\n",
 149             type, frame->srr0, frame, &frame));
 150
 151         switch (type) {
 152         case EXC_DEBUG|EXC_USER:
 153                 {
 154                         int srr2, srr3;
 155
 156                         __asm volatile("mfspr %0,0x3f0" :
 157                             "=r" (rv), "=r" (srr2), "=r" (srr3) :);
 158                         printf("debug reg is %x srr2 %x srr3 %x\n", rv, srr2,
 159                             srr3);
 160                         /* XXX fall through or break here?! */
 161                 }
 162                 /*
 163                  * DEBUG intr -- probably single-step.
 164                  */
 165         case EXC_TRC|EXC_USER:
 166                 frame->srr1 &= ~PSL_SE;
 167                 KSI_INIT_TRAP(&ksi);
 168                 ksi.ksi_signo = SIGTRAP;
 169                 ksi.ksi_trap = EXC_TRC;
 170                 ksi.ksi_addr = (void *)frame->srr0;
 171                 trapsignal(l, &ksi);
 172                 break;
 173
 174         /*
 175          * If we could not find and install appropriate TLB entry, fall through.
 176          */
 177
 178         case EXC_DSI:
 179                 /* FALLTHROUGH */
 180         case EXC_DTMISS:
 181                 {
 182                         struct vm_map *map;
 183                         vaddr_t va;
 184                         struct faultbuf *fb = NULL;
 185
 186                         va = frame->dar;
 187                         if (frame->tf_xtra[TF_PID] == KERNEL_PID) {
 188                                 map = kernel_map;
 189                         } else {
 190                                 map = &p->p_vmspace->vm_map;
 191                                 if ((l->l_flag & LW_SA)
 192                                     && (~l->l_pflag & LP_SA_NOBLOCK)) {
 193                                         l->l_savp->savp_faultaddr = va;
 194                                         l->l_pflag |= LP_SA_PAGEFAULT;
 195                                 }
 196                         }
 197
 198                         if (frame->tf_xtra[TF_ESR] & (ESR_DST|ESR_DIZ))
 199                                 ftype = VM_PROT_WRITE;
 200
 201                         DBPRINTF(TDB_ALL,
 202                             ("trap(EXC_DSI) at %lx %s fault on %p esr %x\n",
 203                             frame->srr0,
 204                             (ftype & VM_PROT_WRITE) ? "write" : "read",
 205                             (void *)va, frame->tf_xtra[TF_ESR]));
 206                         rv = uvm_fault(map, trunc_page(va), ftype);
 207                         if (map != kernel_map) {
 208                                 l->l_pflag &= ~LP_SA_PAGEFAULT;
 209                         }
 210                         if (rv == 0)
 211                                 goto done;
 212                         pcb = lwp_getpcb(l);
 213                         if ((fb = pcb->pcb_onfault) != NULL) {
 214                                 frame->tf_xtra[TF_PID] = KERNEL_PID;
 215                                 frame->srr0 = fb->fb_pc;
 216                                 frame->srr1 |= PSL_IR; /* Re-enable IMMU */
 217                                 frame->fixreg[1] = fb->fb_sp;
 218                                 frame->fixreg[2] = fb->fb_r2;
 219                                 frame->fixreg[3] = 1; /* Return TRUE */
 220                                 frame->cr = fb->fb_cr;
 221                                 memcpy(&frame->fixreg[13], fb->fb_fixreg,
 222                                     sizeof(fb->fb_fixreg));
 223                                 goto done;
 224                         }
 225                 }
 226                 goto brain_damage;
 227
 228         case EXC_DSI|EXC_USER:
 229                 /* FALLTHROUGH */
 230         case EXC_DTMISS|EXC_USER:
 231                 if (frame->tf_xtra[TF_ESR] & (ESR_DST|ESR_DIZ))
 232                         ftype = VM_PROT_WRITE;
 233
 234                 DBPRINTF(TDB_ALL,
 235                     ("trap(EXC_DSI|EXC_USER) at %lx %s fault on %lx %x\n",
 236                     frame->srr0, (ftype & VM_PROT_WRITE) ? "write" : "read",
 237                     frame->dar, frame->tf_xtra[TF_ESR]));
 238                 KASSERT(l == curlwp && (l->l_stat == LSONPROC));
 239                 if (l->l_flag & LW_SA) {
 240                         l->l_savp->savp_faultaddr = (vaddr_t)frame->dar;
 241                         l->l_pflag |= LP_SA_PAGEFAULT;
 242                 }
 243                 rv = uvm_fault(&p->p_vmspace->vm_map, trunc_page(frame->dar),
 244                     ftype);
 245                 if (rv == 0) {
 246                         l->l_pflag &= ~LP_SA_PAGEFAULT;
 247                         break;
 248                 }
 249                 KSI_INIT_TRAP(&ksi);
 250                 ksi.ksi_signo = SIGSEGV;
 251                 ksi.ksi_trap = EXC_DSI;
 252                 ksi.ksi_addr = (void *)frame->dar;
 253                 if (rv == ENOMEM) {
 254                         printf("UVM: pid %d (%s) lid %d, uid %d killed: "
 255                             "out of swap\n",
 256                             p->p_pid, p->p_comm, l->l_lid,
 257                             l->l_cred ?
 258                             kauth_cred_geteuid(l->l_cred) : -1);
 259                         ksi.ksi_signo = SIGKILL;
 260                 }
 261                 trapsignal(l, &ksi);
 262                 l->l_pflag &= ~LP_SA_PAGEFAULT;
 263                 break;
 264
 265         case EXC_ITMISS|EXC_USER:
 266         case EXC_ISI|EXC_USER:
 267                 if (l->l_flag & LW_SA) {
 268                         l->l_savp->savp_faultaddr = (vaddr_t)frame->srr0;
 269                         l->l_pflag |= LP_SA_PAGEFAULT;
 270                 }
 271                 ftype = VM_PROT_EXECUTE;
 272                 DBPRINTF(TDB_ALL,
 273                     ("trap(EXC_ISI|EXC_USER) at %lx execute fault tf %p\n",
 274                     frame->srr0, frame));
 275                 rv = uvm_fault(&p->p_vmspace->vm_map, trunc_page(frame->srr0),
 276                     ftype);
 277                 if (rv == 0) {
 278                         l->l_pflag &= ~LP_SA_PAGEFAULT;
 279                         break;
 280                 }
 281                 KSI_INIT_TRAP(&ksi);
 282                 ksi.ksi_signo = SIGSEGV;
 283                 ksi.ksi_trap = EXC_ISI;
 284                 ksi.ksi_addr = (void *)frame->srr0;
 285                 ksi.ksi_code = (rv == EACCES ? SEGV_ACCERR : SEGV_MAPERR);
 286                 trapsignal(l, &ksi);
 287                 l->l_pflag &= ~LP_SA_PAGEFAULT;
 288                 break;
 289
 290         case EXC_AST|EXC_USER:
 291                 curcpu()->ci_astpending = 0;    /* we are about to do it */
 292                 uvmexp.softs++;
 293                 if (l->l_pflag & LP_OWEUPC) {
 294                         l->l_pflag &= ~LP_OWEUPC;
 295                         ADDUPROF(l);
 296                 }
 297                 /* Check whether we are being preempted. */
 298                 if (curcpu()->ci_want_resched)
 299                         preempt();
 300                 break;
 301
 302
 303         case EXC_ALI|EXC_USER:
 304                 if (fix_unaligned(l, frame) != 0) {
 305                         KSI_INIT_TRAP(&ksi);
 306                         ksi.ksi_signo = SIGBUS;
 307                         ksi.ksi_trap = EXC_ALI;
 308                         ksi.ksi_addr = (void *)frame->dar;
 309                         trapsignal(l, &ksi);
 310                 } else
 311                         frame->srr0 += 4;
 312                 break;
 313
 314         case EXC_PGM|EXC_USER:
 315                 /*
 316                  * Illegal insn:
 317                  *
 318                  * let's try to see if it's FPU and can be emulated.
 319                  */
 320                 uvmexp.traps++;
 321                 pcb = lwp_getpcb(l);
 322
 323                 if (!(pcb->pcb_flags & PCB_FPU)) {
 324                         memset(&pcb->pcb_fpu, 0, sizeof(pcb->pcb_fpu));
 325                         pcb->pcb_flags |= PCB_FPU;
 326                 }
 327
 328                 if ((rv = fpu_emulate(frame, (struct fpreg *)&pcb->pcb_fpu))) {
 329                         KSI_INIT_TRAP(&ksi);
 330                         ksi.ksi_signo = rv;
 331                         ksi.ksi_trap = EXC_PGM;
 332                         ksi.ksi_addr = (void *)frame->srr0;
 333                         trapsignal(l, &ksi);
 334                 }
 335                 break;
 336
 337         case EXC_MCHK:
 338                 {
 339                         struct faultbuf *fb;
 340
 341                         pcb = lwp_getpcb(l);
 342                         if ((fb = pcb->pcb_onfault) != NULL) {
 343                                 frame->tf_xtra[TF_PID] = KERNEL_PID;
 344                                 frame->srr0 = fb->fb_pc;
 345                                 frame->srr1 |= PSL_IR; /* Re-enable IMMU */
 346                                 frame->fixreg[1] = fb->fb_sp;
 347                                 frame->fixreg[2] = fb->fb_r2;
 348                                 frame->fixreg[3] = 1; /* Return TRUE */
 349                                 frame->cr = fb->fb_cr;
 350                                 memcpy(&frame->fixreg[13], fb->fb_fixreg,
 351                                     sizeof(fb->fb_fixreg));
 352                                 goto done;
 353                         }
 354                 }
 355                 goto brain_damage;
 356         default:
 357  brain_damage:
 358                 printf("trap type 0x%x at 0x%lx\n", type, frame->srr0);
 359 #if defined(DDB) || defined(KGDB)
 360                 if (kdb_trap(type, frame))
 361                         goto done;
 362 #endif
 363 #ifdef TRAP_PANICWAIT
 364                 printf("Press a key to panic.\n");
 365                 cngetc();
 366 #endif
 367                 panic("trap");
 368         }
 369
 370         /* Invoke MI userret code */
 371         mi_userret(l);
 372  done:
 373         return;
 374 }
 375
 376 int
 377 ctx_setup(int ctx, int srr1)
 378 {
 379         volatile struct pmap *pm;
 380
 381         /* Update PID if we're returning to user mode. */
 382         if (srr1 & PSL_PR) {
 383                 pm = curproc->p_vmspace->vm_map.pmap;
 384                 if (!pm->pm_ctx) {
 385                         ctx_alloc(__UNVOLATILE(pm));
 386                 }
 387                 ctx = pm->pm_ctx;
 388                 if (srr1 & PSL_SE) {
 389                         int dbreg, mask = 0x48000000;
 390                                 /*
 391                                  * Set the Internal Debug and
 392                                  * Instruction Completion bits of
 393                                  * the DBCR0 register.
 394                                  *
 395                                  * XXX this is also used by jtag debuggers...
 396                                  */
 397                         __asm volatile("mfspr %0,0x3f2;"
 398                             "or %0,%0,%1;"
 399                             "mtspr 0x3f2,%0;" :
 400                             "=&r" (dbreg) : "r" (mask));
 401                 }
 402         }
 403         else if (!ctx) {
 404                 ctx = KERNEL_PID;
 405         }
 406         return (ctx);
 407 }
 408
 409 /*
 410  * Used by copyin()/copyout()
 411  */
 412 extern vaddr_t vmaprange(struct proc *, vaddr_t, vsize_t, int);
 413 extern void vunmaprange(vaddr_t, vsize_t);
 414 static int bigcopyin(const void *, void *, size_t );
 415 static int bigcopyout(const void *, void *, size_t );
 416
 417 int
 418 copyin(const void *udaddr, void *kaddr, size_t len)
 419 {
 420         struct pmap *pm = curproc->p_vmspace->vm_map.pmap;
 421         int msr, pid, tmp, ctx, count=0;
 422         struct faultbuf env;
 423
 424         /* For bigger buffers use the faster copy */
 425         if (len > 1024)
 426                 return (bigcopyin(udaddr, kaddr, len));
 427
 428         if (setfault(&env)) {
 429                 curpcb->pcb_onfault = 0;
 430                 return EFAULT;
 431         }
 432
 433         if (!(ctx = pm->pm_ctx)) {
 434                 /* No context -- assign it one */
 435                 ctx_alloc(pm);
 436                 ctx = pm->pm_ctx;
 437         }
 438
 439         __asm volatile(
 440                 "   mfmsr %[msr];"          /* Save MSR */
 441                 "   li %[pid],0x20; "
 442                 "   andc %[pid],%[msr],%[pid]; mtmsr %[pid];"   /* Disable IMMU */
 443                 "   mfpid %[pid];"          /* Save old PID */
 444                 "   sync; isync;"
 445
 446                 "   srwi. %[count],%[len],0x2;"     /* How many words? */
 447                 "   beq-  2f;"              /* No words. Go do bytes */
 448                 "   mtctr %[count];"
 449                 "1: mtpid %[ctx]; sync;"
 450                 "   lswi %[tmp],%[udaddr],4;"       /* Load user word */
 451                 "   addi %[udaddr],%[udaddr],0x4;"  /* next udaddr word */
 452                 "   sync; isync;"
 453                 "   mtpid %[pid];sync;"
 454                 "   stswi %[tmp],%[kaddr],4;"        /* Store kernel word */
 455                 "   dcbf 0,%[kaddr];"           /* flush cache */
 456                 "   addi %[kaddr],%[kaddr],0x4;"    /* next udaddr word */
 457                 "   sync; isync;"
 458                 "   bdnz 1b;"               /* repeat */
 459
 460                 "2: andi. %[count],%[len],0x3;"     /* How many remaining bytes? */
 461                 "   addi %[count],%[count],0x1;"
 462                 "   mtctr %[count];"
 463                 "3: bdz 10f;"               /* while count */
 464                 "   mtpid %[ctx];sync;"
 465                 "   lbz %[tmp],0(%[udaddr]);"       /* Load user byte */
 466                 "   addi %[udaddr],%[udaddr],0x1;"  /* next udaddr byte */
 467                 "   sync; isync;"
 468                 "   mtpid %[pid]; sync;"
 469                 "   stb %[tmp],0(%[kaddr]);"        /* Store kernel byte */
 470                 "   dcbf 0,%[kaddr];"           /* flush cache */
 471                 "   addi %[kaddr],%[kaddr],0x1;"
 472                 "   sync; isync;"
 473                 "   b 3b;"
 474                 "10:mtpid %[pid]; mtmsr %[msr]; sync; isync;" /* Restore PID and MSR */
 475                 : [msr] "=&r" (msr), [pid] "=&r" (pid), [tmp] "=&r" (tmp)
 476                 : [udaddr] "b" (udaddr), [ctx] "b" (ctx), [kaddr] "b" (kaddr), [len] "b" (len), [count] "b" (count));
 477
 478         curpcb->pcb_onfault = 0;
 479         return 0;
 480 }
 481
 482 static int
 483 bigcopyin(const void *udaddr, void *kaddr, size_t len)
 484 {
 485         const char *up;
 486         char *kp = kaddr;
 487         struct lwp *l = curlwp;
 488         struct proc *p;
 489         int error;
 490
 491         if (!l) {
 492                 return EFAULT;
 493         }
 494
 495         p = l->l_proc;
 496
 497         /*
 498          * Stolen from physio():
 499          */
 500         error = uvm_vslock(p->p_vmspace, __UNCONST(udaddr), len, VM_PROT_READ);
 501         if (error) {
 502                 return EFAULT;
 503         }
 504         up = (char *)vmaprange(p, (vaddr_t)udaddr, len, VM_PROT_READ);
 505
 506         memcpy(kp, up, len);
 507         vunmaprange((vaddr_t)up, len);
 508         uvm_vsunlock(p->p_vmspace, __UNCONST(udaddr), len);
 509
 510         return 0;
 511 }
 512
 513 int
 514 copyout(const void *kaddr, void *udaddr, size_t len)
 515 {
 516         struct pmap *pm = curproc->p_vmspace->vm_map.pmap;
 517         int msr, pid, tmp, ctx, count=0;
 518         struct faultbuf env;
 519
 520         /* For big copies use more efficient routine */
 521         if (len > 1024)
 522                 return (bigcopyout(kaddr, udaddr, len));
 523
 524         if (setfault(&env)) {
 525                 curpcb->pcb_onfault = 0;
 526                 return EFAULT;
 527         }
 528
 529         if (!(ctx = pm->pm_ctx)) {
 530                 /* No context -- assign it one */
 531                 ctx_alloc(pm);
 532                 ctx = pm->pm_ctx;
 533         }
 534
 535         __asm volatile(
 536                 "   mfmsr %[msr];"          /* Save MSR */ \
 537                 "   li %[pid],0x20; " \
 538                 "   andc %[pid],%[msr],%[pid]; mtmsr %[pid];"   /* Disable IMMU */ \
 539                 "   mfpid %[pid];"          /* Save old PID */ \
 540                 "   sync; isync;"
 541
 542                 "   srwi. %[count],%[len],0x2;"     /* How many words? */
 543                 "   beq-  2f;"              /* No words. Go do bytes */
 544                 "   mtctr %[count];"
 545                 "1: mtpid %[pid];sync;"
 546                 "   lswi %[tmp],%[kaddr],4;"        /* Load kernel word */
 547                 "   addi %[kaddr],%[kaddr],0x4;"    /* next kaddr word */
 548                 "   sync; isync;"
 549                 "   mtpid %[ctx]; sync;"
 550                 "   stswi %[tmp],%[udaddr],4;"       /* Store user word */
 551                 "   dcbf 0,%[udaddr];"          /* flush cache */
 552                 "   addi %[udaddr],%[udaddr],0x4;"  /* next udaddr word */
 553                 "   sync; isync;"
 554                 "   bdnz 1b;"               /* repeat */
 555
 556                 "2: andi. %[count],%[len],0x3;"     /* How many remaining bytes? */
 557                 "   addi %[count],%[count],0x1;"
 558                 "   mtctr %[count];"
 559                 "3: bdz  10f;"              /* while count */
 560                 "   mtpid %[pid];sync;"
 561                 "   lbz %[tmp],0(%[kaddr]);"        /* Load kernel byte */
 562                 "   addi %[kaddr],%[kaddr],0x1;"    /* next kaddr byte */
 563                 "   sync; isync;"
 564                 "   mtpid %[ctx]; sync;"
 565                 "   stb %[tmp],0(%[udaddr]);"       /* Store user byte */
 566                 "   dcbf 0,%[udaddr];"          /* flush cache */
 567                 "   addi %[udaddr],%[udaddr],0x1;"
 568                 "   sync; isync;"
 569                 "   b 3b;"
 570                 "10:mtpid %[pid]; mtmsr %[msr]; sync; isync;" /* Restore PID and MSR */
 571                 : [msr] "=&r" (msr), [pid] "=&r" (pid), [tmp] "=&r" (tmp)
 572                 : [udaddr] "b" (udaddr), [ctx] "b" (ctx), [kaddr] "b" (kaddr), [len] "b" (len), [count] "b" (count));
 573
 574         curpcb->pcb_onfault = 0;
 575         return 0;
 576 }
 577
 578 static int
 579 bigcopyout(const void *kaddr, void *udaddr, size_t len)
 580 {
 581         char *up;
 582         const char *kp = (const char *)kaddr;
 583         struct lwp *l = curlwp;
 584         struct proc *p;
 585         int error;
 586
 587         if (!l) {
 588                 return EFAULT;
 589         }
 590
 591         p = l->l_proc;
 592
 593         /*
 594          * Stolen from physio():
 595          */
 596         error = uvm_vslock(p->p_vmspace, udaddr, len, VM_PROT_WRITE);
 597         if (error) {
 598                 return EFAULT;
 599         }
 600         up = (char *)vmaprange(p, (vaddr_t)udaddr, len,
 601             VM_PROT_READ | VM_PROT_WRITE);
 602
 603         memcpy(up, kp, len);
 604         vunmaprange((vaddr_t)up, len);
 605         uvm_vsunlock(p->p_vmspace, udaddr, len);
 606
 607         return 0;
 608 }
 609
 610 /*
 611  * kcopy(const void *src, void *dst, size_t len);
 612  *
 613  * Copy len bytes from src to dst, aborting if we encounter a fatal
 614  * page fault.
 615  *
 616  * kcopy() _must_ save and restore the old fault handler since it is
 617  * called by uiomove(), which may be in the path of servicing a non-fatal
 618  * page fault.
 619  */
 620 int
 621 kcopy(const void *src, void *dst, size_t len)
 622 {
 623         struct faultbuf env, *oldfault;
 624
 625         oldfault = curpcb->pcb_onfault;
 626         if (setfault(&env)) {
 627                 curpcb->pcb_onfault = oldfault;
 628                 return EFAULT;
 629         }
 630
 631         memcpy(dst, src, len);
 632
 633         curpcb->pcb_onfault = oldfault;
 634         return 0;
 635 }
 636
 637 int
 638 badaddr(void *addr, size_t size)
 639 {
 640
 641         return badaddr_read(addr, size, NULL);
 642 }
 643
 644 int
 645 badaddr_read(void *addr, size_t size, int *rptr)
 646 {
 647         struct faultbuf env;
 648         int x;
 649
 650         /* Get rid of any stale machine checks that have been waiting.  */
 651         __asm volatile ("sync; isync");
 652
 653         if (setfault(&env)) {
 654                 curpcb->pcb_onfault = 0;
 655                 __asm volatile ("sync");
 656                 return 1;
 657         }
 658
 659         __asm volatile ("sync");
 660
 661         switch (size) {
 662         case 1:
 663                 x = *(volatile int8_t *)addr;
 664                 break;
 665         case 2:
 666                 x = *(volatile int16_t *)addr;
 667                 break;
 668         case 4:
 669                 x = *(volatile int32_t *)addr;
 670                 break;
 671         default:
 672                 panic("badaddr: invalid size (%d)", size);
 673         }
 674
 675         /* Make sure we took the machine check, if we caused one. */
 676         __asm volatile ("sync; isync");
 677
 678         curpcb->pcb_onfault = 0;
 679         __asm volatile ("sync");        /* To be sure. */
 680
 681         /* Use the value to avoid reorder. */
 682         if (rptr)
 683                 *rptr = x;
 684
 685         return 0;
 686 }
 687
 688 /*
 689  * For now, this only deals with the particular unaligned access case
 690  * that gcc tends to generate.  Eventually it should handle all of the
 691  * possibilities that can happen on a 32-bit PowerPC in big-endian mode.
 692  */
 693
 694 static int
 695 fix_unaligned(struct lwp *l, struct trapframe *frame)
 696 {
 697
 698         return -1;
 699 }
 700
 701 /*
 702  * Start a new LWP
 703  */
 704 void
 705 startlwp(void *arg)
 706 {
 707         int err;
 708         ucontext_t *uc = arg;
 709         struct lwp *l = curlwp;
 710
 711         err = cpu_setmcontext(l, &uc->uc_mcontext, uc->uc_flags);
 712 #if DIAGNOSTIC
 713         if (err) {
 714                 printf("Error %d from cpu_setmcontext.", err);
 715         }
 716 #endif
 717         pool_put(&lwp_uc_pool, uc);
 718
 719         upcallret(l);
 720 }
 721
 722 /*
 723  * XXX This is a terrible name.
 724  */
 725 void
 726 upcallret(struct lwp *l)
 727 {
 728
 729         /* Invoke MI userret code */
 730         mi_userret(l);
 731 }