sys/arch/powerpc/oea/altivec.c

   1 /*      $NetBSD: altivec.c,v 1.14 2008/04/08 02:33:03 garbled Exp $     */
   2
   3 /*
   4  * Copyright (C) 1996 Wolfgang Solfrank.
   5  * Copyright (C) 1996 TooLs GmbH.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. All advertising materials mentioning features or use of this software
  17  *    must display the following acknowledgement:
  18  *      This product includes software developed by TooLs GmbH.
  19  * 4. The name of TooLs GmbH may not be used to endorse or promote products
  20  *    derived from this software without specific prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
  23  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  24  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  25  * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  27  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  28  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  29  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  30  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  31  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __KERNEL_RCSID(0, "$NetBSD: altivec.c,v 1.14 2008/04/08 02:33:03 garbled Exp $");
  36
  37 #include "opt_multiprocessor.h"
  38
  39 #include <sys/param.h>
  40 #include <sys/proc.h>
  41 #include <sys/systm.h>
  42 #include <sys/malloc.h>
  43 #include <sys/pool.h>
  44
  45 #include <uvm/uvm_extern.h>
  46
  47 #include <powerpc/altivec.h>
  48 #include <powerpc/spr.h>
  49 #include <powerpc/psl.h>
  50
  51 #ifdef MULTIPROCESSOR
  52 #include <arch/powerpc/pic/picvar.h>
  53 #include <arch/powerpc/pic/ipivar.h>
  54 static void mp_save_vec_lwp(struct lwp *);
  55 #endif
  56
  57 void
  58 enable_vec(void)
  59 {
  60         struct cpu_info *ci = curcpu();
  61         struct lwp *l = curlwp;
  62         struct pcb *pcb = lwp_getpcb(l);
  63         struct trapframe *tf = trapframe(l);
  64         struct vreg *vr = &pcb->pcb_vr;
  65         register_t msr;
  66
  67         KASSERT(pcb->pcb_veccpu == NULL);
  68
  69         pcb->pcb_flags |= PCB_ALTIVEC;
  70
  71         /*
  72          * Enable AltiVec temporarily (and disable interrupts).
  73          */
  74         msr = mfmsr();
  75         mtmsr((msr & ~PSL_EE) | PSL_VEC);
  76         __asm volatile ("isync");
  77         if (ci->ci_veclwp) {
  78                 save_vec_cpu();
  79         }
  80         KASSERT(curcpu()->ci_veclwp == NULL);
  81
  82         /*
  83          * Restore VSCR by first loading it into a vector and then into VSCR.
  84          * (this needs to done before loading the user's vector registers
  85          * since we need to use a scratch vector register)
  86          */
  87         __asm volatile("vxor %2,%2,%2; lvewx %2,%0,%1; mtvscr %2" \
  88             ::  "b"(vr), "r"(offsetof(struct vreg, vscr)), "n"(0));
  89
  90         /*
  91          * VRSAVE will be restored when trap frame returns
  92          */
  93         tf->tf_xtra[TF_VRSAVE] = vr->vrsave;
  94
  95 #define LVX(n,vr)       __asm /*volatile*/("lvx %2,%0,%1" \
  96             ::  "b"(vr), "r"(offsetof(struct vreg, vreg[n])), "n"(n));
  97
  98         /*
  99          * Load all 32 vector registers
 100          */
 101         LVX( 0,vr);     LVX( 1,vr);     LVX( 2,vr);     LVX( 3,vr);
 102         LVX( 4,vr);     LVX( 5,vr);     LVX( 6,vr);     LVX( 7,vr);
 103         LVX( 8,vr);     LVX( 9,vr);     LVX(10,vr);     LVX(11,vr);
 104         LVX(12,vr);     LVX(13,vr);     LVX(14,vr);     LVX(15,vr);
 105
 106         LVX(16,vr);     LVX(17,vr);     LVX(18,vr);     LVX(19,vr);
 107         LVX(20,vr);     LVX(21,vr);     LVX(22,vr);     LVX(23,vr);
 108         LVX(24,vr);     LVX(25,vr);     LVX(26,vr);     LVX(27,vr);
 109         LVX(28,vr);     LVX(29,vr);     LVX(30,vr);     LVX(31,vr);
 110         __asm volatile ("isync");
 111
 112         /*
 113          * Enable AltiVec when we return to user-mode.
 114          * Record the new ownership of the AltiVec unit.
 115          */
 116         curcpu()->ci_veclwp = l;
 117         pcb->pcb_veccpu = curcpu();
 118         pcb->pcb_flags |= PCB_OWNALTIVEC;
 119         __asm volatile ("sync");
 120
 121         /*
 122          * Restore MSR (turn off AltiVec)
 123          */
 124         mtmsr(msr);
 125 }
 126
 127 void
 128 save_vec_cpu(void)
 129 {
 130         struct cpu_info *ci = curcpu();
 131         struct lwp *l;
 132         struct pcb *pcb;
 133         struct vreg *vr;
 134         struct trapframe *tf;
 135         register_t msr;
 136
 137         /*
 138          * Turn on AltiVEC, turn off interrupts.
 139          */
 140         msr = mfmsr();
 141         mtmsr((msr & ~PSL_EE) | PSL_VEC);
 142         __asm volatile ("isync");
 143         l = ci->ci_veclwp;
 144         if (l == NULL)
 145                 goto out;
 146         pcb = lwp_getpcb(l);
 147         vr = &pcb->pcb_vr;
 148         tf = trapframe(l);
 149
 150 #define STVX(n,vr)      __asm /*volatile*/("stvx %2,%0,%1" \
 151             ::  "b"(vr), "r"(offsetof(struct vreg, vreg[n])), "n"(n));
 152
 153         /*
 154          * Save the vector registers.
 155          */
 156         STVX( 0,vr);    STVX( 1,vr);    STVX( 2,vr);    STVX( 3,vr);
 157         STVX( 4,vr);    STVX( 5,vr);    STVX( 6,vr);    STVX( 7,vr);
 158         STVX( 8,vr);    STVX( 9,vr);    STVX(10,vr);    STVX(11,vr);
 159         STVX(12,vr);    STVX(13,vr);    STVX(14,vr);    STVX(15,vr);
 160
 161         STVX(16,vr);    STVX(17,vr);    STVX(18,vr);    STVX(19,vr);
 162         STVX(20,vr);    STVX(21,vr);    STVX(22,vr);    STVX(23,vr);
 163         STVX(24,vr);    STVX(25,vr);    STVX(26,vr);    STVX(27,vr);
 164         STVX(28,vr);    STVX(29,vr);    STVX(30,vr);    STVX(31,vr);
 165
 166         /*
 167          * Save VSCR (this needs to be done after save the vector registers
 168          * since we need to use one as scratch).
 169          */
 170         __asm volatile("mfvscr %2; stvewx %2,%0,%1" \
 171             ::  "b"(vr), "r"(offsetof(struct vreg, vscr)), "n"(0));
 172
 173         /*
 174          * Save VRSAVE
 175          */
 176         vr->vrsave = tf->tf_xtra[TF_VRSAVE];
 177
 178         /*
 179          * Note that we aren't using any CPU resources and stop any
 180          * data streams.
 181          */
 182         pcb->pcb_veccpu = NULL;
 183         ci->ci_veclwp = NULL;
 184         __asm volatile ("dssall; sync");
 185
 186  out:
 187
 188         /*
 189          * Restore MSR (turn off AltiVec)
 190          */
 191         mtmsr(msr);
 192 }
 193
 194 #ifdef MULTIPROCESSOR
 195 /*
 196  * Save a process's AltiVEC state to its PCB.  The state may be in any CPU.
 197  * The process must either be curproc or traced by curproc (and stopped).
 198  * (The point being that the process must not run on another CPU during
 199  * this function).
 200  */
 201 static void
 202 mp_save_vec_lwp(struct lwp *l)
 203 {
 204         struct pcb *pcb = lwp_getpcb(l);
 205         struct cpu_info *veccpu;
 206         int i;
 207
 208         /*
 209          * Send an IPI to the other CPU with the data and wait for that CPU
 210          * to flush the data.  Note that the other CPU might have switched
 211          * to a different proc's AltiVEC state by the time it receives the IPI,
 212          * but that will only result in an unnecessary reload.
 213          */
 214
 215         veccpu = pcb->pcb_veccpu;
 216         if (veccpu == NULL)
 217                 return;
 218
 219         ppc_send_ipi(veccpu->ci_index, PPC_IPI_FLUSH_VEC);
 220
 221         /* Wait for flush. */
 222         for (i = 0; i < 0x3fffffff; i++)
 223                 if (pcb->pcb_veccpu == NULL)
 224                         return;
 225
 226         aprint_error("mp_save_vec_lwp{%d} pid = %d.%d, veccpu->ci_cpuid = %d\n",
 227             cpu_number(), l->l_proc->p_pid, l->l_lid, veccpu->ci_cpuid);
 228         panic("mp_save_vec_lwp: timed out");
 229 }
 230 #endif /*MULTIPROCESSOR*/
 231
 232 /*
 233  * Save a process's AltiVEC state to its PCB.  The state may be in any CPU.
 234  * The process must either be curproc or traced by curproc (and stopped).
 235  * (The point being that the process must not run on another CPU during
 236  * this function).
 237  */
 238 void
 239 save_vec_lwp(struct lwp *l, int discard)
 240 {
 241         struct pcb * const pcb = lwp_getpcb(l);
 242         struct cpu_info * const ci = curcpu();
 243
 244         /*
 245          * If it's already in the PCB, there's nothing to do.
 246          */
 247         if (pcb->pcb_veccpu == NULL)
 248                 return;
 249
 250         /*
 251          * If we simply need to discard the information, then don't
 252          * to save anything.
 253          */
 254         if (discard) {
 255 #ifndef MULTIPROCESSOR
 256                 KASSERT(ci == pcb->pcb_veccpu);
 257 #endif
 258                 KASSERT(l == pcb->pcb_veccpu->ci_veclwp);
 259                 pcb->pcb_veccpu->ci_veclwp = NULL;
 260                 pcb->pcb_veccpu = NULL;
 261                 pcb->pcb_flags &= ~PCB_OWNALTIVEC;
 262                 return;
 263         }
 264
 265         /*
 266          * If the state is in the current CPU, just flush the current CPU's
 267          * state.
 268          */
 269         if (l == ci->ci_veclwp) {
 270                 save_vec_cpu();
 271                 return;
 272         }
 273
 274
 275 #ifdef MULTIPROCESSOR
 276         /*
 277          * It must be on another CPU, flush it from there.
 278          */
 279
 280         mp_save_vec_lwp(l);
 281 #endif
 282 }
 283
 284 #define ZERO_VEC        19
 285
 286 void
 287 vzeropage(paddr_t pa)
 288 {
 289         const paddr_t ea = pa + PAGE_SIZE;
 290         uint32_t vec[7], *vp = (void *) roundup((uintptr_t) vec, 16);
 291         register_t omsr, msr;
 292
 293         __asm volatile("mfmsr %0" : "=r"(omsr) :);
 294
 295         /*
 296          * Turn on AltiVec, turn off interrupts.
 297          */
 298         msr = (omsr & ~PSL_EE) | PSL_VEC;
 299         __asm volatile("sync; mtmsr %0; isync" :: "r"(msr));
 300
 301         /*
 302          * Save the VEC register we are going to use before we disable
 303          * relocation.
 304          */
 305         __asm("stvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC));
 306         __asm("vxor %0,%0,%0" :: "n"(ZERO_VEC));
 307
 308         /*
 309          * Zero the page using a single cache line.
 310          */
 311         __asm volatile(
 312             "   sync ;"
 313             "   mfmsr  %[msr];"
 314             "   rlwinm %[msr],%[msr],0,28,26;"  /* Clear PSL_DR */
 315             "   mtmsr  %[msr];"                 /* Turn off DMMU */
 316             "   isync;"
 317             "1: stvx   %[zv], %[pa], %[off0];"
 318             "   stvxl  %[zv], %[pa], %[off16];"
 319             "   stvx   %[zv], %[pa], %[off32];"
 320             "   stvxl  %[zv], %[pa], %[off48];"
 321             "   addi   %[pa], %[pa], 64;"
 322             "   cmplw  %[pa], %[ea];"
 323             "   blt+   1b;"
 324             "   ori    %[msr], %[msr], 0x10;"   /* Set PSL_DR */
 325             "   sync;"
 326             "   mtmsr  %[msr];"                 /* Turn on DMMU */
 327             "   isync;"
 328             :: [msr] "r"(msr), [pa] "b"(pa), [ea] "b"(ea),
 329             [off0] "r"(0), [off16] "r"(16), [off32] "r"(32), [off48] "r"(48),
 330             [zv] "n"(ZERO_VEC));
 331
 332         /*
 333          * Restore VEC register (now that we can access the stack again).
 334          */
 335         __asm("lvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC));
 336
 337         /*
 338          * Restore old MSR (AltiVec OFF).
 339          */
 340         __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr));
 341 }
 342
 343 #define LO_VEC  16
 344 #define HI_VEC  17
 345
 346 void
 347 vcopypage(paddr_t dst, paddr_t src)
 348 {
 349         const paddr_t edst = dst + PAGE_SIZE;
 350         uint32_t vec[11], *vp = (void *) roundup((uintptr_t) vec, 16);
 351         register_t omsr, msr;
 352
 353         __asm volatile("mfmsr %0" : "=r"(omsr) :);
 354
 355         /*
 356          * Turn on AltiVec, turn off interrupts.
 357          */
 358         msr = (omsr & ~PSL_EE) | PSL_VEC;
 359         __asm volatile("sync; mtmsr %0; isync" :: "r"(msr));
 360
 361         /*
 362          * Save the VEC registers we will be using before we disable
 363          * relocation.
 364          */
 365         __asm("stvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC));
 366         __asm("stvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC));
 367
 368         /*
 369          * Copy the page using a single cache line, with DMMU
 370          * disabled.  On most PPCs, two vector registers occupy one
 371          * cache line.
 372          */
 373         __asm volatile(
 374             "   sync ;"
 375             "   mfmsr  %[msr];"
 376             "   rlwinm %[msr],%[msr],0,28,26;"  /* Clear PSL_DR */
 377             "   mtmsr  %[msr];"                 /* Turn off DMMU */
 378             "   isync;"
 379             "1: lvx    %[lv], %[src], %[off0];"
 380             "   stvx   %[lv], %[dst], %[off0];"
 381             "   lvxl   %[hv], %[src], %[off16];"
 382             "   stvxl  %[hv], %[dst], %[off16];"
 383             "   addi   %[src], %[src], 32;"
 384             "   addi   %[dst], %[dst], 32;"
 385             "   cmplw  %[dst], %[edst];"
 386             "   blt+   1b;"
 387             "   ori    %[msr], %[msr], 0x10;"   /* Set PSL_DR */
 388             "   sync;"
 389             "   mtmsr  %[msr];"                 /* Turn on DMMU */
 390             "   isync;"
 391             :: [msr] "r"(msr), [src] "b"(src), [dst] "b"(dst),
 392             [edst] "b"(edst), [off0] "r"(0), [off16] "r"(16),
 393             [lv] "n"(LO_VEC), [hv] "n"(HI_VEC));
 394
 395         /*
 396          * Restore VEC registers (now that we can access the stack again).
 397          */
 398         __asm("lvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC));
 399         __asm("lvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC));
 400
 401         /*
 402          * Restore old MSR (AltiVec OFF).
 403          */
 404         __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr));
 405 }