usr/src/uts/sun4u/ml/mach_copy.s

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License, Version 1.0 only
   6  * (the "License").  You may not use this file except in compliance
   7  * with the License.
   8  *
   9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10  * or http://www.opensolaris.org/os/licensing.
  11  * See the License for the specific language governing permissions
  12  * and limitations under the License.
  13  *
  14  * When distributing Covered Code, include this CDDL HEADER in each
  15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16  * If applicable, add the following below this CDDL HEADER, with the
  17  * fields enclosed by brackets "[]" replaced with your own identifying
  18  * information: Portions Copyright [yyyy] [name of copyright owner]
  19  *
  20  * CDDL HEADER END
  21  */
  22 /*
  23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26
  27 #pragma ident   "%Z%%M% %I%     %E% SMI"
  28
  29 #include <sys/param.h>
  30 #include <sys/errno.h>
  31 #include <sys/asm_linkage.h>
  32 #include <sys/vtrace.h>
  33 #include <sys/machthread.h>
  34 #include <sys/clock.h>
  35 #include <sys/asi.h>
  36 #include <sys/fsr.h>
  37 #include <sys/privregs.h>
  38
  39 #if !defined(lint)
  40 #include "assym.h"
  41 #endif  /* lint */
  42
  43 #define FP_USED 1
  44 #define LOFAULT_SET 2
  45
  46 /*
  47  * Error barrier:
  48  * We use membar sync to establish an error barrier for
  49  * deferred errors. Membar syncs are added before any update
  50  * to t_lofault to ensure that deferred errors from earlier
  51  * accesses will not be reported after the membar. This error
  52  * isolation is important when we try to recover from async
  53  * errors which tries to distinguish kernel accesses to user
  54  * data.
  55  */
  56
  57 /*
  58  * Zero a block of storage.
  59  *
  60  * uzero is used by the kernel to zero a block in user address space.
  61  */
  62
  63 #if defined(lint)
  64
  65 /* ARGSUSED */
  66 int
  67 kzero(void *addr, size_t count)
  68 { return(0); }
  69
  70 /* ARGSUSED */
  71 void
  72 uzero(void *addr, size_t count)
  73 {}
  74
  75 #else   /* lint */
  76
  77         ENTRY(uzero)
  78         !
  79         ! Set a new lo_fault handler only if we came in with one
  80         ! already specified.
  81         !
  82         wr      %g0, ASI_USER, %asi
  83         ldn     [THREAD_REG + T_LOFAULT], %o5
  84         tst     %o5
  85         bz,pt   %ncc, .do_zero
  86         sethi   %hi(.zeroerr), %o2
  87         or      %o2, %lo(.zeroerr), %o2
  88         membar  #Sync
  89         ba,pt   %ncc, .do_zero
  90         stn     %o2, [THREAD_REG + T_LOFAULT]
  91
  92         ENTRY(kzero)
  93         !
  94         ! Always set a lo_fault handler
  95         !
  96         wr      %g0, ASI_P, %asi
  97         ldn     [THREAD_REG + T_LOFAULT], %o5
  98         sethi   %hi(.zeroerr), %o2
  99         or      %o5, LOFAULT_SET, %o5
 100         or      %o2, %lo(.zeroerr), %o2
 101         membar  #Sync
 102         ba,pt   %ncc, .do_zero
 103         stn     %o2, [THREAD_REG + T_LOFAULT]
 104
 105 /*
 106  * We got here because of a fault during kzero or if
 107  * uzero or bzero was called with t_lofault non-zero.
 108  * Otherwise we've already run screaming from the room.
 109  * Errno value is in %g1. Note that we're here iff
 110  * we did set t_lofault.
 111  */
 112 .zeroerr:
 113         !
 114         ! Undo asi register setting. Just set it to be the
 115         ! kernel default without checking.
 116         !
 117         wr      %g0, ASI_P, %asi
 118         !
 119         ! If saved t_lofault has FP_USED set, clear the %fprs register
 120         !
 121         btst    FP_USED, %o5
 122         bz,pt   %ncc, 1f                ! skip if not used
 123         nop
 124         membar #Sync
 125         wr      %g0, %g0, %fprs         ! clear fprs
 126         andn    %o5, FP_USED, %o5       ! turn off flag bit
 127         !
 128         ! We did set t_lofault. It may well have been zero coming in.
 129         !
 130 1:
 131         tst     %o5
 132         membar #Sync
 133         bne,pn  %ncc, 3f
 134         andncc  %o5, LOFAULT_SET, %o5
 135 2:
 136         !
 137         ! Old handler was zero. Just return the error.
 138         !
 139         retl                            ! return
 140         mov     %g1, %o0                ! error code from %g1
 141 3:
 142         !
 143         ! We're here because %o5 was non-zero. It was non-zero
 144         ! because either LOFAULT_SET was present, a previous fault
 145         ! handler was present or both. In all cases we need to reset
 146         ! T_LOFAULT to the value of %o5 after clearing LOFAULT_SET
 147         ! before we either simply return the error or we invoke the
 148         ! previously specified handler.
 149         !
 150         be      %ncc, 2b
 151         stn     %o5, [THREAD_REG + T_LOFAULT]
 152         jmp     %o5                     ! goto real handler
 153           nop
 154         SET_SIZE(kzero)
 155         SET_SIZE(uzero)
 156
 157 #endif  /* lint */
 158
 159 /*
 160  * Zero a block of storage.
 161  */
 162
 163 #if defined(lint)
 164
 165 /* ARGSUSED */
 166 void
 167 bzero(void *addr, size_t count)
 168 {}
 169
 170 #else   /* lint */
 171
 172         ENTRY(bzero)
 173         wr      %g0, ASI_P, %asi
 174
 175         ldn     [THREAD_REG + T_LOFAULT], %o5   ! save old vector
 176         tst     %o5
 177         bz,pt   %ncc, .do_zero
 178         sethi   %hi(.zeroerr), %o2
 179         or      %o2, %lo(.zeroerr), %o2
 180         membar  #Sync                           ! sync error barrier
 181         stn     %o2, [THREAD_REG + T_LOFAULT]   ! install new vector
 182
 183 .do_zero:
 184         cmp     %o1, 15                 ! check for small counts
 185         blu,pn  %ncc, .byteclr          ! just clear bytes
 186         nop
 187
 188         cmp     %o1, 192                ! check for large counts
 189         blu     %ncc, .bzero_small
 190         nop
 191
 192         sethi   %hi(use_hw_bzero), %o2
 193         ld      [%o2 + %lo(use_hw_bzero)], %o2
 194         tst     %o2
 195         bz      %icc, .bzero_small
 196         nop
 197
 198         rd      %fprs, %o2              ! check for unused fp
 199         btst    FPRS_FEF, %o2
 200         bnz     %icc, .bzero_small
 201         nop
 202
 203         ldn     [THREAD_REG + T_LWP], %o2
 204         tst     %o2
 205         bz,pn   %ncc, .bzero_small
 206         nop
 207
 208         ! Check for block alignment
 209         btst    (64-1), %o0
 210         bz      %icc, .bzl_block
 211         nop
 212
 213         ! Check for double-word alignment
 214         btst    (8-1), %o0
 215         bz      %icc, .bzl_dword
 216         nop
 217
 218         ! Check for word alignment
 219         btst    (4-1), %o0
 220         bz      %icc, .bzl_word
 221         nop
 222
 223         ! Clear bytes until word aligned
 224 .bzl_byte:
 225         stba    %g0, [%o0]%asi
 226         add     %o0, 1, %o0
 227         btst    (4-1), %o0
 228         bnz     %icc, .bzl_byte
 229         sub     %o1, 1, %o1
 230
 231         ! Check for dword-aligned
 232         btst    (8-1), %o0
 233         bz      %icc, .bzl_dword
 234         nop
 235
 236         ! Clear words until double-word aligned
 237 .bzl_word:
 238         sta     %g0, [%o0]%asi
 239         add     %o0, 4, %o0
 240         btst    (8-1), %o0
 241         bnz     %icc, .bzl_word
 242         sub     %o1, 4, %o1
 243
 244 .bzl_dword:
 245         ! Clear dwords until block aligned
 246         stxa    %g0, [%o0]%asi
 247         add     %o0, 8, %o0
 248         btst    (64-1), %o0
 249         bnz     %icc, .bzl_dword
 250         sub     %o1, 8, %o1
 251
 252 .bzl_block:
 253         membar  #StoreStore|#StoreLoad|#LoadStore
 254         wr      %g0, FPRS_FEF, %fprs
 255
 256         ! Set the lower bit in the saved t_lofault to indicate
 257         ! that we need to clear the %fprs register on the way
 258         ! out
 259         or      %o5, FP_USED, %o5
 260
 261         ! Clear block
 262         fzero   %d0
 263         fzero   %d2
 264         fzero   %d4
 265         fzero   %d6
 266         fzero   %d8
 267         fzero   %d10
 268         fzero   %d12
 269         fzero   %d14
 270         rd      %asi, %o3
 271         wr      %g0, ASI_BLK_P, %asi
 272         cmp     %o3, ASI_P
 273         bne,a   %icc, 1f
 274         wr      %g0, ASI_BLK_AIUS, %asi
 275 1:
 276         mov     256, %o3
 277         ba,pt   %ncc, .bzl_doblock
 278         nop
 279
 280 .bzl_blkstart:
 281       ! stda    %d0, [%o0+192]%asi  ! in dly slot of branch that got us here
 282         stda    %d0, [%o0+128]%asi
 283         stda    %d0, [%o0+64]%asi
 284         stda    %d0, [%o0]%asi
 285 .bzl_zinst:
 286         add     %o0, %o3, %o0
 287         sub     %o1, %o3, %o1
 288 .bzl_doblock:
 289         cmp     %o1, 256
 290         bgeu,a  %ncc, .bzl_blkstart
 291         stda    %d0, [%o0+192]%asi
 292
 293         cmp     %o1, 64
 294         blu     %ncc, .bzl_finish
 295
 296         andn    %o1, (64-1), %o3
 297         srl     %o3, 4, %o2             ! using blocks, 1 instr / 16 words
 298         set     .bzl_zinst, %o4
 299         sub     %o4, %o2, %o4
 300         jmp     %o4
 301         nop
 302
 303 .bzl_finish:
 304         membar  #StoreLoad|#StoreStore
 305         wr      %g0, %g0, %fprs
 306         andn    %o5, FP_USED, %o5
 307
 308         rd      %asi, %o4
 309         wr      %g0, ASI_P, %asi
 310         cmp     %o4, ASI_BLK_P
 311         bne,a   %icc, 1f
 312         wr      %g0, ASI_USER, %asi
 313 1:
 314
 315 .bzlf_dword:
 316         ! double words
 317         cmp     %o1, 8
 318         blu     %ncc, .bzlf_word
 319         nop
 320         stxa    %g0, [%o0]%asi
 321         add     %o0, 8, %o0
 322         sub     %o1, 8, %o1
 323         ba,pt   %ncc, .bzlf_dword
 324         nop
 325
 326 .bzlf_word:
 327         ! words
 328         cmp     %o1, 4
 329         blu     %ncc, .bzlf_byte
 330         nop
 331         sta     %g0, [%o0]%asi
 332         add     %o0, 4, %o0
 333         sub     %o1, 4, %o1
 334         ba,pt   %ncc, .bzlf_word
 335         nop
 336
 337 1:
 338         add     %o0, 1, %o0             ! increment address
 339 .bzlf_byte:
 340         subcc   %o1, 1, %o1             ! decrement count
 341         bgeu,a  %ncc, 1b
 342         stba    %g0, [%o0]%asi          ! zero a byte
 343
 344         !
 345         ! If we used the FP registers, that bit was turned
 346         ! off after we were finished. We're just concerned with
 347         ! whether t_lofault was set when we came in. We end up
 348         ! here from either kzero() or bzero(). kzero() *always*
 349         ! sets a lofault handler. It ors LOFAULT_SET into %o5
 350         ! to indicate it has done this even if the value of %o5
 351         ! is otherwise zero. bzero() sets a lofault handler *only*
 352         ! if one was previously set. Accordingly we need to examine
 353         ! %o5 and if it is non-zero be sure to clear LOFAULT_SET
 354         ! before resetting the error handler.
 355         !
 356         tst     %o5
 357         bz,pt   %ncc, 1f
 358         andn    %o5, LOFAULT_SET, %o5
 359         membar  #Sync                           ! sync error barrier
 360         stn     %o5, [THREAD_REG + T_LOFAULT]   ! restore old t_lofault
 361 1:
 362         retl
 363         clr     %o0                     ! return (0)
 364
 365 .bzero_small:
 366
 367         !
 368         ! Check for word alignment.
 369         !
 370         btst    3, %o0
 371         bz      .bzero_probe
 372         mov     0x100, %o3              ! constant size of main loop
 373         !
 374         !
 375         ! clear bytes until word aligned
 376         !
 377 1:      stba    %g0,[%o0]%asi
 378         add     %o0, 1, %o0
 379         btst    3, %o0
 380         bnz     1b
 381         sub     %o1, 1, %o1
 382 .bzero_probe:
 383
 384         !
 385         ! if needed move a word to become double-word aligned.
 386         !
 387         btst    7, %o0                  ! is double aligned?
 388         bz      %icc, .bzero_nobuf
 389         nop
 390         sta     %g0, [%o0]%asi          ! clr to double boundry
 391         sub     %o1, 4, %o1
 392         ba,pt   %ncc, .bzero_nobuf
 393         add     %o0, 4, %o0
 394
 395         !stxa   %g0, [%o0+0xf8]%asi
 396 .bzero_blk:
 397         stxa    %g0, [%o0+0xf0]%asi
 398         stxa    %g0, [%o0+0xe8]%asi
 399         stxa    %g0, [%o0+0xe0]%asi
 400         stxa    %g0, [%o0+0xd8]%asi
 401         stxa    %g0, [%o0+0xd0]%asi
 402         stxa    %g0, [%o0+0xc8]%asi
 403         stxa    %g0, [%o0+0xc0]%asi
 404         stxa    %g0, [%o0+0xb8]%asi
 405         stxa    %g0, [%o0+0xb0]%asi
 406         stxa    %g0, [%o0+0xa8]%asi
 407         stxa    %g0, [%o0+0xa0]%asi
 408         stxa    %g0, [%o0+0x98]%asi
 409         stxa    %g0, [%o0+0x90]%asi
 410         stxa    %g0, [%o0+0x88]%asi
 411         stxa    %g0, [%o0+0x80]%asi
 412         stxa    %g0, [%o0+0x78]%asi
 413         stxa    %g0, [%o0+0x70]%asi
 414         stxa    %g0, [%o0+0x68]%asi
 415         stxa    %g0, [%o0+0x60]%asi
 416         stxa    %g0, [%o0+0x58]%asi
 417         stxa    %g0, [%o0+0x50]%asi
 418         stxa    %g0, [%o0+0x48]%asi
 419         stxa    %g0, [%o0+0x40]%asi
 420         stxa    %g0, [%o0+0x38]%asi
 421         stxa    %g0, [%o0+0x30]%asi
 422         stxa    %g0, [%o0+0x28]%asi
 423         stxa    %g0, [%o0+0x20]%asi
 424         stxa    %g0, [%o0+0x18]%asi
 425         stxa    %g0, [%o0+0x10]%asi
 426         stxa    %g0, [%o0+0x08]%asi
 427         stxa    %g0, [%o0]%asi
 428 .zinst:
 429         add     %o0, %o3, %o0           ! increment source address
 430         sub     %o1, %o3, %o1           ! decrement count
 431 .bzero_nobuf:
 432         cmp     %o1, 0x100              ! can we do whole chunk?
 433         bgeu,a  %ncc, .bzero_blk
 434         stxa    %g0, [%o0+0xf8]%asi     ! do first double of chunk
 435
 436         cmp     %o1, 7                  ! can we zero any more double words
 437         bleu    %ncc, .byteclr          ! too small go zero bytes
 438
 439         andn    %o1, 7, %o3             ! %o3 bytes left, double-word aligned
 440         srl     %o3, 1, %o2             ! using doubles, need 1 instr / 2 words
 441         set     .zinst, %o4             ! address of clr instructions
 442         sub     %o4, %o2, %o4           ! jmp address relative to instr
 443         jmp     %o4
 444         nop
 445         !
 446         ! do leftover bytes
 447         !
 448 3:
 449         add     %o0, 1, %o0             ! increment address
 450 .byteclr:
 451         subcc   %o1, 1, %o1             ! decrement count
 452         bgeu,a  %ncc, 3b
 453         stba    %g0, [%o0]%asi          ! zero a byte
 454
 455 .bzero_finished:
 456         !
 457         ! We're just concerned with whether t_lofault was set
 458         ! when we came in. We end up here from either kzero()
 459         ! or bzero(). kzero() *always* sets a lofault handler.
 460         ! It ors LOFAULT_SET into %o5 to indicate it has done
 461         ! this even if the value of %o5 is otherwise zero.
 462         ! bzero() sets a lofault handler *only* if one was
 463         ! previously set. Accordingly we need to examine
 464         ! %o5 and if it is non-zero be sure to clear LOFAULT_SET
 465         ! before resetting the error handler.
 466         !
 467         tst     %o5
 468         bz      %ncc, 1f
 469         andn    %o5, LOFAULT_SET, %o5
 470         membar  #Sync                           ! sync error barrier
 471         stn     %o5, [THREAD_REG + T_LOFAULT]   ! restore old t_lofault
 472 1:
 473         retl
 474         clr     %o0                     ! return (0)
 475
 476         SET_SIZE(bzero)
 477 #endif  /* lint */