arch/arm64/include/asm/fpsimdmacros.h

   1 /*
   2  * FP/SIMD state saving and restoring macros
   3  *
   4  * Copyright (C) 2012 ARM Ltd.
   5  * Author: Catalin Marinas <catalin.marinas@arm.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 .macro fpsimd_save state, tmpnr
  21         stp     q0, q1, [\state, #16 * 0]
  22         stp     q2, q3, [\state, #16 * 2]
  23         stp     q4, q5, [\state, #16 * 4]
  24         stp     q6, q7, [\state, #16 * 6]
  25         stp     q8, q9, [\state, #16 * 8]
  26         stp     q10, q11, [\state, #16 * 10]
  27         stp     q12, q13, [\state, #16 * 12]
  28         stp     q14, q15, [\state, #16 * 14]
  29         stp     q16, q17, [\state, #16 * 16]
  30         stp     q18, q19, [\state, #16 * 18]
  31         stp     q20, q21, [\state, #16 * 20]
  32         stp     q22, q23, [\state, #16 * 22]
  33         stp     q24, q25, [\state, #16 * 24]
  34         stp     q26, q27, [\state, #16 * 26]
  35         stp     q28, q29, [\state, #16 * 28]
  36         stp     q30, q31, [\state, #16 * 30]!
  37         mrs     x\tmpnr, fpsr
  38         str     w\tmpnr, [\state, #16 * 2]
  39         mrs     x\tmpnr, fpcr
  40         str     w\tmpnr, [\state, #16 * 2 + 4]
  41 .endm
  42
  43 .macro fpsimd_restore_fpcr state, tmp
  44         /*
  45          * Writes to fpcr may be self-synchronising, so avoid restoring
  46          * the register if it hasn't changed.
  47          */
  48         mrs     \tmp, fpcr
  49         cmp     \tmp, \state
  50         b.eq    9999f
  51         msr     fpcr, \state
  52 9999:
  53 .endm
  54
  55 /* Clobbers \state */
  56 .macro fpsimd_restore state, tmpnr
  57         ldp     q0, q1, [\state, #16 * 0]
  58         ldp     q2, q3, [\state, #16 * 2]
  59         ldp     q4, q5, [\state, #16 * 4]
  60         ldp     q6, q7, [\state, #16 * 6]
  61         ldp     q8, q9, [\state, #16 * 8]
  62         ldp     q10, q11, [\state, #16 * 10]
  63         ldp     q12, q13, [\state, #16 * 12]
  64         ldp     q14, q15, [\state, #16 * 14]
  65         ldp     q16, q17, [\state, #16 * 16]
  66         ldp     q18, q19, [\state, #16 * 18]
  67         ldp     q20, q21, [\state, #16 * 20]
  68         ldp     q22, q23, [\state, #16 * 22]
  69         ldp     q24, q25, [\state, #16 * 24]
  70         ldp     q26, q27, [\state, #16 * 26]
  71         ldp     q28, q29, [\state, #16 * 28]
  72         ldp     q30, q31, [\state, #16 * 30]!
  73         ldr     w\tmpnr, [\state, #16 * 2]
  74         msr     fpsr, x\tmpnr
  75         ldr     w\tmpnr, [\state, #16 * 2 + 4]
  76         fpsimd_restore_fpcr x\tmpnr, \state
  77 .endm
  78
  79 /* Sanity-check macros to help avoid encoding garbage instructions */
  80
  81 .macro _check_general_reg nr
  82         .if (\nr) < 0 || (\nr) > 30
  83                 .error "Bad register number \nr."
  84         .endif
  85 .endm
  86
  87 .macro _sve_check_zreg znr
  88         .if (\znr) < 0 || (\znr) > 31
  89                 .error "Bad Scalable Vector Extension vector register number \znr."
  90         .endif
  91 .endm
  92
  93 .macro _sve_check_preg pnr
  94         .if (\pnr) < 0 || (\pnr) > 15
  95                 .error "Bad Scalable Vector Extension predicate register number \pnr."
  96         .endif
  97 .endm
  98
  99 .macro _check_num n, min, max
 100         .if (\n) < (\min) || (\n) > (\max)
 101                 .error "Number \n out of range [\min,\max]"
 102         .endif
 103 .endm
 104
 105 /* SVE instruction encodings for non-SVE-capable assemblers */
 106
 107 /* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */
 108 .macro _sve_str_v nz, nxbase, offset=0
 109         _sve_check_zreg \nz
 110         _check_general_reg \nxbase
 111         _check_num (\offset), -0x100, 0xff
 112         .inst   0xe5804000                      \
 113                 | (\nz)                         \
 114                 | ((\nxbase) << 5)              \
 115                 | (((\offset) & 7) << 10)       \
 116                 | (((\offset) & 0x1f8) << 13)
 117 .endm
 118
 119 /* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */
 120 .macro _sve_ldr_v nz, nxbase, offset=0
 121         _sve_check_zreg \nz
 122         _check_general_reg \nxbase
 123         _check_num (\offset), -0x100, 0xff
 124         .inst   0x85804000                      \
 125                 | (\nz)                         \
 126                 | ((\nxbase) << 5)              \
 127                 | (((\offset) & 7) << 10)       \
 128                 | (((\offset) & 0x1f8) << 13)
 129 .endm
 130
 131 /* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */
 132 .macro _sve_str_p np, nxbase, offset=0
 133         _sve_check_preg \np
 134         _check_general_reg \nxbase
 135         _check_num (\offset), -0x100, 0xff
 136         .inst   0xe5800000                      \
 137                 | (\np)                         \
 138                 | ((\nxbase) << 5)              \
 139                 | (((\offset) & 7) << 10)       \
 140                 | (((\offset) & 0x1f8) << 13)
 141 .endm
 142
 143 /* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */
 144 .macro _sve_ldr_p np, nxbase, offset=0
 145         _sve_check_preg \np
 146         _check_general_reg \nxbase
 147         _check_num (\offset), -0x100, 0xff
 148         .inst   0x85800000                      \
 149                 | (\np)                         \
 150                 | ((\nxbase) << 5)              \
 151                 | (((\offset) & 7) << 10)       \
 152                 | (((\offset) & 0x1f8) << 13)
 153 .endm
 154
 155 /* RDVL X\nx, #\imm */
 156 .macro _sve_rdvl nx, imm
 157         _check_general_reg \nx
 158         _check_num (\imm), -0x20, 0x1f
 159         .inst   0x04bf5000                      \
 160                 | (\nx)                         \
 161                 | (((\imm) & 0x3f) << 5)
 162 .endm
 163
 164 /* RDFFR (unpredicated): RDFFR P\np.B */
 165 .macro _sve_rdffr np
 166         _sve_check_preg \np
 167         .inst   0x2519f000                      \
 168                 | (\np)
 169 .endm
 170
 171 /* WRFFR P\np.B */
 172 .macro _sve_wrffr np
 173         _sve_check_preg \np
 174         .inst   0x25289000                      \
 175                 | ((\np) << 5)
 176 .endm
 177
 178 .macro __for from:req, to:req
 179         .if (\from) == (\to)
 180                 _for__body \from
 181         .else
 182                 __for \from, (\from) + ((\to) - (\from)) / 2
 183                 __for (\from) + ((\to) - (\from)) / 2 + 1, \to
 184         .endif
 185 .endm
 186
 187 .macro _for var:req, from:req, to:req, insn:vararg
 188         .macro _for__body \var:req
 189                 \insn
 190         .endm
 191
 192         __for \from, \to
 193
 194         .purgem _for__body
 195 .endm
 196
 197 .macro sve_save nxbase, xpfpsr, nxtmp
 198  _for n, 0, 31, _sve_str_v      \n, \nxbase, \n - 34
 199  _for n, 0, 15, _sve_str_p      \n, \nxbase, \n - 16
 200                 _sve_rdffr      0
 201                 _sve_str_p      0, \nxbase
 202                 _sve_ldr_p      0, \nxbase, -16
 203
 204                 mrs             x\nxtmp, fpsr
 205                 str             w\nxtmp, [\xpfpsr]
 206                 mrs             x\nxtmp, fpcr
 207                 str             w\nxtmp, [\xpfpsr, #4]
 208 .endm
 209
 210 .macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2
 211                 mrs_s           x\nxtmp, SYS_ZCR_EL1
 212                 bic             \xtmp2, x\nxtmp, ZCR_ELx_LEN_MASK
 213                 orr             \xtmp2, \xtmp2, \xvqminus1
 214                 cmp             \xtmp2, x\nxtmp
 215                 b.eq            921f
 216                 msr_s           SYS_ZCR_EL1, \xtmp2     // self-synchronising
 217 921:
 218  _for n, 0, 31, _sve_ldr_v      \n, \nxbase, \n - 34
 219                 _sve_ldr_p      0, \nxbase
 220                 _sve_wrffr      0
 221  _for n, 0, 15, _sve_ldr_p      \n, \nxbase, \n - 16
 222
 223                 ldr             w\nxtmp, [\xpfpsr]
 224                 msr             fpsr, x\nxtmp
 225                 ldr             w\nxtmp, [\xpfpsr, #4]
 226                 msr             fpcr, x\nxtmp
 227 .endm