arch/arm64/include/asm/assembler.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S
   4  *
   5  * Copyright (C) 1996-2000 Russell King
   6  * Copyright (C) 2012 ARM Ltd.
   7  */
   8 #ifndef __ASSEMBLY__
   9 #error "Only include this from assembly code"
  10 #endif
  11
  12 #ifndef __ASM_ASSEMBLER_H
  13 #define __ASM_ASSEMBLER_H
  14
  15 #include <asm-generic/export.h>
  16
  17 #include <asm/asm-offsets.h>
  18 #include <asm/cpufeature.h>
  19 #include <asm/cputype.h>
  20 #include <asm/debug-monitors.h>
  21 #include <asm/page.h>
  22 #include <asm/pgtable-hwdef.h>
  23 #include <asm/ptrace.h>
  24 #include <asm/thread_info.h>
  25
  26         .macro save_and_disable_daif, flags
  27         mrs     \flags, daif
  28         msr     daifset, #0xf
  29         .endm
  30
  31         .macro disable_daif
  32         msr     daifset, #0xf
  33         .endm
  34
  35         .macro enable_daif
  36         msr     daifclr, #0xf
  37         .endm
  38
  39         .macro  restore_daif, flags:req
  40         msr     daif, \flags
  41         .endm
  42
  43         /* Only on aarch64 pstate, PSR_D_BIT is different for aarch32 */
  44         .macro  inherit_daif, pstate:req, tmp:req
  45         and     \tmp, \pstate, #(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
  46         msr     daif, \tmp
  47         .endm
  48
  49         /* IRQ is the lowest priority flag, unconditionally unmask the rest. */
  50         .macro enable_da_f
  51         msr     daifclr, #(8 | 4 | 1)
  52         .endm
  53
  54 /*
  55  * Save/restore interrupts.
  56  */
  57         .macro  save_and_disable_irq, flags
  58         mrs     \flags, daif
  59         msr     daifset, #2
  60         .endm
  61
  62         .macro  restore_irq, flags
  63         msr     daif, \flags
  64         .endm
  65
  66         .macro  enable_dbg
  67         msr     daifclr, #8
  68         .endm
  69
  70         .macro  disable_step_tsk, flgs, tmp
  71         tbz     \flgs, #TIF_SINGLESTEP, 9990f
  72         mrs     \tmp, mdscr_el1
  73         bic     \tmp, \tmp, #DBG_MDSCR_SS
  74         msr     mdscr_el1, \tmp
  75         isb     // Synchronise with enable_dbg
  76 9990:
  77         .endm
  78
  79         /* call with daif masked */
  80         .macro  enable_step_tsk, flgs, tmp
  81         tbz     \flgs, #TIF_SINGLESTEP, 9990f
  82         mrs     \tmp, mdscr_el1
  83         orr     \tmp, \tmp, #DBG_MDSCR_SS
  84         msr     mdscr_el1, \tmp
  85 9990:
  86         .endm
  87
  88 /*
  89  * SMP data memory barrier
  90  */
  91         .macro  smp_dmb, opt
  92         dmb     \opt
  93         .endm
  94
  95 /*
  96  * RAS Error Synchronization barrier
  97  */
  98         .macro  esb
  99 #ifdef CONFIG_ARM64_RAS_EXTN
 100         hint    #16
 101 #else
 102         nop
 103 #endif
 104         .endm
 105
 106 /*
 107  * Value prediction barrier
 108  */
 109         .macro  csdb
 110         hint    #20
 111         .endm
 112
 113 /*
 114  * Speculation barrier
 115  */
 116         .macro  sb
 117 alternative_if_not ARM64_HAS_SB
 118         dsb     nsh
 119         isb
 120 alternative_else
 121         SB_BARRIER_INSN
 122         nop
 123 alternative_endif
 124         .endm
 125
 126 /*
 127  * Sanitise a 64-bit bounded index wrt speculation, returning zero if out
 128  * of bounds.
 129  */
 130         .macro  mask_nospec64, idx, limit, tmp
 131         sub     \tmp, \idx, \limit
 132         bic     \tmp, \tmp, \idx
 133         and     \idx, \idx, \tmp, asr #63
 134         csdb
 135         .endm
 136
 137 /*
 138  * NOP sequence
 139  */
 140         .macro  nops, num
 141         .rept   \num
 142         nop
 143         .endr
 144         .endm
 145
 146 /*
 147  * Emit an entry into the exception table
 148  */
 149         .macro          _asm_extable, from, to
 150         .pushsection    __ex_table, "a"
 151         .align          3
 152         .long           (\from - .), (\to - .)
 153         .popsection
 154         .endm
 155
 156 #define USER(l, x...)                           \
 157 9999:   x;                                      \
 158         _asm_extable    9999b, l
 159
 160 /*
 161  * Register aliases.
 162  */
 163 lr      .req    x30             // link register
 164
 165 /*
 166  * Vector entry
 167  */
 168          .macro ventry  label
 169         .align  7
 170         b       \label
 171         .endm
 172
 173 /*
 174  * Select code when configured for BE.
 175  */
 176 #ifdef CONFIG_CPU_BIG_ENDIAN
 177 #define CPU_BE(code...) code
 178 #else
 179 #define CPU_BE(code...)
 180 #endif
 181
 182 /*
 183  * Select code when configured for LE.
 184  */
 185 #ifdef CONFIG_CPU_BIG_ENDIAN
 186 #define CPU_LE(code...)
 187 #else
 188 #define CPU_LE(code...) code
 189 #endif
 190
 191 /*
 192  * Define a macro that constructs a 64-bit value by concatenating two
 193  * 32-bit registers. Note that on big endian systems the order of the
 194  * registers is swapped.
 195  */
 196 #ifndef CONFIG_CPU_BIG_ENDIAN
 197         .macro  regs_to_64, rd, lbits, hbits
 198 #else
 199         .macro  regs_to_64, rd, hbits, lbits
 200 #endif
 201         orr     \rd, \lbits, \hbits, lsl #32
 202         .endm
 203
 204 /*
 205  * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where
 206  * <symbol> is within the range +/- 4 GB of the PC.
 207  */
 208         /*
 209          * @dst: destination register (64 bit wide)
 210          * @sym: name of the symbol
 211          */
 212         .macro  adr_l, dst, sym
 213         adrp    \dst, \sym
 214         add     \dst, \dst, :lo12:\sym
 215         .endm
 216
 217         /*
 218          * @dst: destination register (32 or 64 bit wide)
 219          * @sym: name of the symbol
 220          * @tmp: optional 64-bit scratch register to be used if <dst> is a
 221          *       32-bit wide register, in which case it cannot be used to hold
 222          *       the address
 223          */
 224         .macro  ldr_l, dst, sym, tmp=
 225         .ifb    \tmp
 226         adrp    \dst, \sym
 227         ldr     \dst, [\dst, :lo12:\sym]
 228         .else
 229         adrp    \tmp, \sym
 230         ldr     \dst, [\tmp, :lo12:\sym]
 231         .endif
 232         .endm
 233
 234         /*
 235          * @src: source register (32 or 64 bit wide)
 236          * @sym: name of the symbol
 237          * @tmp: mandatory 64-bit scratch register to calculate the address
 238          *       while <src> needs to be preserved.
 239          */
 240         .macro  str_l, src, sym, tmp
 241         adrp    \tmp, \sym
 242         str     \src, [\tmp, :lo12:\sym]
 243         .endm
 244
 245         /*
 246          * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP)
 247          * @sym: The name of the per-cpu variable
 248          * @tmp: scratch register
 249          */
 250         .macro adr_this_cpu, dst, sym, tmp
 251         adrp    \tmp, \sym
 252         add     \dst, \tmp, #:lo12:\sym
 253 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
 254         mrs     \tmp, tpidr_el1
 255 alternative_else
 256         mrs     \tmp, tpidr_el2
 257 alternative_endif
 258         add     \dst, \dst, \tmp
 259         .endm
 260
 261         /*
 262          * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id()))
 263          * @sym: The name of the per-cpu variable
 264          * @tmp: scratch register
 265          */
 266         .macro ldr_this_cpu dst, sym, tmp
 267         adr_l   \dst, \sym
 268 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
 269         mrs     \tmp, tpidr_el1
 270 alternative_else
 271         mrs     \tmp, tpidr_el2
 272 alternative_endif
 273         ldr     \dst, [\dst, \tmp]
 274         .endm
 275
 276 /*
 277  * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm)
 278  */
 279         .macro  vma_vm_mm, rd, rn
 280         ldr     \rd, [\rn, #VMA_VM_MM]
 281         .endm
 282
 283 /*
 284  * mmid - get context id from mm pointer (mm->context.id)
 285  */
 286         .macro  mmid, rd, rn
 287         ldr     \rd, [\rn, #MM_CONTEXT_ID]
 288         .endm
 289 /*
 290  * read_ctr - read CTR_EL0. If the system has mismatched register fields,
 291  * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val
 292  */
 293         .macro  read_ctr, reg
 294 alternative_if_not ARM64_MISMATCHED_CACHE_TYPE
 295         mrs     \reg, ctr_el0                   // read CTR
 296         nop
 297 alternative_else
 298         ldr_l   \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL
 299 alternative_endif
 300         .endm
 301
 302
 303 /*
 304  * raw_dcache_line_size - get the minimum D-cache line size on this CPU
 305  * from the CTR register.
 306  */
 307         .macro  raw_dcache_line_size, reg, tmp
 308         mrs     \tmp, ctr_el0                   // read CTR
 309         ubfm    \tmp, \tmp, #16, #19            // cache line size encoding
 310         mov     \reg, #4                        // bytes per word
 311         lsl     \reg, \reg, \tmp                // actual cache line size
 312         .endm
 313
 314 /*
 315  * dcache_line_size - get the safe D-cache line size across all CPUs
 316  */
 317         .macro  dcache_line_size, reg, tmp
 318         read_ctr        \tmp
 319         ubfm            \tmp, \tmp, #16, #19    // cache line size encoding
 320         mov             \reg, #4                // bytes per word
 321         lsl             \reg, \reg, \tmp        // actual cache line size
 322         .endm
 323
 324 /*
 325  * raw_icache_line_size - get the minimum I-cache line size on this CPU
 326  * from the CTR register.
 327  */
 328         .macro  raw_icache_line_size, reg, tmp
 329         mrs     \tmp, ctr_el0                   // read CTR
 330         and     \tmp, \tmp, #0xf                // cache line size encoding
 331         mov     \reg, #4                        // bytes per word
 332         lsl     \reg, \reg, \tmp                // actual cache line size
 333         .endm
 334
 335 /*
 336  * icache_line_size - get the safe I-cache line size across all CPUs
 337  */
 338         .macro  icache_line_size, reg, tmp
 339         read_ctr        \tmp
 340         and             \tmp, \tmp, #0xf        // cache line size encoding
 341         mov             \reg, #4                // bytes per word
 342         lsl             \reg, \reg, \tmp        // actual cache line size
 343         .endm
 344
 345 /*
 346  * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map
 347  */
 348         .macro  tcr_set_t0sz, valreg, t0sz
 349         bfi     \valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH
 350         .endm
 351
 352 /*
 353  * tcr_compute_pa_size - set TCR.(I)PS to the highest supported
 354  * ID_AA64MMFR0_EL1.PARange value
 355  *
 356  *      tcr:            register with the TCR_ELx value to be updated
 357  *      pos:            IPS or PS bitfield position
 358  *      tmp{0,1}:       temporary registers
 359  */
 360         .macro  tcr_compute_pa_size, tcr, pos, tmp0, tmp1
 361         mrs     \tmp0, ID_AA64MMFR0_EL1
 362         // Narrow PARange to fit the PS field in TCR_ELx
 363         ubfx    \tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3
 364         mov     \tmp1, #ID_AA64MMFR0_PARANGE_MAX
 365         cmp     \tmp0, \tmp1
 366         csel    \tmp0, \tmp1, \tmp0, hi
 367         bfi     \tcr, \tmp0, \pos, #3
 368         .endm
 369
 370 /*
 371  * Macro to perform a data cache maintenance for the interval
 372  * [kaddr, kaddr + size)
 373  *
 374  *      op:             operation passed to dc instruction
 375  *      domain:         domain used in dsb instruciton
 376  *      kaddr:          starting virtual address of the region
 377  *      size:           size of the region
 378  *      Corrupts:       kaddr, size, tmp1, tmp2
 379  */
 380         .macro __dcache_op_workaround_clean_cache, op, kaddr
 381 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
 382         dc      \op, \kaddr
 383 alternative_else
 384         dc      civac, \kaddr
 385 alternative_endif
 386         .endm
 387
 388         .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
 389         dcache_line_size \tmp1, \tmp2
 390         add     \size, \kaddr, \size
 391         sub     \tmp2, \tmp1, #1
 392         bic     \kaddr, \kaddr, \tmp2
 393 9998:
 394         .ifc    \op, cvau
 395         __dcache_op_workaround_clean_cache \op, \kaddr
 396         .else
 397         .ifc    \op, cvac
 398         __dcache_op_workaround_clean_cache \op, \kaddr
 399         .else
 400         .ifc    \op, cvap
 401         sys     3, c7, c12, 1, \kaddr   // dc cvap
 402         .else
 403         .ifc    \op, cvadp
 404         sys     3, c7, c13, 1, \kaddr   // dc cvadp
 405         .else
 406         dc      \op, \kaddr
 407         .endif
 408         .endif
 409         .endif
 410         .endif
 411         add     \kaddr, \kaddr, \tmp1
 412         cmp     \kaddr, \size
 413         b.lo    9998b
 414         dsb     \domain
 415         .endm
 416
 417 /*
 418  * Macro to perform an instruction cache maintenance for the interval
 419  * [start, end)
 420  *
 421  *      start, end:     virtual addresses describing the region
 422  *      label:          A label to branch to on user fault.
 423  *      Corrupts:       tmp1, tmp2
 424  */
 425         .macro invalidate_icache_by_line start, end, tmp1, tmp2, label
 426         icache_line_size \tmp1, \tmp2
 427         sub     \tmp2, \tmp1, #1
 428         bic     \tmp2, \start, \tmp2
 429 9997:
 430 USER(\label, ic ivau, \tmp2)                    // invalidate I line PoU
 431         add     \tmp2, \tmp2, \tmp1
 432         cmp     \tmp2, \end
 433         b.lo    9997b
 434         dsb     ish
 435         isb
 436         .endm
 437
 438 /*
 439  * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present
 440  */
 441         .macro  reset_pmuserenr_el0, tmpreg
 442         mrs     \tmpreg, id_aa64dfr0_el1
 443         sbfx    \tmpreg, \tmpreg, #ID_AA64DFR0_PMUVER_SHIFT, #4
 444         cmp     \tmpreg, #1                     // Skip if no PMU present
 445         b.lt    9000f
 446         msr     pmuserenr_el0, xzr              // Disable PMU access from EL0
 447 9000:
 448         .endm
 449
 450 /*
 451  * copy_page - copy src to dest using temp registers t1-t8
 452  */
 453         .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req
 454 9998:   ldp     \t1, \t2, [\src]
 455         ldp     \t3, \t4, [\src, #16]
 456         ldp     \t5, \t6, [\src, #32]
 457         ldp     \t7, \t8, [\src, #48]
 458         add     \src, \src, #64
 459         stnp    \t1, \t2, [\dest]
 460         stnp    \t3, \t4, [\dest, #16]
 461         stnp    \t5, \t6, [\dest, #32]
 462         stnp    \t7, \t8, [\dest, #48]
 463         add     \dest, \dest, #64
 464         tst     \src, #(PAGE_SIZE - 1)
 465         b.ne    9998b
 466         .endm
 467
 468 /*
 469  * Annotate a function as position independent, i.e., safe to be called before
 470  * the kernel virtual mapping is activated.
 471  */
 472 #define ENDPIPROC(x)                    \
 473         .globl  __pi_##x;               \
 474         .type   __pi_##x, %function;    \
 475         .set    __pi_##x, x;            \
 476         .size   __pi_##x, . - x;        \
 477         ENDPROC(x)
 478
 479 /*
 480  * Annotate a function as being unsuitable for kprobes.
 481  */
 482 #ifdef CONFIG_KPROBES
 483 #define NOKPROBE(x)                             \
 484         .pushsection "_kprobe_blacklist", "aw"; \
 485         .quad   x;                              \
 486         .popsection;
 487 #else
 488 #define NOKPROBE(x)
 489 #endif
 490
 491 #ifdef CONFIG_KASAN
 492 #define EXPORT_SYMBOL_NOKASAN(name)
 493 #else
 494 #define EXPORT_SYMBOL_NOKASAN(name)     EXPORT_SYMBOL(name)
 495 #endif
 496
 497         /*
 498          * Emit a 64-bit absolute little endian symbol reference in a way that
 499          * ensures that it will be resolved at build time, even when building a
 500          * PIE binary. This requires cooperation from the linker script, which
 501          * must emit the lo32/hi32 halves individually.
 502          */
 503         .macro  le64sym, sym
 504         .long   \sym\()_lo32
 505         .long   \sym\()_hi32
 506         .endm
 507
 508         /*
 509          * mov_q - move an immediate constant into a 64-bit register using
 510          *         between 2 and 4 movz/movk instructions (depending on the
 511          *         magnitude and sign of the operand)
 512          */
 513         .macro  mov_q, reg, val
 514         .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff)
 515         movz    \reg, :abs_g1_s:\val
 516         .else
 517         .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff)
 518         movz    \reg, :abs_g2_s:\val
 519         .else
 520         movz    \reg, :abs_g3:\val
 521         movk    \reg, :abs_g2_nc:\val
 522         .endif
 523         movk    \reg, :abs_g1_nc:\val
 524         .endif
 525         movk    \reg, :abs_g0_nc:\val
 526         .endm
 527
 528 /*
 529  * Return the current task_struct.
 530  */
 531         .macro  get_current_task, rd
 532         mrs     \rd, sp_el0
 533         .endm
 534
 535 /*
 536  * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD.
 537  * orr is used as it can cover the immediate value (and is idempotent).
 538  * In future this may be nop'ed out when dealing with 52-bit kernel VAs.
 539  *      ttbr: Value of ttbr to set, modified.
 540  */
 541         .macro  offset_ttbr1, ttbr
 542 #ifdef CONFIG_ARM64_USER_VA_BITS_52
 543         orr     \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
 544 #endif
 545         .endm
 546
 547 /*
 548  * Perform the reverse of offset_ttbr1.
 549  * bic is used as it can cover the immediate value and, in future, won't need
 550  * to be nop'ed out when dealing with 52-bit kernel VAs.
 551  */
 552         .macro  restore_ttbr1, ttbr
 553 #ifdef CONFIG_ARM64_USER_VA_BITS_52
 554         bic     \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
 555 #endif
 556         .endm
 557
 558 /*
 559  * Arrange a physical address in a TTBR register, taking care of 52-bit
 560  * addresses.
 561  *
 562  *      phys:   physical address, preserved
 563  *      ttbr:   returns the TTBR value
 564  */
 565         .macro  phys_to_ttbr, ttbr, phys
 566 #ifdef CONFIG_ARM64_PA_BITS_52
 567         orr     \ttbr, \phys, \phys, lsr #46
 568         and     \ttbr, \ttbr, #TTBR_BADDR_MASK_52
 569 #else
 570         mov     \ttbr, \phys
 571 #endif
 572         .endm
 573
 574         .macro  phys_to_pte, pte, phys
 575 #ifdef CONFIG_ARM64_PA_BITS_52
 576         /*
 577          * We assume \phys is 64K aligned and this is guaranteed by only
 578          * supporting this configuration with 64K pages.
 579          */
 580         orr     \pte, \phys, \phys, lsr #36
 581         and     \pte, \pte, #PTE_ADDR_MASK
 582 #else
 583         mov     \pte, \phys
 584 #endif
 585         .endm
 586
 587         .macro  pte_to_phys, phys, pte
 588 #ifdef CONFIG_ARM64_PA_BITS_52
 589         ubfiz   \phys, \pte, #(48 - 16 - 12), #16
 590         bfxil   \phys, \pte, #16, #32
 591         lsl     \phys, \phys, #16
 592 #else
 593         and     \phys, \pte, #PTE_ADDR_MASK
 594 #endif
 595         .endm
 596
 597 /*
 598  * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU.
 599  */
 600         .macro  tcr_clear_errata_bits, tcr, tmp1, tmp2
 601 #ifdef CONFIG_FUJITSU_ERRATUM_010001
 602         mrs     \tmp1, midr_el1
 603
 604         mov_q   \tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK
 605         and     \tmp1, \tmp1, \tmp2
 606         mov_q   \tmp2, MIDR_FUJITSU_ERRATUM_010001
 607         cmp     \tmp1, \tmp2
 608         b.ne    10f
 609
 610         mov_q   \tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001
 611         bic     \tcr, \tcr, \tmp2
 612 10:
 613 #endif /* CONFIG_FUJITSU_ERRATUM_010001 */
 614         .endm
 615
 616 /**
 617  * Errata workaround prior to disable MMU. Insert an ISB immediately prior
 618  * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0.
 619  */
 620         .macro pre_disable_mmu_workaround
 621 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041
 622         isb
 623 #endif
 624         .endm
 625
 626         /*
 627          * frame_push - Push @regcount callee saved registers to the stack,
 628          *              starting at x19, as well as x29/x30, and set x29 to
 629          *              the new value of sp. Add @extra bytes of stack space
 630          *              for locals.
 631          */
 632         .macro          frame_push, regcount:req, extra
 633         __frame         st, \regcount, \extra
 634         .endm
 635
 636         /*
 637          * frame_pop  - Pop the callee saved registers from the stack that were
 638          *              pushed in the most recent call to frame_push, as well
 639          *              as x29/x30 and any extra stack space that may have been
 640          *              allocated.
 641          */
 642         .macro          frame_pop
 643         __frame         ld
 644         .endm
 645
 646         .macro          __frame_regs, reg1, reg2, op, num
 647         .if             .Lframe_regcount == \num
 648         \op\()r         \reg1, [sp, #(\num + 1) * 8]
 649         .elseif         .Lframe_regcount > \num
 650         \op\()p         \reg1, \reg2, [sp, #(\num + 1) * 8]
 651         .endif
 652         .endm
 653
 654         .macro          __frame, op, regcount, extra=0
 655         .ifc            \op, st
 656         .if             (\regcount) < 0 || (\regcount) > 10
 657         .error          "regcount should be in the range [0 ... 10]"
 658         .endif
 659         .if             ((\extra) % 16) != 0
 660         .error          "extra should be a multiple of 16 bytes"
 661         .endif
 662         .ifdef          .Lframe_regcount
 663         .if             .Lframe_regcount != -1
 664         .error          "frame_push/frame_pop may not be nested"
 665         .endif
 666         .endif
 667         .set            .Lframe_regcount, \regcount
 668         .set            .Lframe_extra, \extra
 669         .set            .Lframe_local_offset, ((\regcount + 3) / 2) * 16
 670         stp             x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]!
 671         mov             x29, sp
 672         .endif
 673
 674         __frame_regs    x19, x20, \op, 1
 675         __frame_regs    x21, x22, \op, 3
 676         __frame_regs    x23, x24, \op, 5
 677         __frame_regs    x25, x26, \op, 7
 678         __frame_regs    x27, x28, \op, 9
 679
 680         .ifc            \op, ld
 681         .if             .Lframe_regcount == -1
 682         .error          "frame_push/frame_pop may not be nested"
 683         .endif
 684         ldp             x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra
 685         .set            .Lframe_regcount, -1
 686         .endif
 687         .endm
 688
 689 /*
 690  * Check whether to yield to another runnable task from kernel mode NEON code
 691  * (which runs with preemption disabled).
 692  *
 693  * if_will_cond_yield_neon
 694  *        // pre-yield patchup code
 695  * do_cond_yield_neon
 696  *        // post-yield patchup code
 697  * endif_yield_neon    <label>
 698  *
 699  * where <label> is optional, and marks the point where execution will resume
 700  * after a yield has been performed. If omitted, execution resumes right after
 701  * the endif_yield_neon invocation. Note that the entire sequence, including
 702  * the provided patchup code, will be omitted from the image if CONFIG_PREEMPT
 703  * is not defined.
 704  *
 705  * As a convenience, in the case where no patchup code is required, the above
 706  * sequence may be abbreviated to
 707  *
 708  * cond_yield_neon <label>
 709  *
 710  * Note that the patchup code does not support assembler directives that change
 711  * the output section, any use of such directives is undefined.
 712  *
 713  * The yield itself consists of the following:
 714  * - Check whether the preempt count is exactly 1 and a reschedule is also
 715  *   needed. If so, calling of preempt_enable() in kernel_neon_end() will
 716  *   trigger a reschedule. If it is not the case, yielding is pointless.
 717  * - Disable and re-enable kernel mode NEON, and branch to the yield fixup
 718  *   code.
 719  *
 720  * This macro sequence may clobber all CPU state that is not guaranteed by the
 721  * AAPCS to be preserved across an ordinary function call.
 722  */
 723
 724         .macro          cond_yield_neon, lbl
 725         if_will_cond_yield_neon
 726         do_cond_yield_neon
 727         endif_yield_neon        \lbl
 728         .endm
 729
 730         .macro          if_will_cond_yield_neon
 731 #ifdef CONFIG_PREEMPT
 732         get_current_task        x0
 733         ldr             x0, [x0, #TSK_TI_PREEMPT]
 734         sub             x0, x0, #PREEMPT_DISABLE_OFFSET
 735         cbz             x0, .Lyield_\@
 736         /* fall through to endif_yield_neon */
 737         .subsection     1
 738 .Lyield_\@ :
 739 #else
 740         .section        ".discard.cond_yield_neon", "ax"
 741 #endif
 742         .endm
 743
 744         .macro          do_cond_yield_neon
 745         bl              kernel_neon_end
 746         bl              kernel_neon_begin
 747         .endm
 748
 749         .macro          endif_yield_neon, lbl
 750         .ifnb           \lbl
 751         b               \lbl
 752         .else
 753         b               .Lyield_out_\@
 754         .endif
 755         .previous
 756 .Lyield_out_\@ :
 757         .endm
 758
 759 #endif  /* __ASM_ASSEMBLER_H */