arch/x86/include/asm/jump_label.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_X86_JUMP_LABEL_H
   3 #define _ASM_X86_JUMP_LABEL_H
   4
   5 #define JUMP_LABEL_NOP_SIZE 5
   6
   7 #ifdef CONFIG_X86_64
   8 # define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
   9 #else
  10 # define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
  11 #endif
  12
  13 #include <asm/asm.h>
  14 #include <asm/nops.h>
  15
  16 #ifndef __ASSEMBLY__
  17
  18 #include <linux/stringify.h>
  19 #include <linux/types.h>
  20
  21 static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
  22 {
  23         asm_volatile_goto("1:"
  24                 ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
  25                 ".pushsection __jump_table,  \"aw\" \n\t"
  26                 _ASM_ALIGN "\n\t"
  27                 _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
  28                 ".popsection \n\t"
  29                 : :  "i" (key), "i" (branch) : : l_yes);
  30
  31         return false;
  32 l_yes:
  33         return true;
  34 }
  35
  36 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
  37 {
  38         asm_volatile_goto("1:"
  39                 ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
  40                 "2:\n\t"
  41                 ".pushsection __jump_table,  \"aw\" \n\t"
  42                 _ASM_ALIGN "\n\t"
  43                 _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
  44                 ".popsection \n\t"
  45                 : :  "i" (key), "i" (branch) : : l_yes);
  46
  47         return false;
  48 l_yes:
  49         return true;
  50 }
  51
  52 #ifdef CONFIG_X86_64
  53 typedef u64 jump_label_t;
  54 #else
  55 typedef u32 jump_label_t;
  56 #endif
  57
  58 struct jump_entry {
  59         jump_label_t code;
  60         jump_label_t target;
  61         jump_label_t key;
  62 };
  63
  64 #else   /* __ASSEMBLY__ */
  65
  66 .macro STATIC_JUMP_IF_TRUE target, key, def
  67 .Lstatic_jump_\@:
  68         .if \def
  69         /* Equivalent to "jmp.d32 \target" */
  70         .byte           0xe9
  71         .long           \target - .Lstatic_jump_after_\@
  72 .Lstatic_jump_after_\@:
  73         .else
  74         .byte           STATIC_KEY_INIT_NOP
  75         .endif
  76         .pushsection __jump_table, "aw"
  77         _ASM_ALIGN
  78         _ASM_PTR        .Lstatic_jump_\@, \target, \key
  79         .popsection
  80 .endm
  81
  82 .macro STATIC_JUMP_IF_FALSE target, key, def
  83 .Lstatic_jump_\@:
  84         .if \def
  85         .byte           STATIC_KEY_INIT_NOP
  86         .else
  87         /* Equivalent to "jmp.d32 \target" */
  88         .byte           0xe9
  89         .long           \target - .Lstatic_jump_after_\@
  90 .Lstatic_jump_after_\@:
  91         .endif
  92         .pushsection __jump_table, "aw"
  93         _ASM_ALIGN
  94         _ASM_PTR        .Lstatic_jump_\@, \target, \key + 1
  95         .popsection
  96 .endm
  97
  98 #endif  /* __ASSEMBLY__ */
  99
 100 #endif