arch/x86/kernel/jump_label.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * jump label x86 support
   4  *
   5  * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
   6  *
   7  */
   8 #include <linux/jump_label.h>
   9 #include <linux/memory.h>
  10 #include <linux/uaccess.h>
  11 #include <linux/module.h>
  12 #include <linux/list.h>
  13 #include <linux/jhash.h>
  14 #include <linux/cpu.h>
  15 #include <asm/kprobes.h>
  16 #include <asm/alternative.h>
  17 #include <asm/text-patching.h>
  18
  19 static void bug_at(const void *ip, int line)
  20 {
  21         /*
  22          * The location is not an op that we were expecting.
  23          * Something went wrong. Crash the box, as something could be
  24          * corrupting the kernel.
  25          */
  26         pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph) %d\n", ip, ip, ip, line);
  27         BUG();
  28 }
  29
  30 static const void *
  31 __jump_label_set_jump_code(struct jump_entry *entry, enum jump_label_type type, int init)
  32 {
  33         const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
  34         const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
  35         const void *expect, *code;
  36         const void *addr, *dest;
  37         int line;
  38
  39         addr = (void *)jump_entry_code(entry);
  40         dest = (void *)jump_entry_target(entry);
  41
  42         code = text_gen_insn(JMP32_INSN_OPCODE, addr, dest);
  43
  44         if (init) {
  45                 expect = default_nop; line = __LINE__;
  46         } else if (type == JUMP_LABEL_JMP) {
  47                 expect = ideal_nop; line = __LINE__;
  48         } else {
  49                 expect = code; line = __LINE__;
  50         }
  51
  52         if (memcmp(addr, expect, JUMP_LABEL_NOP_SIZE))
  53                 bug_at(addr, line);
  54
  55         if (type == JUMP_LABEL_NOP)
  56                 code = ideal_nop;
  57
  58         return code;
  59 }
  60
  61 static inline void __jump_label_transform(struct jump_entry *entry,
  62                                           enum jump_label_type type,
  63                                           int init)
  64 {
  65         const void *opcode = __jump_label_set_jump_code(entry, type, init);
  66
  67         /*
  68          * As long as only a single processor is running and the code is still
  69          * not marked as RO, text_poke_early() can be used; Checking that
  70          * system_state is SYSTEM_BOOTING guarantees it. It will be set to
  71          * SYSTEM_SCHEDULING before other cores are awaken and before the
  72          * code is write-protected.
  73          *
  74          * At the time the change is being done, just ignore whether we
  75          * are doing nop -> jump or jump -> nop transition, and assume
  76          * always nop being the 'currently valid' instruction
  77          */
  78         if (init || system_state == SYSTEM_BOOTING) {
  79                 text_poke_early((void *)jump_entry_code(entry), opcode,
  80                                 JUMP_LABEL_NOP_SIZE);
  81                 return;
  82         }
  83
  84         text_poke_bp((void *)jump_entry_code(entry), opcode, JUMP_LABEL_NOP_SIZE, NULL);
  85 }
  86
  87 static void __ref jump_label_transform(struct jump_entry *entry,
  88                                        enum jump_label_type type,
  89                                        int init)
  90 {
  91         mutex_lock(&text_mutex);
  92         __jump_label_transform(entry, type, init);
  93         mutex_unlock(&text_mutex);
  94 }
  95
  96 void arch_jump_label_transform(struct jump_entry *entry,
  97                                enum jump_label_type type)
  98 {
  99         jump_label_transform(entry, type, 0);
 100 }
 101
 102 bool arch_jump_label_transform_queue(struct jump_entry *entry,
 103                                      enum jump_label_type type)
 104 {
 105         const void *opcode;
 106
 107         if (system_state == SYSTEM_BOOTING) {
 108                 /*
 109                  * Fallback to the non-batching mode.
 110                  */
 111                 arch_jump_label_transform(entry, type);
 112                 return true;
 113         }
 114
 115         mutex_lock(&text_mutex);
 116         opcode = __jump_label_set_jump_code(entry, type, 0);
 117         text_poke_queue((void *)jump_entry_code(entry),
 118                         opcode, JUMP_LABEL_NOP_SIZE, NULL);
 119         mutex_unlock(&text_mutex);
 120         return true;
 121 }
 122
 123 void arch_jump_label_transform_apply(void)
 124 {
 125         mutex_lock(&text_mutex);
 126         text_poke_finish();
 127         mutex_unlock(&text_mutex);
 128 }
 129
 130 static enum {
 131         JL_STATE_START,
 132         JL_STATE_NO_UPDATE,
 133         JL_STATE_UPDATE,
 134 } jlstate __initdata_or_module = JL_STATE_START;
 135
 136 __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,
 137                                       enum jump_label_type type)
 138 {
 139         /*
 140          * This function is called at boot up and when modules are
 141          * first loaded. Check if the default nop, the one that is
 142          * inserted at compile time, is the ideal nop. If it is, then
 143          * we do not need to update the nop, and we can leave it as is.
 144          * If it is not, then we need to update the nop to the ideal nop.
 145          */
 146         if (jlstate == JL_STATE_START) {
 147                 const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
 148                 const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
 149
 150                 if (memcmp(ideal_nop, default_nop, 5) != 0)
 151                         jlstate = JL_STATE_UPDATE;
 152                 else
 153                         jlstate = JL_STATE_NO_UPDATE;
 154         }
 155         if (jlstate == JL_STATE_UPDATE)
 156                 jump_label_transform(entry, type, 1);
 157 }