kernel/livepatch/patch.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * patch.c - livepatch patching functions
   4  *
   5  * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
   6  * Copyright (C) 2014 SUSE
   7  * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
   8  */
   9
  10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  11
  12 #include <linux/livepatch.h>
  13 #include <linux/list.h>
  14 #include <linux/ftrace.h>
  15 #include <linux/rculist.h>
  16 #include <linux/slab.h>
  17 #include <linux/bug.h>
  18 #include <linux/printk.h>
  19 #include "core.h"
  20 #include "patch.h"
  21 #include "transition.h"
  22
  23 static LIST_HEAD(klp_ops);
  24
  25 struct klp_ops *klp_find_ops(void *old_func)
  26 {
  27         struct klp_ops *ops;
  28         struct klp_func *func;
  29
  30         list_for_each_entry(ops, &klp_ops, node) {
  31                 func = list_first_entry(&ops->func_stack, struct klp_func,
  32                                         stack_node);
  33                 if (func->old_func == old_func)
  34                         return ops;
  35         }
  36
  37         return NULL;
  38 }
  39
  40 static void notrace klp_ftrace_handler(unsigned long ip,
  41                                        unsigned long parent_ip,
  42                                        struct ftrace_ops *fops,
  43                                        struct pt_regs *regs)
  44 {
  45         struct klp_ops *ops;
  46         struct klp_func *func;
  47         int patch_state;
  48
  49         ops = container_of(fops, struct klp_ops, fops);
  50
  51         /*
  52          * A variant of synchronize_rcu() is used to allow patching functions
  53          * where RCU is not watching, see klp_synchronize_transition().
  54          */
  55         preempt_disable_notrace();
  56
  57         func = list_first_or_null_rcu(&ops->func_stack, struct klp_func,
  58                                       stack_node);
  59
  60         /*
  61          * func should never be NULL because preemption should be disabled here
  62          * and unregister_ftrace_function() does the equivalent of a
  63          * synchronize_rcu() before the func_stack removal.
  64          */
  65         if (WARN_ON_ONCE(!func))
  66                 goto unlock;
  67
  68         /*
  69          * In the enable path, enforce the order of the ops->func_stack and
  70          * func->transition reads.  The corresponding write barrier is in
  71          * __klp_enable_patch().
  72          *
  73          * (Note that this barrier technically isn't needed in the disable
  74          * path.  In the rare case where klp_update_patch_state() runs before
  75          * this handler, its TIF_PATCH_PENDING read and this func->transition
  76          * read need to be ordered.  But klp_update_patch_state() already
  77          * enforces that.)
  78          */
  79         smp_rmb();
  80
  81         if (unlikely(func->transition)) {
  82
  83                 /*
  84                  * Enforce the order of the func->transition and
  85                  * current->patch_state reads.  Otherwise we could read an
  86                  * out-of-date task state and pick the wrong function.  The
  87                  * corresponding write barrier is in klp_init_transition().
  88                  */
  89                 smp_rmb();
  90
  91                 patch_state = current->patch_state;
  92
  93                 WARN_ON_ONCE(patch_state == KLP_UNDEFINED);
  94
  95                 if (patch_state == KLP_UNPATCHED) {
  96                         /*
  97                          * Use the previously patched version of the function.
  98                          * If no previous patches exist, continue with the
  99                          * original function.
 100                          */
 101                         func = list_entry_rcu(func->stack_node.next,
 102                                               struct klp_func, stack_node);
 103
 104                         if (&func->stack_node == &ops->func_stack)
 105                                 goto unlock;
 106                 }
 107         }
 108
 109         /*
 110          * NOPs are used to replace existing patches with original code.
 111          * Do nothing! Setting pc would cause an infinite loop.
 112          */
 113         if (func->nop)
 114                 goto unlock;
 115
 116         klp_arch_set_pc(regs, (unsigned long)func->new_func);
 117
 118 unlock:
 119         preempt_enable_notrace();
 120 }
 121
 122 /*
 123  * Convert a function address into the appropriate ftrace location.
 124  *
 125  * Usually this is just the address of the function, but on some architectures
 126  * it's more complicated so allow them to provide a custom behaviour.
 127  */
 128 #ifndef klp_get_ftrace_location
 129 static unsigned long klp_get_ftrace_location(unsigned long faddr)
 130 {
 131         return faddr;
 132 }
 133 #endif
 134
 135 static void klp_unpatch_func(struct klp_func *func)
 136 {
 137         struct klp_ops *ops;
 138
 139         if (WARN_ON(!func->patched))
 140                 return;
 141         if (WARN_ON(!func->old_func))
 142                 return;
 143
 144         ops = klp_find_ops(func->old_func);
 145         if (WARN_ON(!ops))
 146                 return;
 147
 148         if (list_is_singular(&ops->func_stack)) {
 149                 unsigned long ftrace_loc;
 150
 151                 ftrace_loc =
 152                         klp_get_ftrace_location((unsigned long)func->old_func);
 153                 if (WARN_ON(!ftrace_loc))
 154                         return;
 155
 156                 WARN_ON(unregister_ftrace_function(&ops->fops));
 157                 WARN_ON(ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0));
 158
 159                 list_del_rcu(&func->stack_node);
 160                 list_del(&ops->node);
 161                 kfree(ops);
 162         } else {
 163                 list_del_rcu(&func->stack_node);
 164         }
 165
 166         func->patched = false;
 167 }
 168
 169 static int klp_patch_func(struct klp_func *func)
 170 {
 171         struct klp_ops *ops;
 172         int ret;
 173
 174         if (WARN_ON(!func->old_func))
 175                 return -EINVAL;
 176
 177         if (WARN_ON(func->patched))
 178                 return -EINVAL;
 179
 180         ops = klp_find_ops(func->old_func);
 181         if (!ops) {
 182                 unsigned long ftrace_loc;
 183
 184                 ftrace_loc =
 185                         klp_get_ftrace_location((unsigned long)func->old_func);
 186                 if (!ftrace_loc) {
 187                         pr_err("failed to find location for function '%s'\n",
 188                                 func->old_name);
 189                         return -EINVAL;
 190                 }
 191
 192                 ops = kzalloc(sizeof(*ops), GFP_KERNEL);
 193                 if (!ops)
 194                         return -ENOMEM;
 195
 196                 ops->fops.func = klp_ftrace_handler;
 197                 ops->fops.flags = FTRACE_OPS_FL_SAVE_REGS |
 198                                   FTRACE_OPS_FL_DYNAMIC |
 199                                   FTRACE_OPS_FL_IPMODIFY |
 200                                   FTRACE_OPS_FL_PERMANENT;
 201
 202                 list_add(&ops->node, &klp_ops);
 203
 204                 INIT_LIST_HEAD(&ops->func_stack);
 205                 list_add_rcu(&func->stack_node, &ops->func_stack);
 206
 207                 ret = ftrace_set_filter_ip(&ops->fops, ftrace_loc, 0, 0);
 208                 if (ret) {
 209                         pr_err("failed to set ftrace filter for function '%s' (%d)\n",
 210                                func->old_name, ret);
 211                         goto err;
 212                 }
 213
 214                 ret = register_ftrace_function(&ops->fops);
 215                 if (ret) {
 216                         pr_err("failed to register ftrace handler for function '%s' (%d)\n",
 217                                func->old_name, ret);
 218                         ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0);
 219                         goto err;
 220                 }
 221
 222
 223         } else {
 224                 list_add_rcu(&func->stack_node, &ops->func_stack);
 225         }
 226
 227         func->patched = true;
 228
 229         return 0;
 230
 231 err:
 232         list_del_rcu(&func->stack_node);
 233         list_del(&ops->node);
 234         kfree(ops);
 235         return ret;
 236 }
 237
 238 static void __klp_unpatch_object(struct klp_object *obj, bool nops_only)
 239 {
 240         struct klp_func *func;
 241
 242         klp_for_each_func(obj, func) {
 243                 if (nops_only && !func->nop)
 244                         continue;
 245
 246                 if (func->patched)
 247                         klp_unpatch_func(func);
 248         }
 249
 250         if (obj->dynamic || !nops_only)
 251                 obj->patched = false;
 252 }
 253
 254
 255 void klp_unpatch_object(struct klp_object *obj)
 256 {
 257         __klp_unpatch_object(obj, false);
 258 }
 259
 260 int klp_patch_object(struct klp_object *obj)
 261 {
 262         struct klp_func *func;
 263         int ret;
 264
 265         if (WARN_ON(obj->patched))
 266                 return -EINVAL;
 267
 268         klp_for_each_func(obj, func) {
 269                 ret = klp_patch_func(func);
 270                 if (ret) {
 271                         klp_unpatch_object(obj);
 272                         return ret;
 273                 }
 274         }
 275         obj->patched = true;
 276
 277         return 0;
 278 }
 279
 280 static void __klp_unpatch_objects(struct klp_patch *patch, bool nops_only)
 281 {
 282         struct klp_object *obj;
 283
 284         klp_for_each_object(patch, obj)
 285                 if (obj->patched)
 286                         __klp_unpatch_object(obj, nops_only);
 287 }
 288
 289 void klp_unpatch_objects(struct klp_patch *patch)
 290 {
 291         __klp_unpatch_objects(patch, false);
 292 }
 293
 294 void klp_unpatch_objects_dynamic(struct klp_patch *patch)
 295 {
 296         __klp_unpatch_objects(patch, true);
 297 }