arch/riscv/kernel/patch.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020 SiFive
   4  */
   5
   6 #include <linux/spinlock.h>
   7 #include <linux/mm.h>
   8 #include <linux/memory.h>
   9 #include <linux/uaccess.h>
  10 #include <linux/stop_machine.h>
  11 #include <asm/kprobes.h>
  12 #include <asm/cacheflush.h>
  13 #include <asm/fixmap.h>
  14 #include <asm/patch.h>
  15
  16 struct patch_insn {
  17         void *addr;
  18         u32 insn;
  19         atomic_t cpu_count;
  20 };
  21
  22 #ifdef CONFIG_MMU
  23 static void *patch_map(void *addr, int fixmap)
  24 {
  25         uintptr_t uintaddr = (uintptr_t) addr;
  26         struct page *page;
  27
  28         if (core_kernel_text(uintaddr))
  29                 page = phys_to_page(__pa_symbol(addr));
  30         else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
  31                 page = vmalloc_to_page(addr);
  32         else
  33                 return addr;
  34
  35         BUG_ON(!page);
  36
  37         return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
  38                                          (uintaddr & ~PAGE_MASK));
  39 }
  40 NOKPROBE_SYMBOL(patch_map);
  41
  42 static void patch_unmap(int fixmap)
  43 {
  44         clear_fixmap(fixmap);
  45 }
  46 NOKPROBE_SYMBOL(patch_unmap);
  47
  48 static int patch_insn_write(void *addr, const void *insn, size_t len)
  49 {
  50         void *waddr = addr;
  51         bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;
  52         int ret;
  53
  54         /*
  55          * Before reaching here, it was expected to lock the text_mutex
  56          * already, so we don't need to give another lock here and could
  57          * ensure that it was safe between each cores.
  58          */
  59         lockdep_assert_held(&text_mutex);
  60
  61         if (across_pages)
  62                 patch_map(addr + len, FIX_TEXT_POKE1);
  63
  64         waddr = patch_map(addr, FIX_TEXT_POKE0);
  65
  66         ret = copy_to_kernel_nofault(waddr, insn, len);
  67
  68         patch_unmap(FIX_TEXT_POKE0);
  69
  70         if (across_pages)
  71                 patch_unmap(FIX_TEXT_POKE1);
  72
  73         return ret;
  74 }
  75 NOKPROBE_SYMBOL(patch_insn_write);
  76 #else
  77 static int patch_insn_write(void *addr, const void *insn, size_t len)
  78 {
  79         return copy_to_kernel_nofault(addr, insn, len);
  80 }
  81 NOKPROBE_SYMBOL(patch_insn_write);
  82 #endif /* CONFIG_MMU */
  83
  84 int patch_text_nosync(void *addr, const void *insns, size_t len)
  85 {
  86         u32 *tp = addr;
  87         int ret;
  88
  89         ret = patch_insn_write(tp, insns, len);
  90
  91         if (!ret)
  92                 flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len);
  93
  94         return ret;
  95 }
  96 NOKPROBE_SYMBOL(patch_text_nosync);
  97
  98 static int patch_text_cb(void *data)
  99 {
 100         struct patch_insn *patch = data;
 101         int ret = 0;
 102
 103         if (atomic_inc_return(&patch->cpu_count) == 1) {
 104                 ret =
 105                     patch_text_nosync(patch->addr, &patch->insn,
 106                                             GET_INSN_LENGTH(patch->insn));
 107                 atomic_inc(&patch->cpu_count);
 108         } else {
 109                 while (atomic_read(&patch->cpu_count) <= num_online_cpus())
 110                         cpu_relax();
 111                 smp_mb();
 112         }
 113
 114         return ret;
 115 }
 116 NOKPROBE_SYMBOL(patch_text_cb);
 117
 118 int patch_text(void *addr, u32 insn)
 119 {
 120         struct patch_insn patch = {
 121                 .addr = addr,
 122                 .insn = insn,
 123                 .cpu_count = ATOMIC_INIT(0),
 124         };
 125
 126         return stop_machine_cpuslocked(patch_text_cb,
 127                                        &patch, cpu_online_mask);
 128 }
 129 NOKPROBE_SYMBOL(patch_text);