kernel/bpf/trampoline.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2019 Facebook */
   3 #include <linux/hash.h>
   4 #include <linux/bpf.h>
   5 #include <linux/filter.h>
   6
   7 /* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */
   8 #define TRAMPOLINE_HASH_BITS 10
   9 #define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS)
  10
  11 static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE];
  12
  13 /* serializes access to trampoline_table */
  14 static DEFINE_MUTEX(trampoline_mutex);
  15
  16 struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
  17 {
  18         struct bpf_trampoline *tr;
  19         struct hlist_head *head;
  20         void *image;
  21         int i;
  22
  23         mutex_lock(&trampoline_mutex);
  24         head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)];
  25         hlist_for_each_entry(tr, head, hlist) {
  26                 if (tr->key == key) {
  27                         refcount_inc(&tr->refcnt);
  28                         goto out;
  29                 }
  30         }
  31         tr = kzalloc(sizeof(*tr), GFP_KERNEL);
  32         if (!tr)
  33                 goto out;
  34
  35         /* is_root was checked earlier. No need for bpf_jit_charge_modmem() */
  36         image = bpf_jit_alloc_exec(PAGE_SIZE);
  37         if (!image) {
  38                 kfree(tr);
  39                 tr = NULL;
  40                 goto out;
  41         }
  42
  43         tr->key = key;
  44         INIT_HLIST_NODE(&tr->hlist);
  45         hlist_add_head(&tr->hlist, head);
  46         refcount_set(&tr->refcnt, 1);
  47         mutex_init(&tr->mutex);
  48         for (i = 0; i < BPF_TRAMP_MAX; i++)
  49                 INIT_HLIST_HEAD(&tr->progs_hlist[i]);
  50
  51         set_vm_flush_reset_perms(image);
  52         /* Keep image as writeable. The alternative is to keep flipping ro/rw
  53          * everytime new program is attached or detached.
  54          */
  55         set_memory_x((long)image, 1);
  56         tr->image = image;
  57 out:
  58         mutex_unlock(&trampoline_mutex);
  59         return tr;
  60 }
  61
  62 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
  63  * bytes on x86.  Pick a number to fit into PAGE_SIZE / 2
  64  */
  65 #define BPF_MAX_TRAMP_PROGS 40
  66
  67 static int bpf_trampoline_update(struct bpf_trampoline *tr)
  68 {
  69         void *old_image = tr->image + ((tr->selector + 1) & 1) * PAGE_SIZE/2;
  70         void *new_image = tr->image + (tr->selector & 1) * PAGE_SIZE/2;
  71         struct bpf_prog *progs_to_run[BPF_MAX_TRAMP_PROGS];
  72         int fentry_cnt = tr->progs_cnt[BPF_TRAMP_FENTRY];
  73         int fexit_cnt = tr->progs_cnt[BPF_TRAMP_FEXIT];
  74         struct bpf_prog **progs, **fentry, **fexit;
  75         u32 flags = BPF_TRAMP_F_RESTORE_REGS;
  76         struct bpf_prog_aux *aux;
  77         int err;
  78
  79         if (fentry_cnt + fexit_cnt == 0) {
  80                 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL,
  81                                          old_image, NULL);
  82                 tr->selector = 0;
  83                 goto out;
  84         }
  85
  86         /* populate fentry progs */
  87         fentry = progs = progs_to_run;
  88         hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FENTRY], tramp_hlist)
  89                 *progs++ = aux->prog;
  90
  91         /* populate fexit progs */
  92         fexit = progs;
  93         hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FEXIT], tramp_hlist)
  94                 *progs++ = aux->prog;
  95
  96         if (fexit_cnt)
  97                 flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;
  98
  99         err = arch_prepare_bpf_trampoline(new_image, &tr->func.model, flags,
 100                                           fentry, fentry_cnt,
 101                                           fexit, fexit_cnt,
 102                                           tr->func.addr);
 103         if (err)
 104                 goto out;
 105
 106         if (tr->selector)
 107                 /* progs already running at this address */
 108                 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL,
 109                                          old_image, new_image);
 110         else
 111                 /* first time registering */
 112                 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL, NULL,
 113                                          new_image);
 114         if (err)
 115                 goto out;
 116         tr->selector++;
 117 out:
 118         return err;
 119 }
 120
 121 static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(enum bpf_attach_type t)
 122 {
 123         switch (t) {
 124         case BPF_TRACE_FENTRY:
 125                 return BPF_TRAMP_FENTRY;
 126         default:
 127                 return BPF_TRAMP_FEXIT;
 128         }
 129 }
 130
 131 int bpf_trampoline_link_prog(struct bpf_prog *prog)
 132 {
 133         enum bpf_tramp_prog_type kind;
 134         struct bpf_trampoline *tr;
 135         int err = 0;
 136
 137         tr = prog->aux->trampoline;
 138         kind = bpf_attach_type_to_tramp(prog->expected_attach_type);
 139         mutex_lock(&tr->mutex);
 140         if (tr->progs_cnt[BPF_TRAMP_FENTRY] + tr->progs_cnt[BPF_TRAMP_FEXIT]
 141             >= BPF_MAX_TRAMP_PROGS) {
 142                 err = -E2BIG;
 143                 goto out;
 144         }
 145         if (!hlist_unhashed(&prog->aux->tramp_hlist)) {
 146                 /* prog already linked */
 147                 err = -EBUSY;
 148                 goto out;
 149         }
 150         hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]);
 151         tr->progs_cnt[kind]++;
 152         err = bpf_trampoline_update(prog->aux->trampoline);
 153         if (err) {
 154                 hlist_del(&prog->aux->tramp_hlist);
 155                 tr->progs_cnt[kind]--;
 156         }
 157 out:
 158         mutex_unlock(&tr->mutex);
 159         return err;
 160 }
 161
 162 /* bpf_trampoline_unlink_prog() should never fail. */
 163 int bpf_trampoline_unlink_prog(struct bpf_prog *prog)
 164 {
 165         enum bpf_tramp_prog_type kind;
 166         struct bpf_trampoline *tr;
 167         int err;
 168
 169         tr = prog->aux->trampoline;
 170         kind = bpf_attach_type_to_tramp(prog->expected_attach_type);
 171         mutex_lock(&tr->mutex);
 172         hlist_del(&prog->aux->tramp_hlist);
 173         tr->progs_cnt[kind]--;
 174         err = bpf_trampoline_update(prog->aux->trampoline);
 175         mutex_unlock(&tr->mutex);
 176         return err;
 177 }
 178
 179 void bpf_trampoline_put(struct bpf_trampoline *tr)
 180 {
 181         if (!tr)
 182                 return;
 183         mutex_lock(&trampoline_mutex);
 184         if (!refcount_dec_and_test(&tr->refcnt))
 185                 goto out;
 186         WARN_ON_ONCE(mutex_is_locked(&tr->mutex));
 187         if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FENTRY])))
 188                 goto out;
 189         if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FEXIT])))
 190                 goto out;
 191         bpf_jit_free_exec(tr->image);
 192         hlist_del(&tr->hlist);
 193         kfree(tr);
 194 out:
 195         mutex_unlock(&trampoline_mutex);
 196 }
 197
 198 /* The logic is similar to BPF_PROG_RUN, but with explicit rcu and preempt that
 199  * are needed for trampoline. The macro is split into
 200  * call _bpf_prog_enter
 201  * call prog->bpf_func
 202  * call __bpf_prog_exit
 203  */
 204 u64 notrace __bpf_prog_enter(void)
 205 {
 206         u64 start = 0;
 207
 208         rcu_read_lock();
 209         preempt_disable();
 210         if (static_branch_unlikely(&bpf_stats_enabled_key))
 211                 start = sched_clock();
 212         return start;
 213 }
 214
 215 void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start)
 216 {
 217         struct bpf_prog_stats *stats;
 218
 219         if (static_branch_unlikely(&bpf_stats_enabled_key) &&
 220             /* static_key could be enabled in __bpf_prog_enter
 221              * and disabled in __bpf_prog_exit.
 222              * And vice versa.
 223              * Hence check that 'start' is not zero.
 224              */
 225             start) {
 226                 stats = this_cpu_ptr(prog->aux->stats);
 227                 u64_stats_update_begin(&stats->syncp);
 228                 stats->cnt++;
 229                 stats->nsecs += sched_clock() - start;
 230                 u64_stats_update_end(&stats->syncp);
 231         }
 232         preempt_enable();
 233         rcu_read_unlock();
 234 }
 235
 236 int __weak
 237 arch_prepare_bpf_trampoline(void *image, struct btf_func_model *m, u32 flags,
 238                             struct bpf_prog **fentry_progs, int fentry_cnt,
 239                             struct bpf_prog **fexit_progs, int fexit_cnt,
 240                             void *orig_call)
 241 {
 242         return -ENOTSUPP;
 243 }
 244
 245 static int __init init_trampolines(void)
 246 {
 247         int i;
 248
 249         for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++)
 250                 INIT_HLIST_HEAD(&trampoline_table[i]);
 251         return 0;
 252 }
 253 late_initcall(init_trampolines);