net/bpf/test_run.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2017 Facebook
   3  */
   4 #include <linux/bpf.h>
   5 #include <linux/slab.h>
   6 #include <linux/vmalloc.h>
   7 #include <linux/etherdevice.h>
   8 #include <linux/filter.h>
   9 #include <linux/sched/signal.h>
  10 #include <net/bpf_sk_storage.h>
  11 #include <net/sock.h>
  12 #include <net/tcp.h>
  13
  14 #define CREATE_TRACE_POINTS
  15 #include <trace/events/bpf_test_run.h>
  16
  17 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
  18                         u32 *retval, u32 *time)
  19 {
  20         struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL };
  21         enum bpf_cgroup_storage_type stype;
  22         u64 time_start, time_spent = 0;
  23         int ret = 0;
  24         u32 i;
  25
  26         for_each_cgroup_storage_type(stype) {
  27                 storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
  28                 if (IS_ERR(storage[stype])) {
  29                         storage[stype] = NULL;
  30                         for_each_cgroup_storage_type(stype)
  31                                 bpf_cgroup_storage_free(storage[stype]);
  32                         return -ENOMEM;
  33                 }
  34         }
  35
  36         if (!repeat)
  37                 repeat = 1;
  38
  39         rcu_read_lock();
  40         preempt_disable();
  41         time_start = ktime_get_ns();
  42         for (i = 0; i < repeat; i++) {
  43                 bpf_cgroup_storage_set(storage);
  44                 *retval = BPF_PROG_RUN(prog, ctx);
  45
  46                 if (signal_pending(current)) {
  47                         ret = -EINTR;
  48                         break;
  49                 }
  50
  51                 if (need_resched()) {
  52                         time_spent += ktime_get_ns() - time_start;
  53                         preempt_enable();
  54                         rcu_read_unlock();
  55
  56                         cond_resched();
  57
  58                         rcu_read_lock();
  59                         preempt_disable();
  60                         time_start = ktime_get_ns();
  61                 }
  62         }
  63         time_spent += ktime_get_ns() - time_start;
  64         preempt_enable();
  65         rcu_read_unlock();
  66
  67         do_div(time_spent, repeat);
  68         *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
  69
  70         for_each_cgroup_storage_type(stype)
  71                 bpf_cgroup_storage_free(storage[stype]);
  72
  73         return ret;
  74 }
  75
  76 static int bpf_test_finish(const union bpf_attr *kattr,
  77                            union bpf_attr __user *uattr, const void *data,
  78                            u32 size, u32 retval, u32 duration)
  79 {
  80         void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
  81         int err = -EFAULT;
  82         u32 copy_size = size;
  83
  84         /* Clamp copy if the user has provided a size hint, but copy the full
  85          * buffer if not to retain old behaviour.
  86          */
  87         if (kattr->test.data_size_out &&
  88             copy_size > kattr->test.data_size_out) {
  89                 copy_size = kattr->test.data_size_out;
  90                 err = -ENOSPC;
  91         }
  92
  93         if (data_out && copy_to_user(data_out, data, copy_size))
  94                 goto out;
  95         if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
  96                 goto out;
  97         if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
  98                 goto out;
  99         if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
 100                 goto out;
 101         if (err != -ENOSPC)
 102                 err = 0;
 103 out:
 104         trace_bpf_test_finish(&err);
 105         return err;
 106 }
 107
 108 static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
 109                            u32 headroom, u32 tailroom)
 110 {
 111         void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
 112         void *data;
 113
 114         if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
 115                 return ERR_PTR(-EINVAL);
 116
 117         data = kzalloc(size + headroom + tailroom, GFP_USER);
 118         if (!data)
 119                 return ERR_PTR(-ENOMEM);
 120
 121         if (copy_from_user(data + headroom, data_in, size)) {
 122                 kfree(data);
 123                 return ERR_PTR(-EFAULT);
 124         }
 125         return data;
 126 }
 127
 128 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
 129 {
 130         void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
 131         void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
 132         u32 size = kattr->test.ctx_size_in;
 133         void *data;
 134         int err;
 135
 136         if (!data_in && !data_out)
 137                 return NULL;
 138
 139         data = kzalloc(max_size, GFP_USER);
 140         if (!data)
 141                 return ERR_PTR(-ENOMEM);
 142
 143         if (data_in) {
 144                 err = bpf_check_uarg_tail_zero(data_in, max_size, size);
 145                 if (err) {
 146                         kfree(data);
 147                         return ERR_PTR(err);
 148                 }
 149
 150                 size = min_t(u32, max_size, size);
 151                 if (copy_from_user(data, data_in, size)) {
 152                         kfree(data);
 153                         return ERR_PTR(-EFAULT);
 154                 }
 155         }
 156         return data;
 157 }
 158
 159 static int bpf_ctx_finish(const union bpf_attr *kattr,
 160                           union bpf_attr __user *uattr, const void *data,
 161                           u32 size)
 162 {
 163         void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
 164         int err = -EFAULT;
 165         u32 copy_size = size;
 166
 167         if (!data || !data_out)
 168                 return 0;
 169
 170         if (copy_size > kattr->test.ctx_size_out) {
 171                 copy_size = kattr->test.ctx_size_out;
 172                 err = -ENOSPC;
 173         }
 174
 175         if (copy_to_user(data_out, data, copy_size))
 176                 goto out;
 177         if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
 178                 goto out;
 179         if (err != -ENOSPC)
 180                 err = 0;
 181 out:
 182         return err;
 183 }
 184
 185 /**
 186  * range_is_zero - test whether buffer is initialized
 187  * @buf: buffer to check
 188  * @from: check from this position
 189  * @to: check up until (excluding) this position
 190  *
 191  * This function returns true if the there is a non-zero byte
 192  * in the buf in the range [from,to).
 193  */
 194 static inline bool range_is_zero(void *buf, size_t from, size_t to)
 195 {
 196         return !memchr_inv((u8 *)buf + from, 0, to - from);
 197 }
 198
 199 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
 200 {
 201         struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
 202
 203         if (!__skb)
 204                 return 0;
 205
 206         /* make sure the fields we don't use are zeroed */
 207         if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, priority)))
 208                 return -EINVAL;
 209
 210         /* priority is allowed */
 211
 212         if (!range_is_zero(__skb, offsetof(struct __sk_buff, priority) +
 213                            FIELD_SIZEOF(struct __sk_buff, priority),
 214                            offsetof(struct __sk_buff, cb)))
 215                 return -EINVAL;
 216
 217         /* cb is allowed */
 218
 219         if (!range_is_zero(__skb, offsetof(struct __sk_buff, cb) +
 220                            FIELD_SIZEOF(struct __sk_buff, cb),
 221                            sizeof(struct __sk_buff)))
 222                 return -EINVAL;
 223
 224         skb->priority = __skb->priority;
 225         memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
 226
 227         return 0;
 228 }
 229
 230 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
 231 {
 232         struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
 233
 234         if (!__skb)
 235                 return;
 236
 237         __skb->priority = skb->priority;
 238         memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
 239 }
 240
 241 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
 242                           union bpf_attr __user *uattr)
 243 {
 244         bool is_l2 = false, is_direct_pkt_access = false;
 245         u32 size = kattr->test.data_size_in;
 246         u32 repeat = kattr->test.repeat;
 247         struct __sk_buff *ctx = NULL;
 248         u32 retval, duration;
 249         int hh_len = ETH_HLEN;
 250         struct sk_buff *skb;
 251         struct sock *sk;
 252         void *data;
 253         int ret;
 254
 255         data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
 256                              SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
 257         if (IS_ERR(data))
 258                 return PTR_ERR(data);
 259
 260         ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
 261         if (IS_ERR(ctx)) {
 262                 kfree(data);
 263                 return PTR_ERR(ctx);
 264         }
 265
 266         switch (prog->type) {
 267         case BPF_PROG_TYPE_SCHED_CLS:
 268         case BPF_PROG_TYPE_SCHED_ACT:
 269                 is_l2 = true;
 270                 /* fall through */
 271         case BPF_PROG_TYPE_LWT_IN:
 272         case BPF_PROG_TYPE_LWT_OUT:
 273         case BPF_PROG_TYPE_LWT_XMIT:
 274                 is_direct_pkt_access = true;
 275                 break;
 276         default:
 277                 break;
 278         }
 279
 280         sk = kzalloc(sizeof(struct sock), GFP_USER);
 281         if (!sk) {
 282                 kfree(data);
 283                 kfree(ctx);
 284                 return -ENOMEM;
 285         }
 286         sock_net_set(sk, current->nsproxy->net_ns);
 287         sock_init_data(NULL, sk);
 288
 289         skb = build_skb(data, 0);
 290         if (!skb) {
 291                 kfree(data);
 292                 kfree(ctx);
 293                 kfree(sk);
 294                 return -ENOMEM;
 295         }
 296         skb->sk = sk;
 297
 298         skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
 299         __skb_put(skb, size);
 300         skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev);
 301         skb_reset_network_header(skb);
 302
 303         if (is_l2)
 304                 __skb_push(skb, hh_len);
 305         if (is_direct_pkt_access)
 306                 bpf_compute_data_pointers(skb);
 307         ret = convert___skb_to_skb(skb, ctx);
 308         if (ret)
 309                 goto out;
 310         ret = bpf_test_run(prog, skb, repeat, &retval, &duration);
 311         if (ret)
 312                 goto out;
 313         if (!is_l2) {
 314                 if (skb_headroom(skb) < hh_len) {
 315                         int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
 316
 317                         if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
 318                                 ret = -ENOMEM;
 319                                 goto out;
 320                         }
 321                 }
 322                 memset(__skb_push(skb, hh_len), 0, hh_len);
 323         }
 324         convert_skb_to___skb(skb, ctx);
 325
 326         size = skb->len;
 327         /* bpf program can never convert linear skb to non-linear */
 328         if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
 329                 size = skb_headlen(skb);
 330         ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
 331         if (!ret)
 332                 ret = bpf_ctx_finish(kattr, uattr, ctx,
 333                                      sizeof(struct __sk_buff));
 334 out:
 335         kfree_skb(skb);
 336         bpf_sk_storage_free(sk);
 337         kfree(sk);
 338         kfree(ctx);
 339         return ret;
 340 }
 341
 342 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
 343                           union bpf_attr __user *uattr)
 344 {
 345         u32 size = kattr->test.data_size_in;
 346         u32 repeat = kattr->test.repeat;
 347         struct netdev_rx_queue *rxqueue;
 348         struct xdp_buff xdp = {};
 349         u32 retval, duration;
 350         void *data;
 351         int ret;
 352
 353         if (kattr->test.ctx_in || kattr->test.ctx_out)
 354                 return -EINVAL;
 355
 356         data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0);
 357         if (IS_ERR(data))
 358                 return PTR_ERR(data);
 359
 360         xdp.data_hard_start = data;
 361         xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN;
 362         xdp.data_meta = xdp.data;
 363         xdp.data_end = xdp.data + size;
 364
 365         rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
 366         xdp.rxq = &rxqueue->xdp_rxq;
 367
 368         ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration);
 369         if (ret)
 370                 goto out;
 371         if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN ||
 372             xdp.data_end != xdp.data + size)
 373                 size = xdp.data_end - xdp.data;
 374         ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration);
 375 out:
 376         kfree(data);
 377         return ret;
 378 }
 379
 380 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
 381                                      const union bpf_attr *kattr,
 382                                      union bpf_attr __user *uattr)
 383 {
 384         u32 size = kattr->test.data_size_in;
 385         struct bpf_flow_dissector ctx = {};
 386         u32 repeat = kattr->test.repeat;
 387         struct bpf_flow_keys flow_keys;
 388         u64 time_start, time_spent = 0;
 389         const struct ethhdr *eth;
 390         u32 retval, duration;
 391         void *data;
 392         int ret;
 393         u32 i;
 394
 395         if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
 396                 return -EINVAL;
 397
 398         if (kattr->test.ctx_in || kattr->test.ctx_out)
 399                 return -EINVAL;
 400
 401         if (size < ETH_HLEN)
 402                 return -EINVAL;
 403
 404         data = bpf_test_init(kattr, size, 0, 0);
 405         if (IS_ERR(data))
 406                 return PTR_ERR(data);
 407
 408         eth = (struct ethhdr *)data;
 409
 410         if (!repeat)
 411                 repeat = 1;
 412
 413         ctx.flow_keys = &flow_keys;
 414         ctx.data = data;
 415         ctx.data_end = (__u8 *)data + size;
 416
 417         rcu_read_lock();
 418         preempt_disable();
 419         time_start = ktime_get_ns();
 420         for (i = 0; i < repeat; i++) {
 421                 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
 422                                           size);
 423
 424                 if (signal_pending(current)) {
 425                         preempt_enable();
 426                         rcu_read_unlock();
 427
 428                         ret = -EINTR;
 429                         goto out;
 430                 }
 431
 432                 if (need_resched()) {
 433                         time_spent += ktime_get_ns() - time_start;
 434                         preempt_enable();
 435                         rcu_read_unlock();
 436
 437                         cond_resched();
 438
 439                         rcu_read_lock();
 440                         preempt_disable();
 441                         time_start = ktime_get_ns();
 442                 }
 443         }
 444         time_spent += ktime_get_ns() - time_start;
 445         preempt_enable();
 446         rcu_read_unlock();
 447
 448         do_div(time_spent, repeat);
 449         duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
 450
 451         ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys),
 452                               retval, duration);
 453
 454 out:
 455         kfree(data);
 456         return ret;
 457 }