kernel/bpf/mprog.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2023 Isovalent */
   3
   4 #include <linux/bpf.h>
   5 #include <linux/bpf_mprog.h>
   6
   7 static int bpf_mprog_link(struct bpf_tuple *tuple,
   8                           u32 id_or_fd, u32 flags,
   9                           enum bpf_prog_type type)
  10 {
  11         struct bpf_link *link = ERR_PTR(-EINVAL);
  12         bool id = flags & BPF_F_ID;
  13
  14         if (id)
  15                 link = bpf_link_by_id(id_or_fd);
  16         else if (id_or_fd)
  17                 link = bpf_link_get_from_fd(id_or_fd);
  18         if (IS_ERR(link))
  19                 return PTR_ERR(link);
  20         if (type && link->prog->type != type) {
  21                 bpf_link_put(link);
  22                 return -EINVAL;
  23         }
  24
  25         tuple->link = link;
  26         tuple->prog = link->prog;
  27         return 0;
  28 }
  29
  30 static int bpf_mprog_prog(struct bpf_tuple *tuple,
  31                           u32 id_or_fd, u32 flags,
  32                           enum bpf_prog_type type)
  33 {
  34         struct bpf_prog *prog = ERR_PTR(-EINVAL);
  35         bool id = flags & BPF_F_ID;
  36
  37         if (id)
  38                 prog = bpf_prog_by_id(id_or_fd);
  39         else if (id_or_fd)
  40                 prog = bpf_prog_get(id_or_fd);
  41         if (IS_ERR(prog))
  42                 return PTR_ERR(prog);
  43         if (type && prog->type != type) {
  44                 bpf_prog_put(prog);
  45                 return -EINVAL;
  46         }
  47
  48         tuple->link = NULL;
  49         tuple->prog = prog;
  50         return 0;
  51 }
  52
  53 static int bpf_mprog_tuple_relative(struct bpf_tuple *tuple,
  54                                     u32 id_or_fd, u32 flags,
  55                                     enum bpf_prog_type type)
  56 {
  57         bool link = flags & BPF_F_LINK;
  58         bool id = flags & BPF_F_ID;
  59
  60         memset(tuple, 0, sizeof(*tuple));
  61         if (link)
  62                 return bpf_mprog_link(tuple, id_or_fd, flags, type);
  63         /* If no relevant flag is set and no id_or_fd was passed, then
  64          * tuple link/prog is just NULLed. This is the case when before/
  65          * after selects first/last position without passing fd.
  66          */
  67         if (!id && !id_or_fd)
  68                 return 0;
  69         return bpf_mprog_prog(tuple, id_or_fd, flags, type);
  70 }
  71
  72 static void bpf_mprog_tuple_put(struct bpf_tuple *tuple)
  73 {
  74         if (tuple->link)
  75                 bpf_link_put(tuple->link);
  76         else if (tuple->prog)
  77                 bpf_prog_put(tuple->prog);
  78 }
  79
  80 /* The bpf_mprog_{replace,delete}() operate on exact idx position with the
  81  * one exception that for deletion we support delete from front/back. In
  82  * case of front idx is -1, in case of back idx is bpf_mprog_total(entry).
  83  * Adjustment to first and last entry is trivial. The bpf_mprog_insert()
  84  * we have to deal with the following cases:
  85  *
  86  * idx + before:
  87  *
  88  * Insert P4 before P3: idx for old array is 1, idx for new array is 2,
  89  * hence we adjust target idx for the new array, so that memmove copies
  90  * P1 and P2 to the new entry, and we insert P4 into idx 2. Inserting
  91  * before P1 would have old idx -1 and new idx 0.
  92  *
  93  * +--+--+--+     +--+--+--+--+     +--+--+--+--+
  94  * |P1|P2|P3| ==> |P1|P2|  |P3| ==> |P1|P2|P4|P3|
  95  * +--+--+--+     +--+--+--+--+     +--+--+--+--+
  96  *
  97  * idx + after:
  98  *
  99  * Insert P4 after P2: idx for old array is 2, idx for new array is 2.
 100  * Again, memmove copies P1 and P2 to the new entry, and we insert P4
 101  * into idx 2. Inserting after P3 would have both old/new idx at 4 aka
 102  * bpf_mprog_total(entry).
 103  *
 104  * +--+--+--+     +--+--+--+--+     +--+--+--+--+
 105  * |P1|P2|P3| ==> |P1|P2|  |P3| ==> |P1|P2|P4|P3|
 106  * +--+--+--+     +--+--+--+--+     +--+--+--+--+
 107  */
 108 static int bpf_mprog_replace(struct bpf_mprog_entry *entry,
 109                              struct bpf_mprog_entry **entry_new,
 110                              struct bpf_tuple *ntuple, int idx)
 111 {
 112         struct bpf_mprog_fp *fp;
 113         struct bpf_mprog_cp *cp;
 114         struct bpf_prog *oprog;
 115
 116         bpf_mprog_read(entry, idx, &fp, &cp);
 117         oprog = READ_ONCE(fp->prog);
 118         bpf_mprog_write(fp, cp, ntuple);
 119         if (!ntuple->link) {
 120                 WARN_ON_ONCE(cp->link);
 121                 bpf_prog_put(oprog);
 122         }
 123         *entry_new = entry;
 124         return 0;
 125 }
 126
 127 static int bpf_mprog_insert(struct bpf_mprog_entry *entry,
 128                             struct bpf_mprog_entry **entry_new,
 129                             struct bpf_tuple *ntuple, int idx, u32 flags)
 130 {
 131         int total = bpf_mprog_total(entry);
 132         struct bpf_mprog_entry *peer;
 133         struct bpf_mprog_fp *fp;
 134         struct bpf_mprog_cp *cp;
 135
 136         peer = bpf_mprog_peer(entry);
 137         bpf_mprog_entry_copy(peer, entry);
 138         if (idx == total)
 139                 goto insert;
 140         else if (flags & BPF_F_BEFORE)
 141                 idx += 1;
 142         bpf_mprog_entry_grow(peer, idx);
 143 insert:
 144         bpf_mprog_read(peer, idx, &fp, &cp);
 145         bpf_mprog_write(fp, cp, ntuple);
 146         bpf_mprog_inc(peer);
 147         *entry_new = peer;
 148         return 0;
 149 }
 150
 151 static int bpf_mprog_delete(struct bpf_mprog_entry *entry,
 152                             struct bpf_mprog_entry **entry_new,
 153                             struct bpf_tuple *dtuple, int idx)
 154 {
 155         int total = bpf_mprog_total(entry);
 156         struct bpf_mprog_entry *peer;
 157
 158         peer = bpf_mprog_peer(entry);
 159         bpf_mprog_entry_copy(peer, entry);
 160         if (idx == -1)
 161                 idx = 0;
 162         else if (idx == total)
 163                 idx = total - 1;
 164         bpf_mprog_entry_shrink(peer, idx);
 165         bpf_mprog_dec(peer);
 166         bpf_mprog_mark_for_release(peer, dtuple);
 167         *entry_new = peer;
 168         return 0;
 169 }
 170
 171 /* In bpf_mprog_pos_*() we evaluate the target position for the BPF
 172  * program/link that needs to be replaced, inserted or deleted for
 173  * each "rule" independently. If all rules agree on that position
 174  * or existing element, then enact replacement, addition or deletion.
 175  * If this is not the case, then the request cannot be satisfied and
 176  * we bail out with an error.
 177  */
 178 static int bpf_mprog_pos_exact(struct bpf_mprog_entry *entry,
 179                                struct bpf_tuple *tuple)
 180 {
 181         struct bpf_mprog_fp *fp;
 182         struct bpf_mprog_cp *cp;
 183         int i;
 184
 185         for (i = 0; i < bpf_mprog_total(entry); i++) {
 186                 bpf_mprog_read(entry, i, &fp, &cp);
 187                 if (tuple->prog == READ_ONCE(fp->prog))
 188                         return tuple->link == cp->link ? i : -EBUSY;
 189         }
 190         return -ENOENT;
 191 }
 192
 193 static int bpf_mprog_pos_before(struct bpf_mprog_entry *entry,
 194                                 struct bpf_tuple *tuple)
 195 {
 196         struct bpf_mprog_fp *fp;
 197         struct bpf_mprog_cp *cp;
 198         int i;
 199
 200         for (i = 0; i < bpf_mprog_total(entry); i++) {
 201                 bpf_mprog_read(entry, i, &fp, &cp);
 202                 if (tuple->prog == READ_ONCE(fp->prog) &&
 203                     (!tuple->link || tuple->link == cp->link))
 204                         return i - 1;
 205         }
 206         return tuple->prog ? -ENOENT : -1;
 207 }
 208
 209 static int bpf_mprog_pos_after(struct bpf_mprog_entry *entry,
 210                                struct bpf_tuple *tuple)
 211 {
 212         struct bpf_mprog_fp *fp;
 213         struct bpf_mprog_cp *cp;
 214         int i;
 215
 216         for (i = 0; i < bpf_mprog_total(entry); i++) {
 217                 bpf_mprog_read(entry, i, &fp, &cp);
 218                 if (tuple->prog == READ_ONCE(fp->prog) &&
 219                     (!tuple->link || tuple->link == cp->link))
 220                         return i + 1;
 221         }
 222         return tuple->prog ? -ENOENT : bpf_mprog_total(entry);
 223 }
 224
 225 int bpf_mprog_attach(struct bpf_mprog_entry *entry,
 226                      struct bpf_mprog_entry **entry_new,
 227                      struct bpf_prog *prog_new, struct bpf_link *link,
 228                      struct bpf_prog *prog_old,
 229                      u32 flags, u32 id_or_fd, u64 revision)
 230 {
 231         struct bpf_tuple rtuple, ntuple = {
 232                 .prog = prog_new,
 233                 .link = link,
 234         }, otuple = {
 235                 .prog = prog_old,
 236                 .link = link,
 237         };
 238         int ret, idx = -ERANGE, tidx;
 239
 240         if (revision && revision != bpf_mprog_revision(entry))
 241                 return -ESTALE;
 242         if (bpf_mprog_exists(entry, prog_new))
 243                 return -EEXIST;
 244         ret = bpf_mprog_tuple_relative(&rtuple, id_or_fd,
 245                                        flags & ~BPF_F_REPLACE,
 246                                        prog_new->type);
 247         if (ret)
 248                 return ret;
 249         if (flags & BPF_F_REPLACE) {
 250                 tidx = bpf_mprog_pos_exact(entry, &otuple);
 251                 if (tidx < 0) {
 252                         ret = tidx;
 253                         goto out;
 254                 }
 255                 idx = tidx;
 256         } else if (bpf_mprog_total(entry) == bpf_mprog_max()) {
 257                 ret = -ERANGE;
 258                 goto out;
 259         }
 260         if (flags & BPF_F_BEFORE) {
 261                 tidx = bpf_mprog_pos_before(entry, &rtuple);
 262                 if (tidx < -1 || (idx >= -1 && tidx != idx)) {
 263                         ret = tidx < -1 ? tidx : -ERANGE;
 264                         goto out;
 265                 }
 266                 idx = tidx;
 267         }
 268         if (flags & BPF_F_AFTER) {
 269                 tidx = bpf_mprog_pos_after(entry, &rtuple);
 270                 if (tidx < -1 || (idx >= -1 && tidx != idx)) {
 271                         ret = tidx < 0 ? tidx : -ERANGE;
 272                         goto out;
 273                 }
 274                 idx = tidx;
 275         }
 276         if (idx < -1) {
 277                 if (rtuple.prog || flags) {
 278                         ret = -EINVAL;
 279                         goto out;
 280                 }
 281                 idx = bpf_mprog_total(entry);
 282                 flags = BPF_F_AFTER;
 283         }
 284         if (idx >= bpf_mprog_max()) {
 285                 ret = -ERANGE;
 286                 goto out;
 287         }
 288         if (flags & BPF_F_REPLACE)
 289                 ret = bpf_mprog_replace(entry, entry_new, &ntuple, idx);
 290         else
 291                 ret = bpf_mprog_insert(entry, entry_new, &ntuple, idx, flags);
 292 out:
 293         bpf_mprog_tuple_put(&rtuple);
 294         return ret;
 295 }
 296
 297 static int bpf_mprog_fetch(struct bpf_mprog_entry *entry,
 298                            struct bpf_tuple *tuple, int idx)
 299 {
 300         int total = bpf_mprog_total(entry);
 301         struct bpf_mprog_cp *cp;
 302         struct bpf_mprog_fp *fp;
 303         struct bpf_prog *prog;
 304         struct bpf_link *link;
 305
 306         if (idx == -1)
 307                 idx = 0;
 308         else if (idx == total)
 309                 idx = total - 1;
 310         bpf_mprog_read(entry, idx, &fp, &cp);
 311         prog = READ_ONCE(fp->prog);
 312         link = cp->link;
 313         /* The deletion request can either be without filled tuple in which
 314          * case it gets populated here based on idx, or with filled tuple
 315          * where the only thing we end up doing is the WARN_ON_ONCE() assert.
 316          * If we hit a BPF link at the given index, it must not be removed
 317          * from opts path.
 318          */
 319         if (link && !tuple->link)
 320                 return -EBUSY;
 321         WARN_ON_ONCE(tuple->prog && tuple->prog != prog);
 322         WARN_ON_ONCE(tuple->link && tuple->link != link);
 323         tuple->prog = prog;
 324         tuple->link = link;
 325         return 0;
 326 }
 327
 328 int bpf_mprog_detach(struct bpf_mprog_entry *entry,
 329                      struct bpf_mprog_entry **entry_new,
 330                      struct bpf_prog *prog, struct bpf_link *link,
 331                      u32 flags, u32 id_or_fd, u64 revision)
 332 {
 333         struct bpf_tuple rtuple, dtuple = {
 334                 .prog = prog,
 335                 .link = link,
 336         };
 337         int ret, idx = -ERANGE, tidx;
 338
 339         if (flags & BPF_F_REPLACE)
 340                 return -EINVAL;
 341         if (revision && revision != bpf_mprog_revision(entry))
 342                 return -ESTALE;
 343         if (!bpf_mprog_total(entry))
 344                 return -ENOENT;
 345         ret = bpf_mprog_tuple_relative(&rtuple, id_or_fd, flags,
 346                                        prog ? prog->type :
 347                                        BPF_PROG_TYPE_UNSPEC);
 348         if (ret)
 349                 return ret;
 350         if (dtuple.prog) {
 351                 tidx = bpf_mprog_pos_exact(entry, &dtuple);
 352                 if (tidx < 0) {
 353                         ret = tidx;
 354                         goto out;
 355                 }
 356                 idx = tidx;
 357         }
 358         if (flags & BPF_F_BEFORE) {
 359                 tidx = bpf_mprog_pos_before(entry, &rtuple);
 360                 if (tidx < -1 || (idx >= -1 && tidx != idx)) {
 361                         ret = tidx < -1 ? tidx : -ERANGE;
 362                         goto out;
 363                 }
 364                 idx = tidx;
 365         }
 366         if (flags & BPF_F_AFTER) {
 367                 tidx = bpf_mprog_pos_after(entry, &rtuple);
 368                 if (tidx < -1 || (idx >= -1 && tidx != idx)) {
 369                         ret = tidx < 0 ? tidx : -ERANGE;
 370                         goto out;
 371                 }
 372                 idx = tidx;
 373         }
 374         if (idx < -1) {
 375                 if (rtuple.prog || flags) {
 376                         ret = -EINVAL;
 377                         goto out;
 378                 }
 379                 idx = bpf_mprog_total(entry);
 380                 flags = BPF_F_AFTER;
 381         }
 382         if (idx >= bpf_mprog_max()) {
 383                 ret = -ERANGE;
 384                 goto out;
 385         }
 386         ret = bpf_mprog_fetch(entry, &dtuple, idx);
 387         if (ret)
 388                 goto out;
 389         ret = bpf_mprog_delete(entry, entry_new, &dtuple, idx);
 390 out:
 391         bpf_mprog_tuple_put(&rtuple);
 392         return ret;
 393 }
 394
 395 int bpf_mprog_query(const union bpf_attr *attr, union bpf_attr __user *uattr,
 396                     struct bpf_mprog_entry *entry)
 397 {
 398         u32 __user *uprog_flags, *ulink_flags;
 399         u32 __user *uprog_id, *ulink_id;
 400         struct bpf_mprog_fp *fp;
 401         struct bpf_mprog_cp *cp;
 402         struct bpf_prog *prog;
 403         const u32 flags = 0;
 404         u32 id, count = 0;
 405         u64 revision = 1;
 406         int i, ret = 0;
 407
 408         if (attr->query.query_flags || attr->query.attach_flags)
 409                 return -EINVAL;
 410         if (entry) {
 411                 revision = bpf_mprog_revision(entry);
 412                 count = bpf_mprog_total(entry);
 413         }
 414         if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags)))
 415                 return -EFAULT;
 416         if (copy_to_user(&uattr->query.revision, &revision, sizeof(revision)))
 417                 return -EFAULT;
 418         if (copy_to_user(&uattr->query.count, &count, sizeof(count)))
 419                 return -EFAULT;
 420         uprog_id = u64_to_user_ptr(attr->query.prog_ids);
 421         uprog_flags = u64_to_user_ptr(attr->query.prog_attach_flags);
 422         ulink_id = u64_to_user_ptr(attr->query.link_ids);
 423         ulink_flags = u64_to_user_ptr(attr->query.link_attach_flags);
 424         if (attr->query.count == 0 || !uprog_id || !count)
 425                 return 0;
 426         if (attr->query.count < count) {
 427                 count = attr->query.count;
 428                 ret = -ENOSPC;
 429         }
 430         for (i = 0; i < bpf_mprog_max(); i++) {
 431                 bpf_mprog_read(entry, i, &fp, &cp);
 432                 prog = READ_ONCE(fp->prog);
 433                 if (!prog)
 434                         break;
 435                 id = prog->aux->id;
 436                 if (copy_to_user(uprog_id + i, &id, sizeof(id)))
 437                         return -EFAULT;
 438                 if (uprog_flags &&
 439                     copy_to_user(uprog_flags + i, &flags, sizeof(flags)))
 440                         return -EFAULT;
 441                 id = cp->link ? cp->link->id : 0;
 442                 if (ulink_id &&
 443                     copy_to_user(ulink_id + i, &id, sizeof(id)))
 444                         return -EFAULT;
 445                 if (ulink_flags &&
 446                     copy_to_user(ulink_flags + i, &flags, sizeof(flags)))
 447                         return -EFAULT;
 448                 if (i + 1 == count)
 449                         break;
 450         }
 451         return ret;
 452 }