io_uring/notif.c

   1 #include <linux/kernel.h>
   2 #include <linux/errno.h>
   3 #include <linux/file.h>
   4 #include <linux/slab.h>
   5 #include <linux/net.h>
   6 #include <linux/io_uring.h>
   7
   8 #include "io_uring.h"
   9 #include "notif.h"
  10 #include "rsrc.h"
  11
  12 static const struct ubuf_info_ops io_ubuf_ops;
  13
  14 static void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts)
  15 {
  16         struct io_notif_data *nd = io_notif_to_data(notif);
  17
  18         do {
  19                 notif = cmd_to_io_kiocb(nd);
  20
  21                 lockdep_assert(refcount_read(&nd->uarg.refcnt) == 0);
  22
  23                 if (unlikely(nd->zc_report) && (nd->zc_copied || !nd->zc_used))
  24                         notif->cqe.res |= IORING_NOTIF_USAGE_ZC_COPIED;
  25
  26                 if (nd->account_pages && notif->ctx->user) {
  27                         __io_unaccount_mem(notif->ctx->user, nd->account_pages);
  28                         nd->account_pages = 0;
  29                 }
  30
  31                 nd = nd->next;
  32                 io_req_task_complete(notif, ts);
  33         } while (nd);
  34 }
  35
  36 void io_tx_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg,
  37                          bool success)
  38 {
  39         struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);
  40         struct io_kiocb *notif = cmd_to_io_kiocb(nd);
  41         unsigned tw_flags;
  42
  43         if (nd->zc_report) {
  44                 if (success && !nd->zc_used && skb)
  45                         WRITE_ONCE(nd->zc_used, true);
  46                 else if (!success && !nd->zc_copied)
  47                         WRITE_ONCE(nd->zc_copied, true);
  48         }
  49
  50         if (!refcount_dec_and_test(&uarg->refcnt))
  51                 return;
  52
  53         if (nd->head != nd) {
  54                 io_tx_ubuf_complete(skb, &nd->head->uarg, success);
  55                 return;
  56         }
  57
  58         tw_flags = nd->next ? 0 : IOU_F_TWQ_LAZY_WAKE;
  59         notif->io_task_work.func = io_notif_tw_complete;
  60         __io_req_task_work_add(notif, tw_flags);
  61 }
  62
  63 static int io_link_skb(struct sk_buff *skb, struct ubuf_info *uarg)
  64 {
  65         struct io_notif_data *nd, *prev_nd;
  66         struct io_kiocb *prev_notif, *notif;
  67         struct ubuf_info *prev_uarg = skb_zcopy(skb);
  68
  69         nd = container_of(uarg, struct io_notif_data, uarg);
  70         notif = cmd_to_io_kiocb(nd);
  71
  72         if (!prev_uarg) {
  73                 net_zcopy_get(&nd->uarg);
  74                 skb_zcopy_init(skb, &nd->uarg);
  75                 return 0;
  76         }
  77         /* handle it separately as we can't link a notif to itself */
  78         if (unlikely(prev_uarg == &nd->uarg))
  79                 return 0;
  80         /* we can't join two links together, just request a fresh skb */
  81         if (unlikely(nd->head != nd || nd->next))
  82                 return -EEXIST;
  83         /* don't mix zc providers */
  84         if (unlikely(prev_uarg->ops != &io_ubuf_ops))
  85                 return -EEXIST;
  86
  87         prev_nd = container_of(prev_uarg, struct io_notif_data, uarg);
  88         prev_notif = cmd_to_io_kiocb(nd);
  89
  90         /* make sure all noifications can be finished in the same task_work */
  91         if (unlikely(notif->ctx != prev_notif->ctx ||
  92                      notif->tctx != prev_notif->tctx))
  93                 return -EEXIST;
  94
  95         nd->head = prev_nd->head;
  96         nd->next = prev_nd->next;
  97         prev_nd->next = nd;
  98         net_zcopy_get(&nd->head->uarg);
  99         return 0;
 100 }
 101
 102 static const struct ubuf_info_ops io_ubuf_ops = {
 103         .complete = io_tx_ubuf_complete,
 104         .link_skb = io_link_skb,
 105 };
 106
 107 struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
 108         __must_hold(&ctx->uring_lock)
 109 {
 110         struct io_kiocb *notif;
 111         struct io_notif_data *nd;
 112
 113         if (unlikely(!io_alloc_req(ctx, &notif)))
 114                 return NULL;
 115         notif->opcode = IORING_OP_NOP;
 116         notif->flags = 0;
 117         notif->file = NULL;
 118         notif->tctx = current->io_uring;
 119         io_get_task_refs(1);
 120         notif->file_node = NULL;
 121         notif->buf_node = NULL;
 122
 123         nd = io_notif_to_data(notif);
 124         nd->zc_report = false;
 125         nd->account_pages = 0;
 126         nd->next = NULL;
 127         nd->head = nd;
 128
 129         nd->uarg.flags = IO_NOTIF_UBUF_FLAGS;
 130         nd->uarg.ops = &io_ubuf_ops;
 131         refcount_set(&nd->uarg.refcnt, 1);
 132         return notif;
 133 }