include/linux/iov_iter.h

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 /* I/O iterator iteration building functions.
   3  *
   4  * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  */
   7
   8 #ifndef _LINUX_IOV_ITER_H
   9 #define _LINUX_IOV_ITER_H
  10
  11 #include <linux/uio.h>
  12 #include <linux/bvec.h>
  13 #include <linux/folio_queue.h>
  14
  15 typedef size_t (*iov_step_f)(void *iter_base, size_t progress, size_t len,
  16                              void *priv, void *priv2);
  17 typedef size_t (*iov_ustep_f)(void __user *iter_base, size_t progress, size_t len,
  18                               void *priv, void *priv2);
  19
  20 /*
  21  * Handle ITER_UBUF.
  22  */
  23 static __always_inline
  24 size_t iterate_ubuf(struct iov_iter *iter, size_t len, void *priv, void *priv2,
  25                     iov_ustep_f step)
  26 {
  27         void __user *base = iter->ubuf;
  28         size_t progress = 0, remain;
  29
  30         remain = step(base + iter->iov_offset, 0, len, priv, priv2);
  31         progress = len - remain;
  32         iter->iov_offset += progress;
  33         iter->count -= progress;
  34         return progress;
  35 }
  36
  37 /*
  38  * Handle ITER_IOVEC.
  39  */
  40 static __always_inline
  41 size_t iterate_iovec(struct iov_iter *iter, size_t len, void *priv, void *priv2,
  42                      iov_ustep_f step)
  43 {
  44         const struct iovec *p = iter->__iov;
  45         size_t progress = 0, skip = iter->iov_offset;
  46
  47         do {
  48                 size_t remain, consumed;
  49                 size_t part = min(len, p->iov_len - skip);
  50
  51                 if (likely(part)) {
  52                         remain = step(p->iov_base + skip, progress, part, priv, priv2);
  53                         consumed = part - remain;
  54                         progress += consumed;
  55                         skip += consumed;
  56                         len -= consumed;
  57                         if (skip < p->iov_len)
  58                                 break;
  59                 }
  60                 p++;
  61                 skip = 0;
  62         } while (len);
  63
  64         iter->nr_segs -= p - iter->__iov;
  65         iter->__iov = p;
  66         iter->iov_offset = skip;
  67         iter->count -= progress;
  68         return progress;
  69 }
  70
  71 /*
  72  * Handle ITER_KVEC.
  73  */
  74 static __always_inline
  75 size_t iterate_kvec(struct iov_iter *iter, size_t len, void *priv, void *priv2,
  76                     iov_step_f step)
  77 {
  78         const struct kvec *p = iter->kvec;
  79         size_t progress = 0, skip = iter->iov_offset;
  80
  81         do {
  82                 size_t remain, consumed;
  83                 size_t part = min(len, p->iov_len - skip);
  84
  85                 if (likely(part)) {
  86                         remain = step(p->iov_base + skip, progress, part, priv, priv2);
  87                         consumed = part - remain;
  88                         progress += consumed;
  89                         skip += consumed;
  90                         len -= consumed;
  91                         if (skip < p->iov_len)
  92                                 break;
  93                 }
  94                 p++;
  95                 skip = 0;
  96         } while (len);
  97
  98         iter->nr_segs -= p - iter->kvec;
  99         iter->kvec = p;
 100         iter->iov_offset = skip;
 101         iter->count -= progress;
 102         return progress;
 103 }
 104
 105 /*
 106  * Handle ITER_BVEC.
 107  */
 108 static __always_inline
 109 size_t iterate_bvec(struct iov_iter *iter, size_t len, void *priv, void *priv2,
 110                     iov_step_f step)
 111 {
 112         const struct bio_vec *p = iter->bvec;
 113         size_t progress = 0, skip = iter->iov_offset;
 114
 115         do {
 116                 size_t remain, consumed;
 117                 size_t offset = p->bv_offset + skip, part;
 118                 void *kaddr = kmap_local_page(p->bv_page + offset / PAGE_SIZE);
 119
 120                 part = min3(len,
 121                            (size_t)(p->bv_len - skip),
 122                            (size_t)(PAGE_SIZE - offset % PAGE_SIZE));
 123                 remain = step(kaddr + offset % PAGE_SIZE, progress, part, priv, priv2);
 124                 kunmap_local(kaddr);
 125                 consumed = part - remain;
 126                 len -= consumed;
 127                 progress += consumed;
 128                 skip += consumed;
 129                 if (skip >= p->bv_len) {
 130                         skip = 0;
 131                         p++;
 132                 }
 133                 if (remain)
 134                         break;
 135         } while (len);
 136
 137         iter->nr_segs -= p - iter->bvec;
 138         iter->bvec = p;
 139         iter->iov_offset = skip;
 140         iter->count -= progress;
 141         return progress;
 142 }
 143
 144 /*
 145  * Handle ITER_FOLIOQ.
 146  */
 147 static __always_inline
 148 size_t iterate_folioq(struct iov_iter *iter, size_t len, void *priv, void *priv2,
 149                       iov_step_f step)
 150 {
 151         const struct folio_queue *folioq = iter->folioq;
 152         unsigned int slot = iter->folioq_slot;
 153         size_t progress = 0, skip = iter->iov_offset;
 154
 155         if (slot == folioq_nr_slots(folioq)) {
 156                 /* The iterator may have been extended. */
 157                 folioq = folioq->next;
 158                 slot = 0;
 159         }
 160
 161         do {
 162                 struct folio *folio = folioq_folio(folioq, slot);
 163                 size_t part, remain, consumed;
 164                 size_t fsize;
 165                 void *base;
 166
 167                 if (!folio)
 168                         break;
 169
 170                 fsize = folioq_folio_size(folioq, slot);
 171                 base = kmap_local_folio(folio, skip);
 172                 part = umin(len, PAGE_SIZE - skip % PAGE_SIZE);
 173                 remain = step(base, progress, part, priv, priv2);
 174                 kunmap_local(base);
 175                 consumed = part - remain;
 176                 len -= consumed;
 177                 progress += consumed;
 178                 skip += consumed;
 179                 if (skip >= fsize) {
 180                         skip = 0;
 181                         slot++;
 182                         if (slot == folioq_nr_slots(folioq) && folioq->next) {
 183                                 folioq = folioq->next;
 184                                 slot = 0;
 185                         }
 186                 }
 187                 if (remain)
 188                         break;
 189         } while (len);
 190
 191         iter->folioq_slot = slot;
 192         iter->folioq = folioq;
 193         iter->iov_offset = skip;
 194         iter->count -= progress;
 195         return progress;
 196 }
 197
 198 /*
 199  * Handle ITER_XARRAY.
 200  */
 201 static __always_inline
 202 size_t iterate_xarray(struct iov_iter *iter, size_t len, void *priv, void *priv2,
 203                       iov_step_f step)
 204 {
 205         struct folio *folio;
 206         size_t progress = 0;
 207         loff_t start = iter->xarray_start + iter->iov_offset;
 208         pgoff_t index = start / PAGE_SIZE;
 209         XA_STATE(xas, iter->xarray, index);
 210
 211         rcu_read_lock();
 212         xas_for_each(&xas, folio, ULONG_MAX) {
 213                 size_t remain, consumed, offset, part, flen;
 214
 215                 if (xas_retry(&xas, folio))
 216                         continue;
 217                 if (WARN_ON(xa_is_value(folio)))
 218                         break;
 219                 if (WARN_ON(folio_test_hugetlb(folio)))
 220                         break;
 221
 222                 offset = offset_in_folio(folio, start + progress);
 223                 flen = min(folio_size(folio) - offset, len);
 224
 225                 while (flen) {
 226                         void *base = kmap_local_folio(folio, offset);
 227
 228                         part = min_t(size_t, flen,
 229                                      PAGE_SIZE - offset_in_page(offset));
 230                         remain = step(base, progress, part, priv, priv2);
 231                         kunmap_local(base);
 232
 233                         consumed = part - remain;
 234                         progress += consumed;
 235                         len -= consumed;
 236
 237                         if (remain || len == 0)
 238                                 goto out;
 239                         flen -= consumed;
 240                         offset += consumed;
 241                 }
 242         }
 243
 244 out:
 245         rcu_read_unlock();
 246         iter->iov_offset += progress;
 247         iter->count -= progress;
 248         return progress;
 249 }
 250
 251 /*
 252  * Handle ITER_DISCARD.
 253  */
 254 static __always_inline
 255 size_t iterate_discard(struct iov_iter *iter, size_t len, void *priv, void *priv2,
 256                       iov_step_f step)
 257 {
 258         size_t progress = len;
 259
 260         iter->count -= progress;
 261         return progress;
 262 }
 263
 264 /**
 265  * iterate_and_advance2 - Iterate over an iterator
 266  * @iter: The iterator to iterate over.
 267  * @len: The amount to iterate over.
 268  * @priv: Data for the step functions.
 269  * @priv2: More data for the step functions.
 270  * @ustep: Function for UBUF/IOVEC iterators; given __user addresses.
 271  * @step: Function for other iterators; given kernel addresses.
 272  *
 273  * Iterate over the next part of an iterator, up to the specified length.  The
 274  * buffer is presented in segments, which for kernel iteration are broken up by
 275  * physical pages and mapped, with the mapped address being presented.
 276  *
 277  * Two step functions, @step and @ustep, must be provided, one for handling
 278  * mapped kernel addresses and the other is given user addresses which have the
 279  * potential to fault since no pinning is performed.
 280  *
 281  * The step functions are passed the address and length of the segment, @priv,
 282  * @priv2 and the amount of data so far iterated over (which can, for example,
 283  * be added to @priv to point to the right part of a second buffer).  The step
 284  * functions should return the amount of the segment they didn't process (ie. 0
 285  * indicates complete processsing).
 286  *
 287  * This function returns the amount of data processed (ie. 0 means nothing was
 288  * processed and the value of @len means processes to completion).
 289  */
 290 static __always_inline
 291 size_t iterate_and_advance2(struct iov_iter *iter, size_t len, void *priv,
 292                             void *priv2, iov_ustep_f ustep, iov_step_f step)
 293 {
 294         if (unlikely(iter->count < len))
 295                 len = iter->count;
 296         if (unlikely(!len))
 297                 return 0;
 298
 299         if (likely(iter_is_ubuf(iter)))
 300                 return iterate_ubuf(iter, len, priv, priv2, ustep);
 301         if (likely(iter_is_iovec(iter)))
 302                 return iterate_iovec(iter, len, priv, priv2, ustep);
 303         if (iov_iter_is_bvec(iter))
 304                 return iterate_bvec(iter, len, priv, priv2, step);
 305         if (iov_iter_is_kvec(iter))
 306                 return iterate_kvec(iter, len, priv, priv2, step);
 307         if (iov_iter_is_folioq(iter))
 308                 return iterate_folioq(iter, len, priv, priv2, step);
 309         if (iov_iter_is_xarray(iter))
 310                 return iterate_xarray(iter, len, priv, priv2, step);
 311         return iterate_discard(iter, len, priv, priv2, step);
 312 }
 313
 314 /**
 315  * iterate_and_advance - Iterate over an iterator
 316  * @iter: The iterator to iterate over.
 317  * @len: The amount to iterate over.
 318  * @priv: Data for the step functions.
 319  * @ustep: Function for UBUF/IOVEC iterators; given __user addresses.
 320  * @step: Function for other iterators; given kernel addresses.
 321  *
 322  * As iterate_and_advance2(), but priv2 is always NULL.
 323  */
 324 static __always_inline
 325 size_t iterate_and_advance(struct iov_iter *iter, size_t len, void *priv,
 326                            iov_ustep_f ustep, iov_step_f step)
 327 {
 328         return iterate_and_advance2(iter, len, priv, NULL, ustep, step);
 329 }
 330
 331 /**
 332  * iterate_and_advance_kernel - Iterate over a kernel-internal iterator
 333  * @iter: The iterator to iterate over.
 334  * @len: The amount to iterate over.
 335  * @priv: Data for the step functions.
 336  * @priv2: More data for the step functions.
 337  * @step: Function for other iterators; given kernel addresses.
 338  *
 339  * Iterate over the next part of an iterator, up to the specified length.  The
 340  * buffer is presented in segments, which for kernel iteration are broken up by
 341  * physical pages and mapped, with the mapped address being presented.
 342  *
 343  * [!] Note This will only handle BVEC, KVEC, FOLIOQ, XARRAY and DISCARD-type
 344  * iterators; it will not handle UBUF or IOVEC-type iterators.
 345  *
 346  * A step functions, @step, must be provided, one for handling mapped kernel
 347  * addresses and the other is given user addresses which have the potential to
 348  * fault since no pinning is performed.
 349  *
 350  * The step functions are passed the address and length of the segment, @priv,
 351  * @priv2 and the amount of data so far iterated over (which can, for example,
 352  * be added to @priv to point to the right part of a second buffer).  The step
 353  * functions should return the amount of the segment they didn't process (ie. 0
 354  * indicates complete processsing).
 355  *
 356  * This function returns the amount of data processed (ie. 0 means nothing was
 357  * processed and the value of @len means processes to completion).
 358  */
 359 static __always_inline
 360 size_t iterate_and_advance_kernel(struct iov_iter *iter, size_t len, void *priv,
 361                                   void *priv2, iov_step_f step)
 362 {
 363         if (unlikely(iter->count < len))
 364                 len = iter->count;
 365         if (unlikely(!len))
 366                 return 0;
 367         if (iov_iter_is_bvec(iter))
 368                 return iterate_bvec(iter, len, priv, priv2, step);
 369         if (iov_iter_is_kvec(iter))
 370                 return iterate_kvec(iter, len, priv, priv2, step);
 371         if (iov_iter_is_folioq(iter))
 372                 return iterate_folioq(iter, len, priv, priv2, step);
 373         if (iov_iter_is_xarray(iter))
 374                 return iterate_xarray(iter, len, priv, priv2, step);
 375         return iterate_discard(iter, len, priv, priv2, step);
 376 }
 377
 378 #endif /* _LINUX_IOV_ITER_H */