include/linux/bvec.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * bvec iterator
   4  *
   5  * Copyright (C) 2001 Ming Lei <ming.lei@canonical.com>
   6  */
   7 #ifndef __LINUX_BVEC_H
   8 #define __LINUX_BVEC_H
   9
  10 #include <linux/highmem.h>
  11 #include <linux/bug.h>
  12 #include <linux/errno.h>
  13 #include <linux/limits.h>
  14 #include <linux/minmax.h>
  15 #include <linux/types.h>
  16
  17 struct page;
  18
  19 /**
  20  * struct bio_vec - a contiguous range of physical memory addresses
  21  * @bv_page:   First page associated with the address range.
  22  * @bv_len:    Number of bytes in the address range.
  23  * @bv_offset: Start of the address range relative to the start of @bv_page.
  24  *
  25  * The following holds for a bvec if n * PAGE_SIZE < bv_offset + bv_len:
  26  *
  27  *   nth_page(@bv_page, n) == @bv_page + n
  28  *
  29  * This holds because page_is_mergeable() checks the above property.
  30  */
  31 struct bio_vec {
  32         struct page     *bv_page;
  33         unsigned int    bv_len;
  34         unsigned int    bv_offset;
  35 };
  36
  37 /**
  38  * bvec_set_page - initialize a bvec based off a struct page
  39  * @bv:         bvec to initialize
  40  * @page:       page the bvec should point to
  41  * @len:        length of the bvec
  42  * @offset:     offset into the page
  43  */
  44 static inline void bvec_set_page(struct bio_vec *bv, struct page *page,
  45                 unsigned int len, unsigned int offset)
  46 {
  47         bv->bv_page = page;
  48         bv->bv_len = len;
  49         bv->bv_offset = offset;
  50 }
  51
  52 /**
  53  * bvec_set_folio - initialize a bvec based off a struct folio
  54  * @bv:         bvec to initialize
  55  * @folio:      folio the bvec should point to
  56  * @len:        length of the bvec
  57  * @offset:     offset into the folio
  58  */
  59 static inline void bvec_set_folio(struct bio_vec *bv, struct folio *folio,
  60                 unsigned int len, unsigned int offset)
  61 {
  62         bvec_set_page(bv, &folio->page, len, offset);
  63 }
  64
  65 /**
  66  * bvec_set_virt - initialize a bvec based on a virtual address
  67  * @bv:         bvec to initialize
  68  * @vaddr:      virtual address to set the bvec to
  69  * @len:        length of the bvec
  70  */
  71 static inline void bvec_set_virt(struct bio_vec *bv, void *vaddr,
  72                 unsigned int len)
  73 {
  74         bvec_set_page(bv, virt_to_page(vaddr), len, offset_in_page(vaddr));
  75 }
  76
  77 struct bvec_iter {
  78         sector_t                bi_sector;      /* device address in 512 byte
  79                                                    sectors */
  80         unsigned int            bi_size;        /* residual I/O count */
  81
  82         unsigned int            bi_idx;         /* current index into bvl_vec */
  83
  84         unsigned int            bi_bvec_done;   /* number of bytes completed in
  85                                                    current bvec */
  86 } __packed __aligned(4);
  87
  88 struct bvec_iter_all {
  89         struct bio_vec  bv;
  90         int             idx;
  91         unsigned        done;
  92 };
  93
  94 /*
  95  * various member access, note that bio_data should of course not be used
  96  * on highmem page vectors
  97  */
  98 #define __bvec_iter_bvec(bvec, iter)    (&(bvec)[(iter).bi_idx])
  99
 100 /* multi-page (mp_bvec) helpers */
 101 #define mp_bvec_iter_page(bvec, iter)                           \
 102         (__bvec_iter_bvec((bvec), (iter))->bv_page)
 103
 104 #define mp_bvec_iter_len(bvec, iter)                            \
 105         min((iter).bi_size,                                     \
 106             __bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done)
 107
 108 #define mp_bvec_iter_offset(bvec, iter)                         \
 109         (__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done)
 110
 111 #define mp_bvec_iter_page_idx(bvec, iter)                       \
 112         (mp_bvec_iter_offset((bvec), (iter)) / PAGE_SIZE)
 113
 114 #define mp_bvec_iter_bvec(bvec, iter)                           \
 115 ((struct bio_vec) {                                             \
 116         .bv_page        = mp_bvec_iter_page((bvec), (iter)),    \
 117         .bv_len         = mp_bvec_iter_len((bvec), (iter)),     \
 118         .bv_offset      = mp_bvec_iter_offset((bvec), (iter)),  \
 119 })
 120
 121 /* For building single-page bvec in flight */
 122  #define bvec_iter_offset(bvec, iter)                           \
 123         (mp_bvec_iter_offset((bvec), (iter)) % PAGE_SIZE)
 124
 125 #define bvec_iter_len(bvec, iter)                               \
 126         min_t(unsigned, mp_bvec_iter_len((bvec), (iter)),               \
 127               PAGE_SIZE - bvec_iter_offset((bvec), (iter)))
 128
 129 #define bvec_iter_page(bvec, iter)                              \
 130         (mp_bvec_iter_page((bvec), (iter)) +                    \
 131          mp_bvec_iter_page_idx((bvec), (iter)))
 132
 133 #define bvec_iter_bvec(bvec, iter)                              \
 134 ((struct bio_vec) {                                             \
 135         .bv_page        = bvec_iter_page((bvec), (iter)),       \
 136         .bv_len         = bvec_iter_len((bvec), (iter)),        \
 137         .bv_offset      = bvec_iter_offset((bvec), (iter)),     \
 138 })
 139
 140 static inline bool bvec_iter_advance(const struct bio_vec *bv,
 141                 struct bvec_iter *iter, unsigned bytes)
 142 {
 143         unsigned int idx = iter->bi_idx;
 144
 145         if (WARN_ONCE(bytes > iter->bi_size,
 146                      "Attempted to advance past end of bvec iter\n")) {
 147                 iter->bi_size = 0;
 148                 return false;
 149         }
 150
 151         iter->bi_size -= bytes;
 152         bytes += iter->bi_bvec_done;
 153
 154         while (bytes && bytes >= bv[idx].bv_len) {
 155                 bytes -= bv[idx].bv_len;
 156                 idx++;
 157         }
 158
 159         iter->bi_idx = idx;
 160         iter->bi_bvec_done = bytes;
 161         return true;
 162 }
 163
 164 /*
 165  * A simpler version of bvec_iter_advance(), @bytes should not span
 166  * across multiple bvec entries, i.e. bytes <= bv[i->bi_idx].bv_len
 167  */
 168 static inline void bvec_iter_advance_single(const struct bio_vec *bv,
 169                                 struct bvec_iter *iter, unsigned int bytes)
 170 {
 171         unsigned int done = iter->bi_bvec_done + bytes;
 172
 173         if (done == bv[iter->bi_idx].bv_len) {
 174                 done = 0;
 175                 iter->bi_idx++;
 176         }
 177         iter->bi_bvec_done = done;
 178         iter->bi_size -= bytes;
 179 }
 180
 181 #define for_each_bvec(bvl, bio_vec, iter, start)                        \
 182         for (iter = (start);                                            \
 183              (iter).bi_size &&                                          \
 184                 ((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \
 185              bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len))
 186
 187 /* for iterating one bio from start to end */
 188 #define BVEC_ITER_ALL_INIT (struct bvec_iter)                           \
 189 {                                                                       \
 190         .bi_sector      = 0,                                            \
 191         .bi_size        = UINT_MAX,                                     \
 192         .bi_idx         = 0,                                            \
 193         .bi_bvec_done   = 0,                                            \
 194 }
 195
 196 static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all)
 197 {
 198         iter_all->done = 0;
 199         iter_all->idx = 0;
 200
 201         return &iter_all->bv;
 202 }
 203
 204 static inline void bvec_advance(const struct bio_vec *bvec,
 205                                 struct bvec_iter_all *iter_all)
 206 {
 207         struct bio_vec *bv = &iter_all->bv;
 208
 209         if (iter_all->done) {
 210                 bv->bv_page++;
 211                 bv->bv_offset = 0;
 212         } else {
 213                 bv->bv_page = bvec->bv_page + (bvec->bv_offset >> PAGE_SHIFT);
 214                 bv->bv_offset = bvec->bv_offset & ~PAGE_MASK;
 215         }
 216         bv->bv_len = min_t(unsigned int, PAGE_SIZE - bv->bv_offset,
 217                            bvec->bv_len - iter_all->done);
 218         iter_all->done += bv->bv_len;
 219
 220         if (iter_all->done == bvec->bv_len) {
 221                 iter_all->idx++;
 222                 iter_all->done = 0;
 223         }
 224 }
 225
 226 /**
 227  * bvec_kmap_local - map a bvec into the kernel virtual address space
 228  * @bvec: bvec to map
 229  *
 230  * Must be called on single-page bvecs only.  Call kunmap_local on the returned
 231  * address to unmap.
 232  */
 233 static inline void *bvec_kmap_local(struct bio_vec *bvec)
 234 {
 235         return kmap_local_page(bvec->bv_page) + bvec->bv_offset;
 236 }
 237
 238 /**
 239  * memcpy_from_bvec - copy data from a bvec
 240  * @bvec: bvec to copy from
 241  *
 242  * Must be called on single-page bvecs only.
 243  */
 244 static inline void memcpy_from_bvec(char *to, struct bio_vec *bvec)
 245 {
 246         memcpy_from_page(to, bvec->bv_page, bvec->bv_offset, bvec->bv_len);
 247 }
 248
 249 /**
 250  * memcpy_to_bvec - copy data to a bvec
 251  * @bvec: bvec to copy to
 252  *
 253  * Must be called on single-page bvecs only.
 254  */
 255 static inline void memcpy_to_bvec(struct bio_vec *bvec, const char *from)
 256 {
 257         memcpy_to_page(bvec->bv_page, bvec->bv_offset, from, bvec->bv_len);
 258 }
 259
 260 /**
 261  * memzero_bvec - zero all data in a bvec
 262  * @bvec: bvec to zero
 263  *
 264  * Must be called on single-page bvecs only.
 265  */
 266 static inline void memzero_bvec(struct bio_vec *bvec)
 267 {
 268         memzero_page(bvec->bv_page, bvec->bv_offset, bvec->bv_len);
 269 }
 270
 271 /**
 272  * bvec_virt - return the virtual address for a bvec
 273  * @bvec: bvec to return the virtual address for
 274  *
 275  * Note: the caller must ensure that @bvec->bv_page is not a highmem page.
 276  */
 277 static inline void *bvec_virt(struct bio_vec *bvec)
 278 {
 279         WARN_ON_ONCE(PageHighMem(bvec->bv_page));
 280         return page_address(bvec->bv_page) + bvec->bv_offset;
 281 }
 282
 283 /**
 284  * bvec_phys - return the physical address for a bvec
 285  * @bvec: bvec to return the physical address for
 286  */
 287 static inline phys_addr_t bvec_phys(const struct bio_vec *bvec)
 288 {
 289         /*
 290          * Note this open codes page_to_phys because page_to_phys is defined in
 291          * <asm/io.h>, which we don't want to pull in here.  If it ever moves to
 292          * a sensible place we should start using it.
 293          */
 294         return PFN_PHYS(page_to_pfn(bvec->bv_page)) + bvec->bv_offset;
 295 }
 296
 297 #endif /* __LINUX_BVEC_H */