mm/iov_iter.c

   1 #include <linux/export.h>
   2 #include <linux/uio.h>
   3 #include <linux/pagemap.h>
   4 #include <linux/slab.h>
   5 #include <linux/vmalloc.h>
   6
   7 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
   8                          struct iov_iter *i)
   9 {
  10         size_t skip, copy, left, wanted;
  11         const struct iovec *iov;
  12         char __user *buf;
  13         void *kaddr, *from;
  14
  15         if (unlikely(bytes > i->count))
  16                 bytes = i->count;
  17
  18         if (unlikely(!bytes))
  19                 return 0;
  20
  21         wanted = bytes;
  22         iov = i->iov;
  23         skip = i->iov_offset;
  24         buf = iov->iov_base + skip;
  25         copy = min(bytes, iov->iov_len - skip);
  26
  27         if (!fault_in_pages_writeable(buf, copy)) {
  28                 kaddr = kmap_atomic(page);
  29                 from = kaddr + offset;
  30
  31                 /* first chunk, usually the only one */
  32                 left = __copy_to_user_inatomic(buf, from, copy);
  33                 copy -= left;
  34                 skip += copy;
  35                 from += copy;
  36                 bytes -= copy;
  37
  38                 while (unlikely(!left && bytes)) {
  39                         iov++;
  40                         buf = iov->iov_base;
  41                         copy = min(bytes, iov->iov_len);
  42                         left = __copy_to_user_inatomic(buf, from, copy);
  43                         copy -= left;
  44                         skip = copy;
  45                         from += copy;
  46                         bytes -= copy;
  47                 }
  48                 if (likely(!bytes)) {
  49                         kunmap_atomic(kaddr);
  50                         goto done;
  51                 }
  52                 offset = from - kaddr;
  53                 buf += copy;
  54                 kunmap_atomic(kaddr);
  55                 copy = min(bytes, iov->iov_len - skip);
  56         }
  57         /* Too bad - revert to non-atomic kmap */
  58         kaddr = kmap(page);
  59         from = kaddr + offset;
  60         left = __copy_to_user(buf, from, copy);
  61         copy -= left;
  62         skip += copy;
  63         from += copy;
  64         bytes -= copy;
  65         while (unlikely(!left && bytes)) {
  66                 iov++;
  67                 buf = iov->iov_base;
  68                 copy = min(bytes, iov->iov_len);
  69                 left = __copy_to_user(buf, from, copy);
  70                 copy -= left;
  71                 skip = copy;
  72                 from += copy;
  73                 bytes -= copy;
  74         }
  75         kunmap(page);
  76 done:
  77         if (skip == iov->iov_len) {
  78                 iov++;
  79                 skip = 0;
  80         }
  81         i->count -= wanted - bytes;
  82         i->nr_segs -= iov - i->iov;
  83         i->iov = iov;
  84         i->iov_offset = skip;
  85         return wanted - bytes;
  86 }
  87
  88 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
  89                          struct iov_iter *i)
  90 {
  91         size_t skip, copy, left, wanted;
  92         const struct iovec *iov;
  93         char __user *buf;
  94         void *kaddr, *to;
  95
  96         if (unlikely(bytes > i->count))
  97                 bytes = i->count;
  98
  99         if (unlikely(!bytes))
 100                 return 0;
 101
 102         wanted = bytes;
 103         iov = i->iov;
 104         skip = i->iov_offset;
 105         buf = iov->iov_base + skip;
 106         copy = min(bytes, iov->iov_len - skip);
 107
 108         if (!fault_in_pages_readable(buf, copy)) {
 109                 kaddr = kmap_atomic(page);
 110                 to = kaddr + offset;
 111
 112                 /* first chunk, usually the only one */
 113                 left = __copy_from_user_inatomic(to, buf, copy);
 114                 copy -= left;
 115                 skip += copy;
 116                 to += copy;
 117                 bytes -= copy;
 118
 119                 while (unlikely(!left && bytes)) {
 120                         iov++;
 121                         buf = iov->iov_base;
 122                         copy = min(bytes, iov->iov_len);
 123                         left = __copy_from_user_inatomic(to, buf, copy);
 124                         copy -= left;
 125                         skip = copy;
 126                         to += copy;
 127                         bytes -= copy;
 128                 }
 129                 if (likely(!bytes)) {
 130                         kunmap_atomic(kaddr);
 131                         goto done;
 132                 }
 133                 offset = to - kaddr;
 134                 buf += copy;
 135                 kunmap_atomic(kaddr);
 136                 copy = min(bytes, iov->iov_len - skip);
 137         }
 138         /* Too bad - revert to non-atomic kmap */
 139         kaddr = kmap(page);
 140         to = kaddr + offset;
 141         left = __copy_from_user(to, buf, copy);
 142         copy -= left;
 143         skip += copy;
 144         to += copy;
 145         bytes -= copy;
 146         while (unlikely(!left && bytes)) {
 147                 iov++;
 148                 buf = iov->iov_base;
 149                 copy = min(bytes, iov->iov_len);
 150                 left = __copy_from_user(to, buf, copy);
 151                 copy -= left;
 152                 skip = copy;
 153                 to += copy;
 154                 bytes -= copy;
 155         }
 156         kunmap(page);
 157 done:
 158         if (skip == iov->iov_len) {
 159                 iov++;
 160                 skip = 0;
 161         }
 162         i->count -= wanted - bytes;
 163         i->nr_segs -= iov - i->iov;
 164         i->iov = iov;
 165         i->iov_offset = skip;
 166         return wanted - bytes;
 167 }
 168
 169 static size_t __iovec_copy_from_user_inatomic(char *vaddr,
 170                         const struct iovec *iov, size_t base, size_t bytes)
 171 {
 172         size_t copied = 0, left = 0;
 173
 174         while (bytes) {
 175                 char __user *buf = iov->iov_base + base;
 176                 int copy = min(bytes, iov->iov_len - base);
 177
 178                 base = 0;
 179                 left = __copy_from_user_inatomic(vaddr, buf, copy);
 180                 copied += copy;
 181                 bytes -= copy;
 182                 vaddr += copy;
 183                 iov++;
 184
 185                 if (unlikely(left))
 186                         break;
 187         }
 188         return copied - left;
 189 }
 190
 191 /*
 192  * Copy as much as we can into the page and return the number of bytes which
 193  * were successfully copied.  If a fault is encountered then return the number of
 194  * bytes which were copied.
 195  */
 196 static size_t copy_from_user_atomic_iovec(struct page *page,
 197                 struct iov_iter *i, unsigned long offset, size_t bytes)
 198 {
 199         char *kaddr;
 200         size_t copied;
 201
 202         kaddr = kmap_atomic(page);
 203         if (likely(i->nr_segs == 1)) {
 204                 int left;
 205                 char __user *buf = i->iov->iov_base + i->iov_offset;
 206                 left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
 207                 copied = bytes - left;
 208         } else {
 209                 copied = __iovec_copy_from_user_inatomic(kaddr + offset,
 210                                                 i->iov, i->iov_offset, bytes);
 211         }
 212         kunmap_atomic(kaddr);
 213
 214         return copied;
 215 }
 216
 217 static void advance_iovec(struct iov_iter *i, size_t bytes)
 218 {
 219         BUG_ON(i->count < bytes);
 220
 221         if (likely(i->nr_segs == 1)) {
 222                 i->iov_offset += bytes;
 223                 i->count -= bytes;
 224         } else {
 225                 const struct iovec *iov = i->iov;
 226                 size_t base = i->iov_offset;
 227                 unsigned long nr_segs = i->nr_segs;
 228
 229                 /*
 230                  * The !iov->iov_len check ensures we skip over unlikely
 231                  * zero-length segments (without overruning the iovec).
 232                  */
 233                 while (bytes || unlikely(i->count && !iov->iov_len)) {
 234                         int copy;
 235
 236                         copy = min(bytes, iov->iov_len - base);
 237                         BUG_ON(!i->count || i->count < copy);
 238                         i->count -= copy;
 239                         bytes -= copy;
 240                         base += copy;
 241                         if (iov->iov_len == base) {
 242                                 iov++;
 243                                 nr_segs--;
 244                                 base = 0;
 245                         }
 246                 }
 247                 i->iov = iov;
 248                 i->iov_offset = base;
 249                 i->nr_segs = nr_segs;
 250         }
 251 }
 252
 253 /*
 254  * Fault in the first iovec of the given iov_iter, to a maximum length
 255  * of bytes. Returns 0 on success, or non-zero if the memory could not be
 256  * accessed (ie. because it is an invalid address).
 257  *
 258  * writev-intensive code may want this to prefault several iovecs -- that
 259  * would be possible (callers must not rely on the fact that _only_ the
 260  * first iovec will be faulted with the current implementation).
 261  */
 262 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
 263 {
 264         if (!(i->type & ITER_BVEC)) {
 265                 char __user *buf = i->iov->iov_base + i->iov_offset;
 266                 bytes = min(bytes, i->iov->iov_len - i->iov_offset);
 267                 return fault_in_pages_readable(buf, bytes);
 268         }
 269         return 0;
 270 }
 271 EXPORT_SYMBOL(iov_iter_fault_in_readable);
 272
 273 static unsigned long alignment_iovec(const struct iov_iter *i)
 274 {
 275         const struct iovec *iov = i->iov;
 276         unsigned long res;
 277         size_t size = i->count;
 278         size_t n;
 279
 280         if (!size)
 281                 return 0;
 282
 283         res = (unsigned long)iov->iov_base + i->iov_offset;
 284         n = iov->iov_len - i->iov_offset;
 285         if (n >= size)
 286                 return res | size;
 287         size -= n;
 288         res |= n;
 289         while (size > (++iov)->iov_len) {
 290                 res |= (unsigned long)iov->iov_base | iov->iov_len;
 291                 size -= iov->iov_len;
 292         }
 293         res |= (unsigned long)iov->iov_base | size;
 294         return res;
 295 }
 296
 297 void iov_iter_init(struct iov_iter *i, int direction,
 298                         const struct iovec *iov, unsigned long nr_segs,
 299                         size_t count)
 300 {
 301         /* It will get better.  Eventually... */
 302         if (segment_eq(get_fs(), KERNEL_DS))
 303                 direction |= ITER_KVEC;
 304         i->type = direction;
 305         i->iov = iov;
 306         i->nr_segs = nr_segs;
 307         i->iov_offset = 0;
 308         i->count = count;
 309 }
 310 EXPORT_SYMBOL(iov_iter_init);
 311
 312 static ssize_t get_pages_iovec(struct iov_iter *i,
 313                    struct page **pages, size_t maxsize, unsigned maxpages,
 314                    size_t *start)
 315 {
 316         size_t offset = i->iov_offset;
 317         const struct iovec *iov = i->iov;
 318         size_t len;
 319         unsigned long addr;
 320         int n;
 321         int res;
 322
 323         len = iov->iov_len - offset;
 324         if (len > i->count)
 325                 len = i->count;
 326         if (len > maxsize)
 327                 len = maxsize;
 328         addr = (unsigned long)iov->iov_base + offset;
 329         len += *start = addr & (PAGE_SIZE - 1);
 330         if (len > maxpages * PAGE_SIZE)
 331                 len = maxpages * PAGE_SIZE;
 332         addr &= ~(PAGE_SIZE - 1);
 333         n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
 334         res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
 335         if (unlikely(res < 0))
 336                 return res;
 337         return (res == n ? len : res * PAGE_SIZE) - *start;
 338 }
 339
 340 static ssize_t get_pages_alloc_iovec(struct iov_iter *i,
 341                    struct page ***pages, size_t maxsize,
 342                    size_t *start)
 343 {
 344         size_t offset = i->iov_offset;
 345         const struct iovec *iov = i->iov;
 346         size_t len;
 347         unsigned long addr;
 348         void *p;
 349         int n;
 350         int res;
 351
 352         len = iov->iov_len - offset;
 353         if (len > i->count)
 354                 len = i->count;
 355         if (len > maxsize)
 356                 len = maxsize;
 357         addr = (unsigned long)iov->iov_base + offset;
 358         len += *start = addr & (PAGE_SIZE - 1);
 359         addr &= ~(PAGE_SIZE - 1);
 360         n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
 361
 362         p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
 363         if (!p)
 364                 p = vmalloc(n * sizeof(struct page *));
 365         if (!p)
 366                 return -ENOMEM;
 367
 368         res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
 369         if (unlikely(res < 0)) {
 370                 kvfree(p);
 371                 return res;
 372         }
 373         *pages = p;
 374         return (res == n ? len : res * PAGE_SIZE) - *start;
 375 }
 376
 377 static int iov_iter_npages_iovec(const struct iov_iter *i, int maxpages)
 378 {
 379         size_t offset = i->iov_offset;
 380         size_t size = i->count;
 381         const struct iovec *iov = i->iov;
 382         int npages = 0;
 383         int n;
 384
 385         for (n = 0; size && n < i->nr_segs; n++, iov++) {
 386                 unsigned long addr = (unsigned long)iov->iov_base + offset;
 387                 size_t len = iov->iov_len - offset;
 388                 offset = 0;
 389                 if (unlikely(!len))     /* empty segment */
 390                         continue;
 391                 if (len > size)
 392                         len = size;
 393                 npages += (addr + len + PAGE_SIZE - 1) / PAGE_SIZE
 394                           - addr / PAGE_SIZE;
 395                 if (npages >= maxpages) /* don't bother going further */
 396                         return maxpages;
 397                 size -= len;
 398                 offset = 0;
 399         }
 400         return min(npages, maxpages);
 401 }
 402
 403 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
 404 {
 405         char *from = kmap_atomic(page);
 406         memcpy(to, from + offset, len);
 407         kunmap_atomic(from);
 408 }
 409
 410 static void memcpy_to_page(struct page *page, size_t offset, char *from, size_t len)
 411 {
 412         char *to = kmap_atomic(page);
 413         memcpy(to + offset, from, len);
 414         kunmap_atomic(to);
 415 }
 416
 417 static size_t copy_page_to_iter_bvec(struct page *page, size_t offset, size_t bytes,
 418                          struct iov_iter *i)
 419 {
 420         size_t skip, copy, wanted;
 421         const struct bio_vec *bvec;
 422         void *kaddr, *from;
 423
 424         if (unlikely(bytes > i->count))
 425                 bytes = i->count;
 426
 427         if (unlikely(!bytes))
 428                 return 0;
 429
 430         wanted = bytes;
 431         bvec = i->bvec;
 432         skip = i->iov_offset;
 433         copy = min_t(size_t, bytes, bvec->bv_len - skip);
 434
 435         kaddr = kmap_atomic(page);
 436         from = kaddr + offset;
 437         memcpy_to_page(bvec->bv_page, skip + bvec->bv_offset, from, copy);
 438         skip += copy;
 439         from += copy;
 440         bytes -= copy;
 441         while (bytes) {
 442                 bvec++;
 443                 copy = min(bytes, (size_t)bvec->bv_len);
 444                 memcpy_to_page(bvec->bv_page, bvec->bv_offset, from, copy);
 445                 skip = copy;
 446                 from += copy;
 447                 bytes -= copy;
 448         }
 449         kunmap_atomic(kaddr);
 450         if (skip == bvec->bv_len) {
 451                 bvec++;
 452                 skip = 0;
 453         }
 454         i->count -= wanted - bytes;
 455         i->nr_segs -= bvec - i->bvec;
 456         i->bvec = bvec;
 457         i->iov_offset = skip;
 458         return wanted - bytes;
 459 }
 460
 461 static size_t copy_page_from_iter_bvec(struct page *page, size_t offset, size_t bytes,
 462                          struct iov_iter *i)
 463 {
 464         size_t skip, copy, wanted;
 465         const struct bio_vec *bvec;
 466         void *kaddr, *to;
 467
 468         if (unlikely(bytes > i->count))
 469                 bytes = i->count;
 470
 471         if (unlikely(!bytes))
 472                 return 0;
 473
 474         wanted = bytes;
 475         bvec = i->bvec;
 476         skip = i->iov_offset;
 477
 478         kaddr = kmap_atomic(page);
 479
 480         to = kaddr + offset;
 481
 482         copy = min(bytes, bvec->bv_len - skip);
 483
 484         memcpy_from_page(to, bvec->bv_page, bvec->bv_offset + skip, copy);
 485
 486         to += copy;
 487         skip += copy;
 488         bytes -= copy;
 489
 490         while (bytes) {
 491                 bvec++;
 492                 copy = min(bytes, (size_t)bvec->bv_len);
 493                 memcpy_from_page(to, bvec->bv_page, bvec->bv_offset, copy);
 494                 skip = copy;
 495                 to += copy;
 496                 bytes -= copy;
 497         }
 498         kunmap_atomic(kaddr);
 499         if (skip == bvec->bv_len) {
 500                 bvec++;
 501                 skip = 0;
 502         }
 503         i->count -= wanted;
 504         i->nr_segs -= bvec - i->bvec;
 505         i->bvec = bvec;
 506         i->iov_offset = skip;
 507         return wanted;
 508 }
 509
 510 static size_t copy_from_user_bvec(struct page *page,
 511                 struct iov_iter *i, unsigned long offset, size_t bytes)
 512 {
 513         char *kaddr;
 514         size_t left;
 515         const struct bio_vec *bvec;
 516         size_t base = i->iov_offset;
 517
 518         kaddr = kmap_atomic(page);
 519         for (left = bytes, bvec = i->bvec; left; bvec++, base = 0) {
 520                 size_t copy = min(left, bvec->bv_len - base);
 521                 if (!bvec->bv_len)
 522                         continue;
 523                 memcpy_from_page(kaddr + offset, bvec->bv_page,
 524                                  bvec->bv_offset + base, copy);
 525                 offset += copy;
 526                 left -= copy;
 527         }
 528         kunmap_atomic(kaddr);
 529         return bytes;
 530 }
 531
 532 static void advance_bvec(struct iov_iter *i, size_t bytes)
 533 {
 534         BUG_ON(i->count < bytes);
 535
 536         if (likely(i->nr_segs == 1)) {
 537                 i->iov_offset += bytes;
 538                 i->count -= bytes;
 539         } else {
 540                 const struct bio_vec *bvec = i->bvec;
 541                 size_t base = i->iov_offset;
 542                 unsigned long nr_segs = i->nr_segs;
 543
 544                 /*
 545                  * The !iov->iov_len check ensures we skip over unlikely
 546                  * zero-length segments (without overruning the iovec).
 547                  */
 548                 while (bytes || unlikely(i->count && !bvec->bv_len)) {
 549                         int copy;
 550
 551                         copy = min(bytes, bvec->bv_len - base);
 552                         BUG_ON(!i->count || i->count < copy);
 553                         i->count -= copy;
 554                         bytes -= copy;
 555                         base += copy;
 556                         if (bvec->bv_len == base) {
 557                                 bvec++;
 558                                 nr_segs--;
 559                                 base = 0;
 560                         }
 561                 }
 562                 i->bvec = bvec;
 563                 i->iov_offset = base;
 564                 i->nr_segs = nr_segs;
 565         }
 566 }
 567
 568 static unsigned long alignment_bvec(const struct iov_iter *i)
 569 {
 570         const struct bio_vec *bvec = i->bvec;
 571         unsigned long res;
 572         size_t size = i->count;
 573         size_t n;
 574
 575         if (!size)
 576                 return 0;
 577
 578         res = bvec->bv_offset + i->iov_offset;
 579         n = bvec->bv_len - i->iov_offset;
 580         if (n >= size)
 581                 return res | size;
 582         size -= n;
 583         res |= n;
 584         while (size > (++bvec)->bv_len) {
 585                 res |= bvec->bv_offset | bvec->bv_len;
 586                 size -= bvec->bv_len;
 587         }
 588         res |= bvec->bv_offset | size;
 589         return res;
 590 }
 591
 592 static ssize_t get_pages_bvec(struct iov_iter *i,
 593                    struct page **pages, size_t maxsize, unsigned maxpages,
 594                    size_t *start)
 595 {
 596         const struct bio_vec *bvec = i->bvec;
 597         size_t len = bvec->bv_len - i->iov_offset;
 598         if (len > i->count)
 599                 len = i->count;
 600         if (len > maxsize)
 601                 len = maxsize;
 602         /* can't be more than PAGE_SIZE */
 603         *start = bvec->bv_offset + i->iov_offset;
 604
 605         get_page(*pages = bvec->bv_page);
 606
 607         return len;
 608 }
 609
 610 static ssize_t get_pages_alloc_bvec(struct iov_iter *i,
 611                    struct page ***pages, size_t maxsize,
 612                    size_t *start)
 613 {
 614         const struct bio_vec *bvec = i->bvec;
 615         size_t len = bvec->bv_len - i->iov_offset;
 616         if (len > i->count)
 617                 len = i->count;
 618         if (len > maxsize)
 619                 len = maxsize;
 620         *start = bvec->bv_offset + i->iov_offset;
 621
 622         *pages = kmalloc(sizeof(struct page *), GFP_KERNEL);
 623         if (!*pages)
 624                 return -ENOMEM;
 625
 626         get_page(**pages = bvec->bv_page);
 627
 628         return len;
 629 }
 630
 631 static int iov_iter_npages_bvec(const struct iov_iter *i, int maxpages)
 632 {
 633         size_t offset = i->iov_offset;
 634         size_t size = i->count;
 635         const struct bio_vec *bvec = i->bvec;
 636         int npages = 0;
 637         int n;
 638
 639         for (n = 0; size && n < i->nr_segs; n++, bvec++) {
 640                 size_t len = bvec->bv_len - offset;
 641                 offset = 0;
 642                 if (unlikely(!len))     /* empty segment */
 643                         continue;
 644                 if (len > size)
 645                         len = size;
 646                 npages++;
 647                 if (npages >= maxpages) /* don't bother going further */
 648                         return maxpages;
 649                 size -= len;
 650                 offset = 0;
 651         }
 652         return min(npages, maxpages);
 653 }
 654
 655 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
 656                          struct iov_iter *i)
 657 {
 658         if (i->type & ITER_BVEC)
 659                 return copy_page_to_iter_bvec(page, offset, bytes, i);
 660         else
 661                 return copy_page_to_iter_iovec(page, offset, bytes, i);
 662 }
 663 EXPORT_SYMBOL(copy_page_to_iter);
 664
 665 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
 666                          struct iov_iter *i)
 667 {
 668         if (i->type & ITER_BVEC)
 669                 return copy_page_from_iter_bvec(page, offset, bytes, i);
 670         else
 671                 return copy_page_from_iter_iovec(page, offset, bytes, i);
 672 }
 673 EXPORT_SYMBOL(copy_page_from_iter);
 674
 675 size_t iov_iter_copy_from_user_atomic(struct page *page,
 676                 struct iov_iter *i, unsigned long offset, size_t bytes)
 677 {
 678         if (i->type & ITER_BVEC)
 679                 return copy_from_user_bvec(page, i, offset, bytes);
 680         else
 681                 return copy_from_user_atomic_iovec(page, i, offset, bytes);
 682 }
 683 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
 684
 685 void iov_iter_advance(struct iov_iter *i, size_t size)
 686 {
 687         if (i->type & ITER_BVEC)
 688                 advance_bvec(i, size);
 689         else
 690                 advance_iovec(i, size);
 691 }
 692 EXPORT_SYMBOL(iov_iter_advance);
 693
 694 /*
 695  * Return the count of just the current iov_iter segment.
 696  */
 697 size_t iov_iter_single_seg_count(const struct iov_iter *i)
 698 {
 699         if (i->nr_segs == 1)
 700                 return i->count;
 701         else if (i->type & ITER_BVEC)
 702                 return min(i->count, i->iov->iov_len - i->iov_offset);
 703         else
 704                 return min(i->count, i->bvec->bv_len - i->iov_offset);
 705 }
 706 EXPORT_SYMBOL(iov_iter_single_seg_count);
 707
 708 unsigned long iov_iter_alignment(const struct iov_iter *i)
 709 {
 710         if (i->type & ITER_BVEC)
 711                 return alignment_bvec(i);
 712         else
 713                 return alignment_iovec(i);
 714 }
 715 EXPORT_SYMBOL(iov_iter_alignment);
 716
 717 ssize_t iov_iter_get_pages(struct iov_iter *i,
 718                    struct page **pages, size_t maxsize, unsigned maxpages,
 719                    size_t *start)
 720 {
 721         if (i->type & ITER_BVEC)
 722                 return get_pages_bvec(i, pages, maxsize, maxpages, start);
 723         else
 724                 return get_pages_iovec(i, pages, maxsize, maxpages, start);
 725 }
 726 EXPORT_SYMBOL(iov_iter_get_pages);
 727
 728 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
 729                    struct page ***pages, size_t maxsize,
 730                    size_t *start)
 731 {
 732         if (i->type & ITER_BVEC)
 733                 return get_pages_alloc_bvec(i, pages, maxsize, start);
 734         else
 735                 return get_pages_alloc_iovec(i, pages, maxsize, start);
 736 }
 737 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
 738
 739 int iov_iter_npages(const struct iov_iter *i, int maxpages)
 740 {
 741         if (i->type & ITER_BVEC)
 742                 return iov_iter_npages_bvec(i, maxpages);
 743         else
 744                 return iov_iter_npages_iovec(i, maxpages);
 745 }
 746 EXPORT_SYMBOL(iov_iter_npages);