fs/read_write.c

   1 /*
   2  *  linux/fs/read_write.c
   3  *
   4  *  Copyright (C) 1991, 1992  Linus Torvalds
   5  */
   6
   7 #include <linux/slab.h>
   8 #include <linux/stat.h>
   9 #include <linux/fcntl.h>
  10 #include <linux/file.h>
  11 #include <linux/uio.h>
  12 #include <linux/smp_lock.h>
  13 #include <linux/fsnotify.h>
  14 #include <linux/security.h>
  15 #include <linux/module.h>
  16 #include <linux/syscalls.h>
  17 #include <linux/pagemap.h>
  18 #include <linux/splice.h>
  19 #include "read_write.h"
  20
  21 #include <asm/uaccess.h>
  22 #include <asm/unistd.h>
  23
  24 const struct file_operations generic_ro_fops = {
  25         .llseek         = generic_file_llseek,
  26         .read           = do_sync_read,
  27         .aio_read       = generic_file_aio_read,
  28         .mmap           = generic_file_readonly_mmap,
  29         .splice_read    = generic_file_splice_read,
  30 };
  31
  32 EXPORT_SYMBOL(generic_ro_fops);
  33
  34 loff_t generic_file_llseek(struct file *file, loff_t offset, int origin)
  35 {
  36         long long retval;
  37         struct inode *inode = file->f_mapping->host;
  38
  39         mutex_lock(&inode->i_mutex);
  40         switch (origin) {
  41                 case SEEK_END:
  42                         offset += inode->i_size;
  43                         break;
  44                 case SEEK_CUR:
  45                         offset += file->f_pos;
  46         }
  47         retval = -EINVAL;
  48         if (offset>=0 && offset<=inode->i_sb->s_maxbytes) {
  49                 if (offset != file->f_pos) {
  50                         file->f_pos = offset;
  51                         file->f_version = 0;
  52                 }
  53                 retval = offset;
  54         }
  55         mutex_unlock(&inode->i_mutex);
  56         return retval;
  57 }
  58
  59 EXPORT_SYMBOL(generic_file_llseek);
  60
  61 loff_t remote_llseek(struct file *file, loff_t offset, int origin)
  62 {
  63         long long retval;
  64
  65         lock_kernel();
  66         switch (origin) {
  67                 case SEEK_END:
  68                         offset += i_size_read(file->f_path.dentry->d_inode);
  69                         break;
  70                 case SEEK_CUR:
  71                         offset += file->f_pos;
  72         }
  73         retval = -EINVAL;
  74         if (offset>=0 && offset<=file->f_path.dentry->d_inode->i_sb->s_maxbytes) {
  75                 if (offset != file->f_pos) {
  76                         file->f_pos = offset;
  77                         file->f_version = 0;
  78                 }
  79                 retval = offset;
  80         }
  81         unlock_kernel();
  82         return retval;
  83 }
  84 EXPORT_SYMBOL(remote_llseek);
  85
  86 loff_t no_llseek(struct file *file, loff_t offset, int origin)
  87 {
  88         return -ESPIPE;
  89 }
  90 EXPORT_SYMBOL(no_llseek);
  91
  92 loff_t default_llseek(struct file *file, loff_t offset, int origin)
  93 {
  94         long long retval;
  95
  96         lock_kernel();
  97         switch (origin) {
  98                 case SEEK_END:
  99                         offset += i_size_read(file->f_path.dentry->d_inode);
 100                         break;
 101                 case SEEK_CUR:
 102                         offset += file->f_pos;
 103         }
 104         retval = -EINVAL;
 105         if (offset >= 0) {
 106                 if (offset != file->f_pos) {
 107                         file->f_pos = offset;
 108                         file->f_version = 0;
 109                 }
 110                 retval = offset;
 111         }
 112         unlock_kernel();
 113         return retval;
 114 }
 115 EXPORT_SYMBOL(default_llseek);
 116
 117 loff_t vfs_llseek(struct file *file, loff_t offset, int origin)
 118 {
 119         loff_t (*fn)(struct file *, loff_t, int);
 120
 121         fn = no_llseek;
 122         if (file->f_mode & FMODE_LSEEK) {
 123                 fn = default_llseek;
 124                 if (file->f_op && file->f_op->llseek)
 125                         fn = file->f_op->llseek;
 126         }
 127         return fn(file, offset, origin);
 128 }
 129 EXPORT_SYMBOL(vfs_llseek);
 130
 131 asmlinkage off_t sys_lseek(unsigned int fd, off_t offset, unsigned int origin)
 132 {
 133         off_t retval;
 134         struct file * file;
 135         int fput_needed;
 136
 137         retval = -EBADF;
 138         file = fget_light(fd, &fput_needed);
 139         if (!file)
 140                 goto bad;
 141
 142         retval = -EINVAL;
 143         if (origin <= SEEK_MAX) {
 144                 loff_t res = vfs_llseek(file, offset, origin);
 145                 retval = res;
 146                 if (res != (loff_t)retval)
 147                         retval = -EOVERFLOW;    /* LFS: should only happen on 32 bit platforms */
 148         }
 149         fput_light(file, fput_needed);
 150 bad:
 151         return retval;
 152 }
 153
 154 #ifdef __ARCH_WANT_SYS_LLSEEK
 155 asmlinkage long sys_llseek(unsigned int fd, unsigned long offset_high,
 156                            unsigned long offset_low, loff_t __user * result,
 157                            unsigned int origin)
 158 {
 159         int retval;
 160         struct file * file;
 161         loff_t offset;
 162         int fput_needed;
 163
 164         retval = -EBADF;
 165         file = fget_light(fd, &fput_needed);
 166         if (!file)
 167                 goto bad;
 168
 169         retval = -EINVAL;
 170         if (origin > SEEK_MAX)
 171                 goto out_putf;
 172
 173         offset = vfs_llseek(file, ((loff_t) offset_high << 32) | offset_low,
 174                         origin);
 175
 176         retval = (int)offset;
 177         if (offset >= 0) {
 178                 retval = -EFAULT;
 179                 if (!copy_to_user(result, &offset, sizeof(offset)))
 180                         retval = 0;
 181         }
 182 out_putf:
 183         fput_light(file, fput_needed);
 184 bad:
 185         return retval;
 186 }
 187 #endif
 188
 189 /*
 190  * rw_verify_area doesn't like huge counts. We limit
 191  * them to something that fits in "int" so that others
 192  * won't have to do range checks all the time.
 193  */
 194 #define MAX_RW_COUNT (INT_MAX & PAGE_CACHE_MASK)
 195
 196 int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count)
 197 {
 198         struct inode *inode;
 199         loff_t pos;
 200
 201         inode = file->f_path.dentry->d_inode;
 202         if (unlikely((ssize_t) count < 0))
 203                 goto Einval;
 204         pos = *ppos;
 205         if (unlikely((pos < 0) || (loff_t) (pos + count) < 0))
 206                 goto Einval;
 207
 208         if (unlikely(inode->i_flock && mandatory_lock(inode))) {
 209                 int retval = locks_mandatory_area(
 210                         read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
 211                         inode, file, pos, count);
 212                 if (retval < 0)
 213                         return retval;
 214         }
 215         return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
 216
 217 Einval:
 218         return -EINVAL;
 219 }
 220
 221 static void wait_on_retry_sync_kiocb(struct kiocb *iocb)
 222 {
 223         set_current_state(TASK_UNINTERRUPTIBLE);
 224         if (!kiocbIsKicked(iocb))
 225                 schedule();
 226         else
 227                 kiocbClearKicked(iocb);
 228         __set_current_state(TASK_RUNNING);
 229 }
 230
 231 ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
 232 {
 233         struct iovec iov = { .iov_base = buf, .iov_len = len };
 234         struct kiocb kiocb;
 235         ssize_t ret;
 236
 237         init_sync_kiocb(&kiocb, filp);
 238         kiocb.ki_pos = *ppos;
 239         kiocb.ki_left = len;
 240
 241         for (;;) {
 242                 ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
 243                 if (ret != -EIOCBRETRY)
 244                         break;
 245                 wait_on_retry_sync_kiocb(&kiocb);
 246         }
 247
 248         if (-EIOCBQUEUED == ret)
 249                 ret = wait_on_sync_kiocb(&kiocb);
 250         *ppos = kiocb.ki_pos;
 251         return ret;
 252 }
 253
 254 EXPORT_SYMBOL(do_sync_read);
 255
 256 ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
 257 {
 258         ssize_t ret;
 259
 260         if (!(file->f_mode & FMODE_READ))
 261                 return -EBADF;
 262         if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
 263                 return -EINVAL;
 264         if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
 265                 return -EFAULT;
 266
 267         ret = rw_verify_area(READ, file, pos, count);
 268         if (ret >= 0) {
 269                 count = ret;
 270                 ret = security_file_permission (file, MAY_READ);
 271                 if (!ret) {
 272                         if (file->f_op->read)
 273                                 ret = file->f_op->read(file, buf, count, pos);
 274                         else
 275                                 ret = do_sync_read(file, buf, count, pos);
 276                         if (ret > 0) {
 277                                 fsnotify_access(file->f_path.dentry);
 278                                 add_rchar(current, ret);
 279                         }
 280                         inc_syscr(current);
 281                 }
 282         }
 283
 284         return ret;
 285 }
 286
 287 EXPORT_SYMBOL(vfs_read);
 288
 289 ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
 290 {
 291         struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
 292         struct kiocb kiocb;
 293         ssize_t ret;
 294
 295         init_sync_kiocb(&kiocb, filp);
 296         kiocb.ki_pos = *ppos;
 297         kiocb.ki_left = len;
 298
 299         for (;;) {
 300                 ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
 301                 if (ret != -EIOCBRETRY)
 302                         break;
 303                 wait_on_retry_sync_kiocb(&kiocb);
 304         }
 305
 306         if (-EIOCBQUEUED == ret)
 307                 ret = wait_on_sync_kiocb(&kiocb);
 308         *ppos = kiocb.ki_pos;
 309         return ret;
 310 }
 311
 312 EXPORT_SYMBOL(do_sync_write);
 313
 314 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
 315 {
 316         ssize_t ret;
 317
 318         if (!(file->f_mode & FMODE_WRITE))
 319                 return -EBADF;
 320         if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
 321                 return -EINVAL;
 322         if (unlikely(!access_ok(VERIFY_READ, buf, count)))
 323                 return -EFAULT;
 324
 325         ret = rw_verify_area(WRITE, file, pos, count);
 326         if (ret >= 0) {
 327                 count = ret;
 328                 ret = security_file_permission (file, MAY_WRITE);
 329                 if (!ret) {
 330                         if (file->f_op->write)
 331                                 ret = file->f_op->write(file, buf, count, pos);
 332                         else
 333                                 ret = do_sync_write(file, buf, count, pos);
 334                         if (ret > 0) {
 335                                 fsnotify_modify(file->f_path.dentry);
 336                                 add_wchar(current, ret);
 337                         }
 338                         inc_syscw(current);
 339                 }
 340         }
 341
 342         return ret;
 343 }
 344
 345 EXPORT_SYMBOL(vfs_write);
 346
 347 static inline loff_t file_pos_read(struct file *file)
 348 {
 349         return file->f_pos;
 350 }
 351
 352 static inline void file_pos_write(struct file *file, loff_t pos)
 353 {
 354         file->f_pos = pos;
 355 }
 356
 357 asmlinkage ssize_t sys_read(unsigned int fd, char __user * buf, size_t count)
 358 {
 359         struct file *file;
 360         ssize_t ret = -EBADF;
 361         int fput_needed;
 362
 363         file = fget_light(fd, &fput_needed);
 364         if (file) {
 365                 loff_t pos = file_pos_read(file);
 366                 ret = vfs_read(file, buf, count, &pos);
 367                 file_pos_write(file, pos);
 368                 fput_light(file, fput_needed);
 369         }
 370
 371         return ret;
 372 }
 373 EXPORT_UNUSED_SYMBOL_GPL(sys_read); /* to be deleted for 2.6.25 */
 374
 375 asmlinkage ssize_t sys_write(unsigned int fd, const char __user * buf, size_t count)
 376 {
 377         struct file *file;
 378         ssize_t ret = -EBADF;
 379         int fput_needed;
 380
 381         file = fget_light(fd, &fput_needed);
 382         if (file) {
 383                 loff_t pos = file_pos_read(file);
 384                 ret = vfs_write(file, buf, count, &pos);
 385                 file_pos_write(file, pos);
 386                 fput_light(file, fput_needed);
 387         }
 388
 389         return ret;
 390 }
 391
 392 asmlinkage ssize_t sys_pread64(unsigned int fd, char __user *buf,
 393                              size_t count, loff_t pos)
 394 {
 395         struct file *file;
 396         ssize_t ret = -EBADF;
 397         int fput_needed;
 398
 399         if (pos < 0)
 400                 return -EINVAL;
 401
 402         file = fget_light(fd, &fput_needed);
 403         if (file) {
 404                 ret = -ESPIPE;
 405                 if (file->f_mode & FMODE_PREAD)
 406                         ret = vfs_read(file, buf, count, &pos);
 407                 fput_light(file, fput_needed);
 408         }
 409
 410         return ret;
 411 }
 412
 413 asmlinkage ssize_t sys_pwrite64(unsigned int fd, const char __user *buf,
 414                               size_t count, loff_t pos)
 415 {
 416         struct file *file;
 417         ssize_t ret = -EBADF;
 418         int fput_needed;
 419
 420         if (pos < 0)
 421                 return -EINVAL;
 422
 423         file = fget_light(fd, &fput_needed);
 424         if (file) {
 425                 ret = -ESPIPE;
 426                 if (file->f_mode & FMODE_PWRITE)
 427                         ret = vfs_write(file, buf, count, &pos);
 428                 fput_light(file, fput_needed);
 429         }
 430
 431         return ret;
 432 }
 433
 434 /*
 435  * Reduce an iovec's length in-place.  Return the resulting number of segments
 436  */
 437 unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
 438 {
 439         unsigned long seg = 0;
 440         size_t len = 0;
 441
 442         while (seg < nr_segs) {
 443                 seg++;
 444                 if (len + iov->iov_len >= to) {
 445                         iov->iov_len = to - len;
 446                         break;
 447                 }
 448                 len += iov->iov_len;
 449                 iov++;
 450         }
 451         return seg;
 452 }
 453
 454 ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
 455                 unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
 456 {
 457         struct kiocb kiocb;
 458         ssize_t ret;
 459
 460         init_sync_kiocb(&kiocb, filp);
 461         kiocb.ki_pos = *ppos;
 462         kiocb.ki_left = len;
 463         kiocb.ki_nbytes = len;
 464
 465         for (;;) {
 466                 ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
 467                 if (ret != -EIOCBRETRY)
 468                         break;
 469                 wait_on_retry_sync_kiocb(&kiocb);
 470         }
 471
 472         if (ret == -EIOCBQUEUED)
 473                 ret = wait_on_sync_kiocb(&kiocb);
 474         *ppos = kiocb.ki_pos;
 475         return ret;
 476 }
 477
 478 /* Do it by hand, with file-ops */
 479 ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
 480                 unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
 481 {
 482         struct iovec *vector = iov;
 483         ssize_t ret = 0;
 484
 485         while (nr_segs > 0) {
 486                 void __user *base;
 487                 size_t len;
 488                 ssize_t nr;
 489
 490                 base = vector->iov_base;
 491                 len = vector->iov_len;
 492                 vector++;
 493                 nr_segs--;
 494
 495                 nr = fn(filp, base, len, ppos);
 496
 497                 if (nr < 0) {
 498                         if (!ret)
 499                                 ret = nr;
 500                         break;
 501                 }
 502                 ret += nr;
 503                 if (nr != len)
 504                         break;
 505         }
 506
 507         return ret;
 508 }
 509
 510 /* A write operation does a read from user space and vice versa */
 511 #define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
 512
 513 ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
 514                               unsigned long nr_segs, unsigned long fast_segs,
 515                               struct iovec *fast_pointer,
 516                               struct iovec **ret_pointer)
 517   {
 518         unsigned long seg;
 519         ssize_t ret;
 520         struct iovec *iov = fast_pointer;
 521
 522         /*
 523          * SuS says "The readv() function *may* fail if the iovcnt argument
 524          * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has
 525          * traditionally returned zero for zero segments, so...
 526          */
 527         if (nr_segs == 0) {
 528                 ret = 0;
 529                 goto out;
 530         }
 531
 532         /*
 533          * First get the "struct iovec" from user memory and
 534          * verify all the pointers
 535          */
 536         if (nr_segs > UIO_MAXIOV) {
 537                 ret = -EINVAL;
 538                 goto out;
 539         }
 540         if (nr_segs > fast_segs) {
 541                 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
 542                 if (iov == NULL) {
 543                         ret = -ENOMEM;
 544                         goto out;
 545                 }
 546         }
 547         if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
 548                 ret = -EFAULT;
 549                 goto out;
 550         }
 551
 552         /*
 553          * According to the Single Unix Specification we should return EINVAL
 554          * if an element length is < 0 when cast to ssize_t or if the
 555          * total length would overflow the ssize_t return value of the
 556          * system call.
 557          */
 558         ret = 0;
 559         for (seg = 0; seg < nr_segs; seg++) {
 560                 void __user *buf = iov[seg].iov_base;
 561                 ssize_t len = (ssize_t)iov[seg].iov_len;
 562
 563                 /* see if we we're about to use an invalid len or if
 564                  * it's about to overflow ssize_t */
 565                 if (len < 0 || (ret + len < ret)) {
 566                         ret = -EINVAL;
 567                         goto out;
 568                 }
 569                 if (unlikely(!access_ok(vrfy_dir(type), buf, len))) {
 570                         ret = -EFAULT;
 571                         goto out;
 572                 }
 573
 574                 ret += len;
 575         }
 576 out:
 577         *ret_pointer = iov;
 578         return ret;
 579 }
 580
 581 static ssize_t do_readv_writev(int type, struct file *file,
 582                                const struct iovec __user * uvector,
 583                                unsigned long nr_segs, loff_t *pos)
 584 {
 585         size_t tot_len;
 586         struct iovec iovstack[UIO_FASTIOV];
 587         struct iovec *iov = iovstack;
 588         ssize_t ret;
 589         io_fn_t fn;
 590         iov_fn_t fnv;
 591
 592         if (!file->f_op) {
 593                 ret = -EINVAL;
 594                 goto out;
 595         }
 596
 597         ret = rw_copy_check_uvector(type, uvector, nr_segs,
 598                         ARRAY_SIZE(iovstack), iovstack, &iov);
 599         if (ret <= 0)
 600                 goto out;
 601
 602         tot_len = ret;
 603         ret = rw_verify_area(type, file, pos, tot_len);
 604         if (ret < 0)
 605                 goto out;
 606         ret = security_file_permission(file, type == READ ? MAY_READ : MAY_WRITE);
 607         if (ret)
 608                 goto out;
 609
 610         fnv = NULL;
 611         if (type == READ) {
 612                 fn = file->f_op->read;
 613                 fnv = file->f_op->aio_read;
 614         } else {
 615                 fn = (io_fn_t)file->f_op->write;
 616                 fnv = file->f_op->aio_write;
 617         }
 618
 619         if (fnv)
 620                 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
 621                                                 pos, fnv);
 622         else
 623                 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
 624
 625 out:
 626         if (iov != iovstack)
 627                 kfree(iov);
 628         if ((ret + (type == READ)) > 0) {
 629                 if (type == READ)
 630                         fsnotify_access(file->f_path.dentry);
 631                 else
 632                         fsnotify_modify(file->f_path.dentry);
 633         }
 634         return ret;
 635 }
 636
 637 ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
 638                   unsigned long vlen, loff_t *pos)
 639 {
 640         if (!(file->f_mode & FMODE_READ))
 641                 return -EBADF;
 642         if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
 643                 return -EINVAL;
 644
 645         return do_readv_writev(READ, file, vec, vlen, pos);
 646 }
 647
 648 EXPORT_SYMBOL(vfs_readv);
 649
 650 ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
 651                    unsigned long vlen, loff_t *pos)
 652 {
 653         if (!(file->f_mode & FMODE_WRITE))
 654                 return -EBADF;
 655         if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
 656                 return -EINVAL;
 657
 658         return do_readv_writev(WRITE, file, vec, vlen, pos);
 659 }
 660
 661 EXPORT_SYMBOL(vfs_writev);
 662
 663 asmlinkage ssize_t
 664 sys_readv(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
 665 {
 666         struct file *file;
 667         ssize_t ret = -EBADF;
 668         int fput_needed;
 669
 670         file = fget_light(fd, &fput_needed);
 671         if (file) {
 672                 loff_t pos = file_pos_read(file);
 673                 ret = vfs_readv(file, vec, vlen, &pos);
 674                 file_pos_write(file, pos);
 675                 fput_light(file, fput_needed);
 676         }
 677
 678         if (ret > 0)
 679                 add_rchar(current, ret);
 680         inc_syscr(current);
 681         return ret;
 682 }
 683
 684 asmlinkage ssize_t
 685 sys_writev(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
 686 {
 687         struct file *file;
 688         ssize_t ret = -EBADF;
 689         int fput_needed;
 690
 691         file = fget_light(fd, &fput_needed);
 692         if (file) {
 693                 loff_t pos = file_pos_read(file);
 694                 ret = vfs_writev(file, vec, vlen, &pos);
 695                 file_pos_write(file, pos);
 696                 fput_light(file, fput_needed);
 697         }
 698
 699         if (ret > 0)
 700                 add_wchar(current, ret);
 701         inc_syscw(current);
 702         return ret;
 703 }
 704
 705 static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
 706                            size_t count, loff_t max)
 707 {
 708         struct file * in_file, * out_file;
 709         struct inode * in_inode, * out_inode;
 710         loff_t pos;
 711         ssize_t retval;
 712         int fput_needed_in, fput_needed_out, fl;
 713
 714         /*
 715          * Get input file, and verify that it is ok..
 716          */
 717         retval = -EBADF;
 718         in_file = fget_light(in_fd, &fput_needed_in);
 719         if (!in_file)
 720                 goto out;
 721         if (!(in_file->f_mode & FMODE_READ))
 722                 goto fput_in;
 723         retval = -EINVAL;
 724         in_inode = in_file->f_path.dentry->d_inode;
 725         if (!in_inode)
 726                 goto fput_in;
 727         if (!in_file->f_op || !in_file->f_op->splice_read)
 728                 goto fput_in;
 729         retval = -ESPIPE;
 730         if (!ppos)
 731                 ppos = &in_file->f_pos;
 732         else
 733                 if (!(in_file->f_mode & FMODE_PREAD))
 734                         goto fput_in;
 735         retval = rw_verify_area(READ, in_file, ppos, count);
 736         if (retval < 0)
 737                 goto fput_in;
 738         count = retval;
 739
 740         retval = security_file_permission (in_file, MAY_READ);
 741         if (retval)
 742                 goto fput_in;
 743
 744         /*
 745          * Get output file, and verify that it is ok..
 746          */
 747         retval = -EBADF;
 748         out_file = fget_light(out_fd, &fput_needed_out);
 749         if (!out_file)
 750                 goto fput_in;
 751         if (!(out_file->f_mode & FMODE_WRITE))
 752                 goto fput_out;
 753         retval = -EINVAL;
 754         if (!out_file->f_op || !out_file->f_op->sendpage)
 755                 goto fput_out;
 756         out_inode = out_file->f_path.dentry->d_inode;
 757         retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
 758         if (retval < 0)
 759                 goto fput_out;
 760         count = retval;
 761
 762         retval = security_file_permission (out_file, MAY_WRITE);
 763         if (retval)
 764                 goto fput_out;
 765
 766         if (!max)
 767                 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
 768
 769         pos = *ppos;
 770         retval = -EINVAL;
 771         if (unlikely(pos < 0))
 772                 goto fput_out;
 773         if (unlikely(pos + count > max)) {
 774                 retval = -EOVERFLOW;
 775                 if (pos >= max)
 776                         goto fput_out;
 777                 count = max - pos;
 778         }
 779
 780         fl = 0;
 781 #if 0
 782         /*
 783          * We need to debate whether we can enable this or not. The
 784          * man page documents EAGAIN return for the output at least,
 785          * and the application is arguably buggy if it doesn't expect
 786          * EAGAIN on a non-blocking file descriptor.
 787          */
 788         if (in_file->f_flags & O_NONBLOCK)
 789                 fl = SPLICE_F_NONBLOCK;
 790 #endif
 791         retval = do_splice_direct(in_file, ppos, out_file, count, fl);
 792
 793         if (retval > 0) {
 794                 add_rchar(current, retval);
 795                 add_wchar(current, retval);
 796         }
 797
 798         inc_syscr(current);
 799         inc_syscw(current);
 800         if (*ppos > max)
 801                 retval = -EOVERFLOW;
 802
 803 fput_out:
 804         fput_light(out_file, fput_needed_out);
 805 fput_in:
 806         fput_light(in_file, fput_needed_in);
 807 out:
 808         return retval;
 809 }
 810
 811 asmlinkage ssize_t sys_sendfile(int out_fd, int in_fd, off_t __user *offset, size_t count)
 812 {
 813         loff_t pos;
 814         off_t off;
 815         ssize_t ret;
 816
 817         if (offset) {
 818                 if (unlikely(get_user(off, offset)))
 819                         return -EFAULT;
 820                 pos = off;
 821                 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
 822                 if (unlikely(put_user(pos, offset)))
 823                         return -EFAULT;
 824                 return ret;
 825         }
 826
 827         return do_sendfile(out_fd, in_fd, NULL, count, 0);
 828 }
 829
 830 asmlinkage ssize_t sys_sendfile64(int out_fd, int in_fd, loff_t __user *offset, size_t count)
 831 {
 832         loff_t pos;
 833         ssize_t ret;
 834
 835         if (offset) {
 836                 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
 837                         return -EFAULT;
 838                 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
 839                 if (unlikely(put_user(pos, offset)))
 840                         return -EFAULT;
 841                 return ret;
 842         }
 843
 844         return do_sendfile(out_fd, in_fd, NULL, count, 0);
 845 }