4 * Copyright (C) 1991, 1992, 1999 Linus Torvalds
8 #include <linux/file.h>
9 #include <linux/poll.h>
10 #include <linux/slab.h>
11 #include <linux/module.h>
12 #include <linux/init.h>
14 #include <linux/mount.h>
15 #include <linux/pipe_fs_i.h>
16 #include <linux/uio.h>
17 #include <linux/highmem.h>
18 #include <linux/pagemap.h>
20 #include <asm/uaccess.h>
21 #include <asm/ioctls.h>
24 * We use a start+len construction, which provides full use of the
26 * -- Florian Coosmann (FGC)
28 * Reads with count = 0 should always return 0.
29 * -- Julian Bradfield 1999-06-07.
31 * FIFOs and Pipes now generate SIGIO for both readers and writers.
32 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
34 * pipe_read & write cleanup
35 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
38 /* Drop the inode semaphore and wait for a pipe event, atomically */
39 void pipe_wait(struct inode
* inode
)
44 * Pipes are system-local resources, so sleeping on them
45 * is considered a noninteractive wait:
47 prepare_to_wait(PIPE_WAIT(*inode
), &wait
, TASK_INTERRUPTIBLE
|TASK_NONINTERACTIVE
);
48 mutex_unlock(PIPE_MUTEX(*inode
));
50 finish_wait(PIPE_WAIT(*inode
), &wait
);
51 mutex_lock(PIPE_MUTEX(*inode
));
55 pipe_iov_copy_from_user(void *to
, struct iovec
*iov
, unsigned long len
)
62 copy
= min_t(unsigned long, len
, iov
->iov_len
);
64 if (copy_from_user(to
, iov
->iov_base
, copy
))
68 iov
->iov_base
+= copy
;
75 pipe_iov_copy_to_user(struct iovec
*iov
, const void *from
, unsigned long len
)
82 copy
= min_t(unsigned long, len
, iov
->iov_len
);
84 if (copy_to_user(iov
->iov_base
, from
, copy
))
88 iov
->iov_base
+= copy
;
94 static void anon_pipe_buf_release(struct pipe_inode_info
*info
, struct pipe_buffer
*buf
)
96 struct page
*page
= buf
->page
;
98 buf
->flags
&= ~PIPE_BUF_FLAG_STOLEN
;
101 * If nobody else uses this page, and we don't already have a
102 * temporary page, let's keep track of it as a one-deep
105 if (page_count(page
) == 1 && !info
->tmp_page
) {
106 info
->tmp_page
= page
;
111 * Otherwise just release our reference to it
113 page_cache_release(page
);
116 static void *anon_pipe_buf_map(struct file
*file
, struct pipe_inode_info
*info
, struct pipe_buffer
*buf
)
118 return kmap(buf
->page
);
121 static void anon_pipe_buf_unmap(struct pipe_inode_info
*info
, struct pipe_buffer
*buf
)
126 static int anon_pipe_buf_steal(struct pipe_inode_info
*info
,
127 struct pipe_buffer
*buf
)
129 buf
->flags
|= PIPE_BUF_FLAG_STOLEN
;
133 static struct pipe_buf_operations anon_pipe_buf_ops
= {
135 .map
= anon_pipe_buf_map
,
136 .unmap
= anon_pipe_buf_unmap
,
137 .release
= anon_pipe_buf_release
,
138 .steal
= anon_pipe_buf_steal
,
142 pipe_readv(struct file
*filp
, const struct iovec
*_iov
,
143 unsigned long nr_segs
, loff_t
*ppos
)
145 struct inode
*inode
= filp
->f_dentry
->d_inode
;
146 struct pipe_inode_info
*info
;
149 struct iovec
*iov
= (struct iovec
*)_iov
;
152 total_len
= iov_length(iov
, nr_segs
);
153 /* Null read succeeds. */
154 if (unlikely(total_len
== 0))
159 mutex_lock(PIPE_MUTEX(*inode
));
160 info
= inode
->i_pipe
;
162 int bufs
= info
->nrbufs
;
164 int curbuf
= info
->curbuf
;
165 struct pipe_buffer
*buf
= info
->bufs
+ curbuf
;
166 struct pipe_buf_operations
*ops
= buf
->ops
;
168 size_t chars
= buf
->len
;
171 if (chars
> total_len
)
174 addr
= ops
->map(filp
, info
, buf
);
180 error
= pipe_iov_copy_to_user(iov
, addr
+ buf
->offset
, chars
);
181 ops
->unmap(info
, buf
);
182 if (unlikely(error
)) {
183 if (!ret
) ret
= -EFAULT
;
187 buf
->offset
+= chars
;
191 ops
->release(info
, buf
);
192 curbuf
= (curbuf
+ 1) & (PIPE_BUFFERS
-1);
193 info
->curbuf
= curbuf
;
194 info
->nrbufs
= --bufs
;
199 break; /* common path: read succeeded */
201 if (bufs
) /* More to do? */
203 if (!PIPE_WRITERS(*inode
))
205 if (!PIPE_WAITING_WRITERS(*inode
)) {
206 /* syscall merging: Usually we must not sleep
207 * if O_NONBLOCK is set, or if we got some data.
208 * But if a writer sleeps in kernel space, then
209 * we can wait for that data without violating POSIX.
213 if (filp
->f_flags
& O_NONBLOCK
) {
218 if (signal_pending(current
)) {
219 if (!ret
) ret
= -ERESTARTSYS
;
223 wake_up_interruptible_sync(PIPE_WAIT(*inode
));
224 kill_fasync(PIPE_FASYNC_WRITERS(*inode
), SIGIO
, POLL_OUT
);
228 mutex_unlock(PIPE_MUTEX(*inode
));
229 /* Signal writers asynchronously that there is more room. */
231 wake_up_interruptible(PIPE_WAIT(*inode
));
232 kill_fasync(PIPE_FASYNC_WRITERS(*inode
), SIGIO
, POLL_OUT
);
240 pipe_read(struct file
*filp
, char __user
*buf
, size_t count
, loff_t
*ppos
)
242 struct iovec iov
= { .iov_base
= buf
, .iov_len
= count
};
243 return pipe_readv(filp
, &iov
, 1, ppos
);
247 pipe_writev(struct file
*filp
, const struct iovec
*_iov
,
248 unsigned long nr_segs
, loff_t
*ppos
)
250 struct inode
*inode
= filp
->f_dentry
->d_inode
;
251 struct pipe_inode_info
*info
;
254 struct iovec
*iov
= (struct iovec
*)_iov
;
258 total_len
= iov_length(iov
, nr_segs
);
259 /* Null write succeeds. */
260 if (unlikely(total_len
== 0))
265 mutex_lock(PIPE_MUTEX(*inode
));
266 info
= inode
->i_pipe
;
268 if (!PIPE_READERS(*inode
)) {
269 send_sig(SIGPIPE
, current
, 0);
274 /* We try to merge small writes */
275 chars
= total_len
& (PAGE_SIZE
-1); /* size of the last buffer */
276 if (info
->nrbufs
&& chars
!= 0) {
277 int lastbuf
= (info
->curbuf
+ info
->nrbufs
- 1) & (PIPE_BUFFERS
-1);
278 struct pipe_buffer
*buf
= info
->bufs
+ lastbuf
;
279 struct pipe_buf_operations
*ops
= buf
->ops
;
280 int offset
= buf
->offset
+ buf
->len
;
281 if (ops
->can_merge
&& offset
+ chars
<= PAGE_SIZE
) {
285 addr
= ops
->map(filp
, info
, buf
);
287 error
= PTR_ERR(addr
);
290 error
= pipe_iov_copy_from_user(offset
+ addr
, iov
,
292 ops
->unmap(info
, buf
);
307 if (!PIPE_READERS(*inode
)) {
308 send_sig(SIGPIPE
, current
, 0);
309 if (!ret
) ret
= -EPIPE
;
313 if (bufs
< PIPE_BUFFERS
) {
314 int newbuf
= (info
->curbuf
+ bufs
) & (PIPE_BUFFERS
-1);
315 struct pipe_buffer
*buf
= info
->bufs
+ newbuf
;
316 struct page
*page
= info
->tmp_page
;
320 page
= alloc_page(GFP_HIGHUSER
);
321 if (unlikely(!page
)) {
322 ret
= ret
? : -ENOMEM
;
325 info
->tmp_page
= page
;
327 /* Always wakeup, even if the copy fails. Otherwise
328 * we lock up (O_NONBLOCK-)readers that sleep due to
330 * FIXME! Is this really true?
334 if (chars
> total_len
)
337 error
= pipe_iov_copy_from_user(kmap(page
), iov
, chars
);
339 if (unlikely(error
)) {
340 if (!ret
) ret
= -EFAULT
;
345 /* Insert it into the buffer array */
347 buf
->ops
= &anon_pipe_buf_ops
;
350 info
->nrbufs
= ++bufs
;
351 info
->tmp_page
= NULL
;
357 if (bufs
< PIPE_BUFFERS
)
359 if (filp
->f_flags
& O_NONBLOCK
) {
360 if (!ret
) ret
= -EAGAIN
;
363 if (signal_pending(current
)) {
364 if (!ret
) ret
= -ERESTARTSYS
;
368 wake_up_interruptible_sync(PIPE_WAIT(*inode
));
369 kill_fasync(PIPE_FASYNC_READERS(*inode
), SIGIO
, POLL_IN
);
372 PIPE_WAITING_WRITERS(*inode
)++;
374 PIPE_WAITING_WRITERS(*inode
)--;
377 mutex_unlock(PIPE_MUTEX(*inode
));
379 wake_up_interruptible(PIPE_WAIT(*inode
));
380 kill_fasync(PIPE_FASYNC_READERS(*inode
), SIGIO
, POLL_IN
);
383 file_update_time(filp
);
388 pipe_write(struct file
*filp
, const char __user
*buf
,
389 size_t count
, loff_t
*ppos
)
391 struct iovec iov
= { .iov_base
= (void __user
*)buf
, .iov_len
= count
};
392 return pipe_writev(filp
, &iov
, 1, ppos
);
396 bad_pipe_r(struct file
*filp
, char __user
*buf
, size_t count
, loff_t
*ppos
)
402 bad_pipe_w(struct file
*filp
, const char __user
*buf
, size_t count
, loff_t
*ppos
)
408 pipe_ioctl(struct inode
*pino
, struct file
*filp
,
409 unsigned int cmd
, unsigned long arg
)
411 struct inode
*inode
= filp
->f_dentry
->d_inode
;
412 struct pipe_inode_info
*info
;
413 int count
, buf
, nrbufs
;
417 mutex_lock(PIPE_MUTEX(*inode
));
418 info
= inode
->i_pipe
;
421 nrbufs
= info
->nrbufs
;
422 while (--nrbufs
>= 0) {
423 count
+= info
->bufs
[buf
].len
;
424 buf
= (buf
+1) & (PIPE_BUFFERS
-1);
426 mutex_unlock(PIPE_MUTEX(*inode
));
427 return put_user(count
, (int __user
*)arg
);
433 /* No kernel lock held - fine */
435 pipe_poll(struct file
*filp
, poll_table
*wait
)
438 struct inode
*inode
= filp
->f_dentry
->d_inode
;
439 struct pipe_inode_info
*info
= inode
->i_pipe
;
442 poll_wait(filp
, PIPE_WAIT(*inode
), wait
);
444 /* Reading only -- no need for acquiring the semaphore. */
445 nrbufs
= info
->nrbufs
;
447 if (filp
->f_mode
& FMODE_READ
) {
448 mask
= (nrbufs
> 0) ? POLLIN
| POLLRDNORM
: 0;
449 if (!PIPE_WRITERS(*inode
) && filp
->f_version
!= PIPE_WCOUNTER(*inode
))
453 if (filp
->f_mode
& FMODE_WRITE
) {
454 mask
|= (nrbufs
< PIPE_BUFFERS
) ? POLLOUT
| POLLWRNORM
: 0;
456 * Most Unices do not set POLLERR for FIFOs but on Linux they
457 * behave exactly like pipes for poll().
459 if (!PIPE_READERS(*inode
))
467 pipe_release(struct inode
*inode
, int decr
, int decw
)
469 mutex_lock(PIPE_MUTEX(*inode
));
470 PIPE_READERS(*inode
) -= decr
;
471 PIPE_WRITERS(*inode
) -= decw
;
472 if (!PIPE_READERS(*inode
) && !PIPE_WRITERS(*inode
)) {
473 free_pipe_info(inode
);
475 wake_up_interruptible(PIPE_WAIT(*inode
));
476 kill_fasync(PIPE_FASYNC_READERS(*inode
), SIGIO
, POLL_IN
);
477 kill_fasync(PIPE_FASYNC_WRITERS(*inode
), SIGIO
, POLL_OUT
);
479 mutex_unlock(PIPE_MUTEX(*inode
));
485 pipe_read_fasync(int fd
, struct file
*filp
, int on
)
487 struct inode
*inode
= filp
->f_dentry
->d_inode
;
490 mutex_lock(PIPE_MUTEX(*inode
));
491 retval
= fasync_helper(fd
, filp
, on
, PIPE_FASYNC_READERS(*inode
));
492 mutex_unlock(PIPE_MUTEX(*inode
));
502 pipe_write_fasync(int fd
, struct file
*filp
, int on
)
504 struct inode
*inode
= filp
->f_dentry
->d_inode
;
507 mutex_lock(PIPE_MUTEX(*inode
));
508 retval
= fasync_helper(fd
, filp
, on
, PIPE_FASYNC_WRITERS(*inode
));
509 mutex_unlock(PIPE_MUTEX(*inode
));
519 pipe_rdwr_fasync(int fd
, struct file
*filp
, int on
)
521 struct inode
*inode
= filp
->f_dentry
->d_inode
;
524 mutex_lock(PIPE_MUTEX(*inode
));
526 retval
= fasync_helper(fd
, filp
, on
, PIPE_FASYNC_READERS(*inode
));
529 retval
= fasync_helper(fd
, filp
, on
, PIPE_FASYNC_WRITERS(*inode
));
531 mutex_unlock(PIPE_MUTEX(*inode
));
541 pipe_read_release(struct inode
*inode
, struct file
*filp
)
543 pipe_read_fasync(-1, filp
, 0);
544 return pipe_release(inode
, 1, 0);
548 pipe_write_release(struct inode
*inode
, struct file
*filp
)
550 pipe_write_fasync(-1, filp
, 0);
551 return pipe_release(inode
, 0, 1);
555 pipe_rdwr_release(struct inode
*inode
, struct file
*filp
)
559 pipe_rdwr_fasync(-1, filp
, 0);
560 decr
= (filp
->f_mode
& FMODE_READ
) != 0;
561 decw
= (filp
->f_mode
& FMODE_WRITE
) != 0;
562 return pipe_release(inode
, decr
, decw
);
566 pipe_read_open(struct inode
*inode
, struct file
*filp
)
568 /* We could have perhaps used atomic_t, but this and friends
569 below are the only places. So it doesn't seem worthwhile. */
570 mutex_lock(PIPE_MUTEX(*inode
));
571 PIPE_READERS(*inode
)++;
572 mutex_unlock(PIPE_MUTEX(*inode
));
578 pipe_write_open(struct inode
*inode
, struct file
*filp
)
580 mutex_lock(PIPE_MUTEX(*inode
));
581 PIPE_WRITERS(*inode
)++;
582 mutex_unlock(PIPE_MUTEX(*inode
));
588 pipe_rdwr_open(struct inode
*inode
, struct file
*filp
)
590 mutex_lock(PIPE_MUTEX(*inode
));
591 if (filp
->f_mode
& FMODE_READ
)
592 PIPE_READERS(*inode
)++;
593 if (filp
->f_mode
& FMODE_WRITE
)
594 PIPE_WRITERS(*inode
)++;
595 mutex_unlock(PIPE_MUTEX(*inode
));
601 * The file_operations structs are not static because they
602 * are also used in linux/fs/fifo.c to do operations on FIFOs.
604 const struct file_operations read_fifo_fops
= {
611 .open
= pipe_read_open
,
612 .release
= pipe_read_release
,
613 .fasync
= pipe_read_fasync
,
616 const struct file_operations write_fifo_fops
= {
620 .writev
= pipe_writev
,
623 .open
= pipe_write_open
,
624 .release
= pipe_write_release
,
625 .fasync
= pipe_write_fasync
,
628 const struct file_operations rdwr_fifo_fops
= {
633 .writev
= pipe_writev
,
636 .open
= pipe_rdwr_open
,
637 .release
= pipe_rdwr_release
,
638 .fasync
= pipe_rdwr_fasync
,
641 static struct file_operations read_pipe_fops
= {
648 .open
= pipe_read_open
,
649 .release
= pipe_read_release
,
650 .fasync
= pipe_read_fasync
,
653 static struct file_operations write_pipe_fops
= {
657 .writev
= pipe_writev
,
660 .open
= pipe_write_open
,
661 .release
= pipe_write_release
,
662 .fasync
= pipe_write_fasync
,
665 static struct file_operations rdwr_pipe_fops
= {
670 .writev
= pipe_writev
,
673 .open
= pipe_rdwr_open
,
674 .release
= pipe_rdwr_release
,
675 .fasync
= pipe_rdwr_fasync
,
678 void free_pipe_info(struct inode
*inode
)
681 struct pipe_inode_info
*info
= inode
->i_pipe
;
683 inode
->i_pipe
= NULL
;
684 for (i
= 0; i
< PIPE_BUFFERS
; i
++) {
685 struct pipe_buffer
*buf
= info
->bufs
+ i
;
687 buf
->ops
->release(info
, buf
);
690 __free_page(info
->tmp_page
);
694 struct inode
* pipe_new(struct inode
* inode
)
696 struct pipe_inode_info
*info
;
698 info
= kzalloc(sizeof(struct pipe_inode_info
), GFP_KERNEL
);
701 inode
->i_pipe
= info
;
703 init_waitqueue_head(PIPE_WAIT(*inode
));
704 PIPE_RCOUNTER(*inode
) = PIPE_WCOUNTER(*inode
) = 1;
711 static struct vfsmount
*pipe_mnt __read_mostly
;
712 static int pipefs_delete_dentry(struct dentry
*dentry
)
716 static struct dentry_operations pipefs_dentry_operations
= {
717 .d_delete
= pipefs_delete_dentry
,
720 static struct inode
* get_pipe_inode(void)
722 struct inode
*inode
= new_inode(pipe_mnt
->mnt_sb
);
729 PIPE_READERS(*inode
) = PIPE_WRITERS(*inode
) = 1;
730 inode
->i_fop
= &rdwr_pipe_fops
;
733 * Mark the inode dirty from the very beginning,
734 * that way it will never be moved to the dirty
735 * list because "mark_inode_dirty()" will think
736 * that it already _is_ on the dirty list.
738 inode
->i_state
= I_DIRTY
;
739 inode
->i_mode
= S_IFIFO
| S_IRUSR
| S_IWUSR
;
740 inode
->i_uid
= current
->fsuid
;
741 inode
->i_gid
= current
->fsgid
;
742 inode
->i_atime
= inode
->i_mtime
= inode
->i_ctime
= CURRENT_TIME
;
743 inode
->i_blksize
= PAGE_SIZE
;
756 struct dentry
*dentry
;
757 struct inode
* inode
;
758 struct file
*f1
, *f2
;
763 f1
= get_empty_filp();
767 f2
= get_empty_filp();
771 inode
= get_pipe_inode();
775 error
= get_unused_fd();
777 goto close_f12_inode
;
780 error
= get_unused_fd();
782 goto close_f12_inode_i
;
786 sprintf(name
, "[%lu]", inode
->i_ino
);
788 this.len
= strlen(name
);
789 this.hash
= inode
->i_ino
; /* will go */
790 dentry
= d_alloc(pipe_mnt
->mnt_sb
->s_root
, &this);
792 goto close_f12_inode_i_j
;
793 dentry
->d_op
= &pipefs_dentry_operations
;
794 d_add(dentry
, inode
);
795 f1
->f_vfsmnt
= f2
->f_vfsmnt
= mntget(mntget(pipe_mnt
));
796 f1
->f_dentry
= f2
->f_dentry
= dget(dentry
);
797 f1
->f_mapping
= f2
->f_mapping
= inode
->i_mapping
;
800 f1
->f_pos
= f2
->f_pos
= 0;
801 f1
->f_flags
= O_RDONLY
;
802 f1
->f_op
= &read_pipe_fops
;
803 f1
->f_mode
= FMODE_READ
;
807 f2
->f_flags
= O_WRONLY
;
808 f2
->f_op
= &write_pipe_fops
;
809 f2
->f_mode
= FMODE_WRITE
;
823 free_pipe_info(inode
);
834 * pipefs should _never_ be mounted by userland - too much of security hassle,
835 * no real gain from having the whole whorehouse mounted. So we don't need
836 * any operations on the root directory. However, we need a non-trivial
837 * d_name - pipe: will go nicely and kill the special-casing in procfs.
840 static struct super_block
*pipefs_get_sb(struct file_system_type
*fs_type
,
841 int flags
, const char *dev_name
, void *data
)
843 return get_sb_pseudo(fs_type
, "pipe:", NULL
, PIPEFS_MAGIC
);
846 static struct file_system_type pipe_fs_type
= {
848 .get_sb
= pipefs_get_sb
,
849 .kill_sb
= kill_anon_super
,
852 static int __init
init_pipe_fs(void)
854 int err
= register_filesystem(&pipe_fs_type
);
856 pipe_mnt
= kern_mount(&pipe_fs_type
);
857 if (IS_ERR(pipe_mnt
)) {
858 err
= PTR_ERR(pipe_mnt
);
859 unregister_filesystem(&pipe_fs_type
);
865 static void __exit
exit_pipe_fs(void)
867 unregister_filesystem(&pipe_fs_type
);
871 fs_initcall(init_pipe_fs
);
872 module_exit(exit_pipe_fs
);