4 * Copyright (C) 1998-1999, Stephen Tweedie and Bill Hawes
6 * Manage the dynamic fd arrays in the process files_struct.
11 #include <linux/time.h>
12 #include <linux/slab.h>
13 #include <linux/vmalloc.h>
14 #include <linux/file.h>
15 #include <linux/bitops.h>
16 #include <linux/interrupt.h>
17 #include <linux/spinlock.h>
18 #include <linux/rcupdate.h>
19 #include <linux/workqueue.h>
21 struct fdtable_defer
{
23 struct work_struct wq
;
28 * We use this list to defer free fdtables that have vmalloced
29 * sets/arrays. By keeping a per-cpu list, we avoid having to embed
30 * the work_struct in fdtable itself which avoids a 64 byte (i386) increase in
31 * this per-task structure.
33 static DEFINE_PER_CPU(struct fdtable_defer
, fdtable_defer_list
);
37 * Allocate an fd array, using kmalloc or vmalloc.
38 * Note: the array isn't cleared at allocation time.
40 struct file
** alloc_fd_array(int num
)
42 struct file
**new_fds
;
43 int size
= num
* sizeof(struct file
*);
45 if (size
<= PAGE_SIZE
)
46 new_fds
= (struct file
**) kmalloc(size
, GFP_KERNEL
);
48 new_fds
= (struct file
**) vmalloc(size
);
52 void free_fd_array(struct file
**array
, int num
)
54 int size
= num
* sizeof(struct file
*);
57 printk (KERN_ERR
"free_fd_array: array = 0 (num = %d)\n", num
);
61 if (num
<= NR_OPEN_DEFAULT
) /* Don't free the embedded fd array! */
63 else if (size
<= PAGE_SIZE
)
69 static void __free_fdtable(struct fdtable
*fdt
)
71 free_fdset(fdt
->open_fds
, fdt
->max_fdset
);
72 free_fdset(fdt
->close_on_exec
, fdt
->max_fdset
);
73 free_fd_array(fdt
->fd
, fdt
->max_fds
);
77 static void free_fdtable_work(struct work_struct
*work
)
79 struct fdtable_defer
*f
=
80 container_of(work
, struct fdtable_defer
, wq
);
83 spin_lock_bh(&f
->lock
);
86 spin_unlock_bh(&f
->lock
);
88 struct fdtable
*next
= fdt
->next
;
94 static void free_fdtable_rcu(struct rcu_head
*rcu
)
96 struct fdtable
*fdt
= container_of(rcu
, struct fdtable
, rcu
);
97 int fdset_size
, fdarray_size
;
98 struct fdtable_defer
*fddef
;
101 fdset_size
= fdt
->max_fdset
/ 8;
102 fdarray_size
= fdt
->max_fds
* sizeof(struct file
*);
104 if (fdt
->free_files
) {
106 * The this fdtable was embedded in the files structure
107 * and the files structure itself was getting destroyed.
108 * It is now safe to free the files structure.
110 kmem_cache_free(files_cachep
, fdt
->free_files
);
113 if (fdt
->max_fdset
<= EMBEDDED_FD_SET_SIZE
&&
114 fdt
->max_fds
<= NR_OPEN_DEFAULT
) {
116 * The fdtable was embedded
120 if (fdset_size
<= PAGE_SIZE
&& fdarray_size
<= PAGE_SIZE
) {
121 kfree(fdt
->open_fds
);
122 kfree(fdt
->close_on_exec
);
126 fddef
= &get_cpu_var(fdtable_defer_list
);
127 spin_lock(&fddef
->lock
);
128 fdt
->next
= fddef
->next
;
130 /* vmallocs are handled from the workqueue context */
131 schedule_work(&fddef
->wq
);
132 spin_unlock(&fddef
->lock
);
133 put_cpu_var(fdtable_defer_list
);
137 void free_fdtable(struct fdtable
*fdt
)
139 if (fdt
->free_files
||
140 fdt
->max_fdset
> EMBEDDED_FD_SET_SIZE
||
141 fdt
->max_fds
> NR_OPEN_DEFAULT
)
142 call_rcu(&fdt
->rcu
, free_fdtable_rcu
);
146 * Expand the fdset in the files_struct. Called with the files spinlock
149 static void copy_fdtable(struct fdtable
*nfdt
, struct fdtable
*fdt
)
154 BUG_ON(nfdt
->max_fdset
< fdt
->max_fdset
);
155 BUG_ON(nfdt
->max_fds
< fdt
->max_fds
);
156 /* Copy the existing tables and install the new pointers */
158 i
= fdt
->max_fdset
/ (sizeof(unsigned long) * 8);
159 count
= (nfdt
->max_fdset
- fdt
->max_fdset
) / 8;
162 * Don't copy the entire array if the current fdset is
163 * not yet initialised.
166 memcpy (nfdt
->open_fds
, fdt
->open_fds
,
168 memcpy (nfdt
->close_on_exec
, fdt
->close_on_exec
,
170 memset (&nfdt
->open_fds
->fds_bits
[i
], 0, count
);
171 memset (&nfdt
->close_on_exec
->fds_bits
[i
], 0, count
);
174 /* Don't copy/clear the array if we are creating a new
175 fd array for fork() */
177 memcpy(nfdt
->fd
, fdt
->fd
,
178 fdt
->max_fds
* sizeof(struct file
*));
179 /* clear the remainder of the array */
180 memset(&nfdt
->fd
[fdt
->max_fds
], 0,
181 (nfdt
->max_fds
- fdt
->max_fds
) *
182 sizeof(struct file
*));
187 * Allocate an fdset array, using kmalloc or vmalloc.
188 * Note: the array isn't cleared at allocation time.
190 fd_set
* alloc_fdset(int num
)
195 if (size
<= PAGE_SIZE
)
196 new_fdset
= (fd_set
*) kmalloc(size
, GFP_KERNEL
);
198 new_fdset
= (fd_set
*) vmalloc(size
);
202 void free_fdset(fd_set
*array
, int num
)
204 if (num
<= EMBEDDED_FD_SET_SIZE
) /* Don't free an embedded fdset */
206 else if (num
<= 8 * PAGE_SIZE
)
212 static struct fdtable
*alloc_fdtable(int nr
)
214 struct fdtable
*fdt
= NULL
;
216 fd_set
*new_openset
= NULL
, *new_execset
= NULL
;
217 struct file
**new_fds
;
219 fdt
= kzalloc(sizeof(*fdt
), GFP_KERNEL
);
223 nfds
= max_t(int, 8 * L1_CACHE_BYTES
, roundup_pow_of_two(nr
+ 1));
227 new_openset
= alloc_fdset(nfds
);
228 new_execset
= alloc_fdset(nfds
);
229 if (!new_openset
|| !new_execset
)
231 fdt
->open_fds
= new_openset
;
232 fdt
->close_on_exec
= new_execset
;
233 fdt
->max_fdset
= nfds
;
235 nfds
= NR_OPEN_DEFAULT
;
237 * Expand to the max in easy steps, and keep expanding it until
238 * we have enough for the requested fd array size.
241 #if NR_OPEN_DEFAULT < 256
246 if (nfds
< (PAGE_SIZE
/ sizeof(struct file
*)))
247 nfds
= PAGE_SIZE
/ sizeof(struct file
*);
253 } while (nfds
<= nr
);
254 new_fds
= alloc_fd_array(nfds
);
259 fdt
->free_files
= NULL
;
262 nfds
= fdt
->max_fdset
;
264 free_fdset(new_openset
, nfds
);
265 free_fdset(new_execset
, nfds
);
271 * Expand the file descriptor table.
272 * This function will allocate a new fdtable and both fd array and fdset, of
274 * Return <0 error code on error; 1 on successful completion.
275 * The files->file_lock should be held on entry, and will be held on exit.
277 static int expand_fdtable(struct files_struct
*files
, int nr
)
278 __releases(files
->file_lock
)
279 __acquires(files
->file_lock
)
281 struct fdtable
*new_fdt
, *cur_fdt
;
283 spin_unlock(&files
->file_lock
);
284 new_fdt
= alloc_fdtable(nr
);
285 spin_lock(&files
->file_lock
);
289 * Check again since another task may have expanded the fd table while
290 * we dropped the lock
292 cur_fdt
= files_fdtable(files
);
293 if (nr
>= cur_fdt
->max_fds
|| nr
>= cur_fdt
->max_fdset
) {
294 /* Continue as planned */
295 copy_fdtable(new_fdt
, cur_fdt
);
296 rcu_assign_pointer(files
->fdt
, new_fdt
);
297 free_fdtable(cur_fdt
);
299 /* Somebody else expanded, so undo our attempt */
300 __free_fdtable(new_fdt
);
307 * This function will expand the file structures, if the requested size exceeds
308 * the current capacity and there is room for expansion.
309 * Return <0 error code on error; 0 when nothing done; 1 when files were
310 * expanded and execution may have blocked.
311 * The files->file_lock should be held on entry, and will be held on exit.
313 int expand_files(struct files_struct
*files
, int nr
)
317 fdt
= files_fdtable(files
);
318 /* Do we need to expand? */
319 if (nr
< fdt
->max_fdset
&& nr
< fdt
->max_fds
)
322 if (fdt
->max_fdset
>= NR_OPEN
|| fdt
->max_fds
>= NR_OPEN
||
326 /* All good, so we try */
327 return expand_fdtable(files
, nr
);
330 static void __devinit
fdtable_defer_list_init(int cpu
)
332 struct fdtable_defer
*fddef
= &per_cpu(fdtable_defer_list
, cpu
);
333 spin_lock_init(&fddef
->lock
);
334 INIT_WORK(&fddef
->wq
, free_fdtable_work
);
338 void __init
files_defer_init(void)
341 for_each_possible_cpu(i
)
342 fdtable_defer_list_init(i
);