?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.18.10
[linux/fpc-iii.git] / fs / hugetlbfs / inode.c
blob40d4c66c7751dd895302f42e4ec0d4bc60c1f87b
1 /*
2 * hugetlbpage-backed filesystem. Based on ramfs.
3 *
4 * Nadia Yvette Chambers, 2002
5 *
6 * Copyright (C) 2002 Linus Torvalds.
7 * License: GPL
8 */
9
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11
12 #include <linux/thread_info.h>
13 #include <asm/current.h>
14 #include <linux/sched/signal.h> /* remove ASAP */
15 #include <linux/falloc.h>
16 #include <linux/fs.h>
17 #include <linux/mount.h>
18 #include <linux/file.h>
19 #include <linux/kernel.h>
20 #include <linux/writeback.h>
21 #include <linux/pagemap.h>
22 #include <linux/highmem.h>
23 #include <linux/init.h>
24 #include <linux/string.h>
25 #include <linux/capability.h>
26 #include <linux/ctype.h>
27 #include <linux/backing-dev.h>
28 #include <linux/hugetlb.h>
29 #include <linux/pagevec.h>
30 #include <linux/parser.h>
31 #include <linux/mman.h>
32 #include <linux/slab.h>
33 #include <linux/dnotify.h>
34 #include <linux/statfs.h>
35 #include <linux/security.h>
36 #include <linux/magic.h>
37 #include <linux/migrate.h>
38 #include <linux/uio.h>
39
40 #include <linux/uaccess.h>
41
42 static const struct super_operations hugetlbfs_ops;
43 static const struct address_space_operations hugetlbfs_aops;
44 const struct file_operations hugetlbfs_file_operations;
45 static const struct inode_operations hugetlbfs_dir_inode_operations;
46 static const struct inode_operations hugetlbfs_inode_operations;
47
48 struct hugetlbfs_config {
49 struct hstate *hstate;
50 long max_hpages;
51 long nr_inodes;
52 long min_hpages;
53 kuid_t uid;
54 kgid_t gid;
55 umode_t mode;
56 };
57
58 int sysctl_hugetlb_shm_group;
59
60 enum {
61 Opt_size, Opt_nr_inodes,
62 Opt_mode, Opt_uid, Opt_gid,
63 Opt_pagesize, Opt_min_size,
64 Opt_err,
65 };
66
67 static const match_table_t tokens = {
68 {Opt_size, "size=%s"},
69 {Opt_nr_inodes, "nr_inodes=%s"},
70 {Opt_mode, "mode=%o"},
71 {Opt_uid, "uid=%u"},
72 {Opt_gid, "gid=%u"},
73 {Opt_pagesize, "pagesize=%s"},
74 {Opt_min_size, "min_size=%s"},
75 {Opt_err, NULL},
76 };
77
78 #ifdef CONFIG_NUMA
79 static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
80 struct inode *inode, pgoff_t index)
81 {
82 vma->vm_policy = mpol_shared_policy_lookup(&HUGETLBFS_I(inode)->policy,
83 index);
84 }
85
86 static inline void hugetlb_drop_vma_policy(struct vm_area_struct *vma)
87 {
88 mpol_cond_put(vma->vm_policy);
89 }
90 #else
91 static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
92 struct inode *inode, pgoff_t index)
93 {
94 }
95
96 static inline void hugetlb_drop_vma_policy(struct vm_area_struct *vma)
97 {
98 }
99 #endif
100
101 static void huge_pagevec_release(struct pagevec *pvec)
102 {
103 int i;
104
105 for (i = 0; i < pagevec_count(pvec); ++i)
106 put_page(pvec->pages[i]);
107
108 pagevec_reinit(pvec);
109 }
110
111 /*
112 * Mask used when checking the page offset value passed in via system
113 * calls. This value will be converted to a loff_t which is signed.
114 * Therefore, we want to check the upper PAGE_SHIFT + 1 bits of the
115 * value. The extra bit (- 1 in the shift value) is to take the sign
116 * bit into account.
117 */
118 #define PGOFF_LOFFT_MAX \
119 (((1UL << (PAGE_SHIFT + 1)) - 1) << (BITS_PER_LONG - (PAGE_SHIFT + 1)))
120
121 static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
122 {
123 struct inode *inode = file_inode(file);
124 loff_t len, vma_len;
125 int ret;
126 struct hstate *h = hstate_file(file);
127
128 /*
129 * vma address alignment (but not the pgoff alignment) has
130 * already been checked by prepare_hugepage_range. If you add
131 * any error returns here, do so after setting VM_HUGETLB, so
132 * is_vm_hugetlb_page tests below unmap_region go the right
133 * way when do_mmap_pgoff unwinds (may be important on powerpc
134 * and ia64).
135 */
136 vma->vm_flags |= VM_HUGETLB | VM_DONTEXPAND;
137 vma->vm_ops = &hugetlb_vm_ops;
138
139 /*
140 * page based offset in vm_pgoff could be sufficiently large to
141 * overflow a loff_t when converted to byte offset. This can
142 * only happen on architectures where sizeof(loff_t) ==
143 * sizeof(unsigned long). So, only check in those instances.
144 */
145 if (sizeof(unsigned long) == sizeof(loff_t)) {
146 if (vma->vm_pgoff & PGOFF_LOFFT_MAX)
147 return -EINVAL;
148 }
149
150 /* must be huge page aligned */
151 if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
152 return -EINVAL;
153
154 vma_len = (loff_t)(vma->vm_end - vma->vm_start);
155 len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
156 /* check for overflow */
157 if (len < vma_len)
158 return -EINVAL;
159
160 inode_lock(inode);
161 file_accessed(file);
162
163 ret = -ENOMEM;
164 if (hugetlb_reserve_pages(inode,
165 vma->vm_pgoff >> huge_page_order(h),
166 len >> huge_page_shift(h), vma,
167 vma->vm_flags))
168 goto out;
169
170 ret = 0;
171 if (vma->vm_flags & VM_WRITE && inode->i_size < len)
172 i_size_write(inode, len);
173 out:
174 inode_unlock(inode);
175
176 return ret;
177 }
178
179 /*
180 * Called under down_write(mmap_sem).
181 */
182
183 #ifndef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
184 static unsigned long
185 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
186 unsigned long len, unsigned long pgoff, unsigned long flags)
187 {
188 struct mm_struct *mm = current->mm;
189 struct vm_area_struct *vma;
190 struct hstate *h = hstate_file(file);
191 struct vm_unmapped_area_info info;
192
193 if (len & ~huge_page_mask(h))
194 return -EINVAL;
195 if (len > TASK_SIZE)
196 return -ENOMEM;
197
198 if (flags & MAP_FIXED) {
199 if (prepare_hugepage_range(file, addr, len))
200 return -EINVAL;
201 return addr;
202 }
203
204 if (addr) {
205 addr = ALIGN(addr, huge_page_size(h));
206 vma = find_vma(mm, addr);
207 if (TASK_SIZE - len >= addr &&
208 (!vma || addr + len <= vm_start_gap(vma)))
209 return addr;
210 }
211
212 info.flags = 0;
213 info.length = len;
214 info.low_limit = TASK_UNMAPPED_BASE;
215 info.high_limit = TASK_SIZE;
216 info.align_mask = PAGE_MASK & ~huge_page_mask(h);
217 info.align_offset = 0;
218 return vm_unmapped_area(&info);
219 }
220 #endif
221
222 static size_t
223 hugetlbfs_read_actor(struct page *page, unsigned long offset,
224 struct iov_iter *to, unsigned long size)
225 {
226 size_t copied = 0;
227 int i, chunksize;
228
229 /* Find which 4k chunk and offset with in that chunk */
230 i = offset >> PAGE_SHIFT;
231 offset = offset & ~PAGE_MASK;
232
233 while (size) {
234 size_t n;
235 chunksize = PAGE_SIZE;
236 if (offset)
237 chunksize -= offset;
238 if (chunksize > size)
239 chunksize = size;
240 n = copy_page_to_iter(&page[i], offset, chunksize, to);
241 copied += n;
242 if (n != chunksize)
243 return copied;
244 offset = 0;
245 size -= chunksize;
246 i++;
247 }
248 return copied;
249 }
250
251 /*
252 * Support for read() - Find the page attached to f_mapping and copy out the
253 * data. Its *very* similar to do_generic_mapping_read(), we can't use that
254 * since it has PAGE_SIZE assumptions.
255 */
256 static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to)
257 {
258 struct file *file = iocb->ki_filp;
259 struct hstate *h = hstate_file(file);
260 struct address_space *mapping = file->f_mapping;
261 struct inode *inode = mapping->host;
262 unsigned long index = iocb->ki_pos >> huge_page_shift(h);
263 unsigned long offset = iocb->ki_pos & ~huge_page_mask(h);
264 unsigned long end_index;
265 loff_t isize;
266 ssize_t retval = 0;
267
268 while (iov_iter_count(to)) {
269 struct page *page;
270 size_t nr, copied;
271
272 /* nr is the maximum number of bytes to copy from this page */
273 nr = huge_page_size(h);
274 isize = i_size_read(inode);
275 if (!isize)
276 break;
277 end_index = (isize - 1) >> huge_page_shift(h);
278 if (index > end_index)
279 break;
280 if (index == end_index) {
281 nr = ((isize - 1) & ~huge_page_mask(h)) + 1;
282 if (nr <= offset)
283 break;
284 }
285 nr = nr - offset;
286
287 /* Find the page */
288 page = find_lock_page(mapping, index);
289 if (unlikely(page == NULL)) {
290 /*
291 * We have a HOLE, zero out the user-buffer for the
292 * length of the hole or request.
293 */
294 copied = iov_iter_zero(nr, to);
295 } else {
296 unlock_page(page);
297
298 /*
299 * We have the page, copy it to user space buffer.
300 */
301 copied = hugetlbfs_read_actor(page, offset, to, nr);
302 put_page(page);
303 }
304 offset += copied;
305 retval += copied;
306 if (copied != nr && iov_iter_count(to)) {
307 if (!retval)
308 retval = -EFAULT;
309 break;
310 }
311 index += offset >> huge_page_shift(h);
312 offset &= ~huge_page_mask(h);
313 }
314 iocb->ki_pos = ((loff_t)index << huge_page_shift(h)) + offset;
315 return retval;
316 }
317
318 static int hugetlbfs_write_begin(struct file *file,
319 struct address_space *mapping,
320 loff_t pos, unsigned len, unsigned flags,
321 struct page **pagep, void **fsdata)
322 {
323 return -EINVAL;
324 }
325
326 static int hugetlbfs_write_end(struct file *file, struct address_space *mapping,
327 loff_t pos, unsigned len, unsigned copied,
328 struct page *page, void *fsdata)
329 {
330 BUG();
331 return -EINVAL;
332 }
333
334 static void remove_huge_page(struct page *page)
335 {
336 ClearPageDirty(page);
337 ClearPageUptodate(page);
338 delete_from_page_cache(page);
339 }
340
341 static void
342 hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end)
343 {
344 struct vm_area_struct *vma;
345
346 /*
347 * end == 0 indicates that the entire range after
348 * start should be unmapped.
349 */
350 vma_interval_tree_foreach(vma, root, start, end ? end : ULONG_MAX) {
351 unsigned long v_offset;
352 unsigned long v_end;
353
354 /*
355 * Can the expression below overflow on 32-bit arches?
356 * No, because the interval tree returns us only those vmas
357 * which overlap the truncated area starting at pgoff,
358 * and no vma on a 32-bit arch can span beyond the 4GB.
359 */
360 if (vma->vm_pgoff < start)
361 v_offset = (start - vma->vm_pgoff) << PAGE_SHIFT;
362 else
363 v_offset = 0;
364
365 if (!end)
366 v_end = vma->vm_end;
367 else {
368 v_end = ((end - vma->vm_pgoff) << PAGE_SHIFT)
369 + vma->vm_start;
370 if (v_end > vma->vm_end)
371 v_end = vma->vm_end;
372 }
373
374 unmap_hugepage_range(vma, vma->vm_start + v_offset, v_end,
375 NULL);
376 }
377 }
378
379 /*
380 * remove_inode_hugepages handles two distinct cases: truncation and hole
381 * punch. There are subtle differences in operation for each case.
382 *
383 * truncation is indicated by end of range being LLONG_MAX
384 * In this case, we first scan the range and release found pages.
385 * After releasing pages, hugetlb_unreserve_pages cleans up region/reserv
386 * maps and global counts. Page faults can not race with truncation
387 * in this routine. hugetlb_no_page() prevents page faults in the
388 * truncated range. It checks i_size before allocation, and again after
389 * with the page table lock for the page held. The same lock must be
390 * acquired to unmap a page.
391 * hole punch is indicated if end is not LLONG_MAX
392 * In the hole punch case we scan the range and release found pages.
393 * Only when releasing a page is the associated region/reserv map
394 * deleted. The region/reserv map for ranges without associated
395 * pages are not modified. Page faults can race with hole punch.
396 * This is indicated if we find a mapped page.
397 * Note: If the passed end of range value is beyond the end of file, but
398 * not LLONG_MAX this routine still performs a hole punch operation.
399 */
400 static void remove_inode_hugepages(struct inode *inode, loff_t lstart,
401 loff_t lend)
402 {
403 struct hstate *h = hstate_inode(inode);
404 struct address_space *mapping = &inode->i_data;
405 const pgoff_t start = lstart >> huge_page_shift(h);
406 const pgoff_t end = lend >> huge_page_shift(h);
407 struct vm_area_struct pseudo_vma;
408 struct pagevec pvec;
409 pgoff_t next, index;
410 int i, freed = 0;
411 bool truncate_op = (lend == LLONG_MAX);
412
413 memset(&pseudo_vma, 0, sizeof(struct vm_area_struct));
414 vma_init(&pseudo_vma, current->mm);
415 pseudo_vma.vm_flags = (VM_HUGETLB | VM_MAYSHARE | VM_SHARED);
416 pagevec_init(&pvec);
417 next = start;
418 while (next < end) {
419 /*
420 * When no more pages are found, we are done.
421 */
422 if (!pagevec_lookup_range(&pvec, mapping, &next, end - 1))
423 break;
424
425 for (i = 0; i < pagevec_count(&pvec); ++i) {
426 struct page *page = pvec.pages[i];
427 u32 hash;
428
429 index = page->index;
430 hash = hugetlb_fault_mutex_hash(h, current->mm,
431 &pseudo_vma,
432 mapping, index, 0);
433 mutex_lock(&hugetlb_fault_mutex_table[hash]);
434
435 /*
436 * If page is mapped, it was faulted in after being
437 * unmapped in caller. Unmap (again) now after taking
438 * the fault mutex. The mutex will prevent faults
439 * until we finish removing the page.
440 *
441 * This race can only happen in the hole punch case.
442 * Getting here in a truncate operation is a bug.
443 */
444 if (unlikely(page_mapped(page))) {
445 BUG_ON(truncate_op);
446
447 i_mmap_lock_write(mapping);
448 hugetlb_vmdelete_list(&mapping->i_mmap,
449 index * pages_per_huge_page(h),
450 (index + 1) * pages_per_huge_page(h));
451 i_mmap_unlock_write(mapping);
452 }
453
454 lock_page(page);
455 /*
456 * We must free the huge page and remove from page
457 * cache (remove_huge_page) BEFORE removing the
458 * region/reserve map (hugetlb_unreserve_pages). In
459 * rare out of memory conditions, removal of the
460 * region/reserve map could fail. Correspondingly,
461 * the subpool and global reserve usage count can need
462 * to be adjusted.
463 */
464 VM_BUG_ON(PagePrivate(page));
465 remove_huge_page(page);
466 freed++;
467 if (!truncate_op) {
468 if (unlikely(hugetlb_unreserve_pages(inode,
469 index, index + 1, 1)))
470 hugetlb_fix_reserve_counts(inode);
471 }
472
473 unlock_page(page);
474 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
475 }
476 huge_pagevec_release(&pvec);
477 cond_resched();
478 }
479
480 if (truncate_op)
481 (void)hugetlb_unreserve_pages(inode, start, LONG_MAX, freed);
482 }
483
484 static void hugetlbfs_evict_inode(struct inode *inode)
485 {
486 struct resv_map *resv_map;
487
488 remove_inode_hugepages(inode, 0, LLONG_MAX);
489 resv_map = (struct resv_map *)inode->i_mapping->private_data;
490 /* root inode doesn't have the resv_map, so we should check it */
491 if (resv_map)
492 resv_map_release(&resv_map->refs);
493 clear_inode(inode);
494 }
495
496 static int hugetlb_vmtruncate(struct inode *inode, loff_t offset)
497 {
498 pgoff_t pgoff;
499 struct address_space *mapping = inode->i_mapping;
500 struct hstate *h = hstate_inode(inode);
501
502 BUG_ON(offset & ~huge_page_mask(h));
503 pgoff = offset >> PAGE_SHIFT;
504
505 i_size_write(inode, offset);
506 i_mmap_lock_write(mapping);
507 if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
508 hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0);
509 i_mmap_unlock_write(mapping);
510 remove_inode_hugepages(inode, offset, LLONG_MAX);
511 return 0;
512 }
513
514 static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
515 {
516 struct hstate *h = hstate_inode(inode);
517 loff_t hpage_size = huge_page_size(h);
518 loff_t hole_start, hole_end;
519
520 /*
521 * For hole punch round up the beginning offset of the hole and
522 * round down the end.
523 */
524 hole_start = round_up(offset, hpage_size);
525 hole_end = round_down(offset + len, hpage_size);
526
527 if (hole_end > hole_start) {
528 struct address_space *mapping = inode->i_mapping;
529 struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
530
531 inode_lock(inode);
532
533 /* protected by i_mutex */
534 if (info->seals & F_SEAL_WRITE) {
535 inode_unlock(inode);
536 return -EPERM;
537 }
538
539 i_mmap_lock_write(mapping);
540 if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
541 hugetlb_vmdelete_list(&mapping->i_mmap,
542 hole_start >> PAGE_SHIFT,
543 hole_end >> PAGE_SHIFT);
544 i_mmap_unlock_write(mapping);
545 remove_inode_hugepages(inode, hole_start, hole_end);
546 inode_unlock(inode);
547 }
548
549 return 0;
550 }
551
552 static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset,
553 loff_t len)
554 {
555 struct inode *inode = file_inode(file);
556 struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
557 struct address_space *mapping = inode->i_mapping;
558 struct hstate *h = hstate_inode(inode);
559 struct vm_area_struct pseudo_vma;
560 struct mm_struct *mm = current->mm;
561 loff_t hpage_size = huge_page_size(h);
562 unsigned long hpage_shift = huge_page_shift(h);
563 pgoff_t start, index, end;
564 int error;
565 u32 hash;
566
567 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
568 return -EOPNOTSUPP;
569
570 if (mode & FALLOC_FL_PUNCH_HOLE)
571 return hugetlbfs_punch_hole(inode, offset, len);
572
573 /*
574 * Default preallocate case.
575 * For this range, start is rounded down and end is rounded up
576 * as well as being converted to page offsets.
577 */
578 start = offset >> hpage_shift;
579 end = (offset + len + hpage_size - 1) >> hpage_shift;
580
581 inode_lock(inode);
582
583 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
584 error = inode_newsize_ok(inode, offset + len);
585 if (error)
586 goto out;
587
588 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
589 error = -EPERM;
590 goto out;
591 }
592
593 /*
594 * Initialize a pseudo vma as this is required by the huge page
595 * allocation routines. If NUMA is configured, use page index
596 * as input to create an allocation policy.
597 */
598 memset(&pseudo_vma, 0, sizeof(struct vm_area_struct));
599 vma_init(&pseudo_vma, mm);
600 pseudo_vma.vm_flags = (VM_HUGETLB | VM_MAYSHARE | VM_SHARED);
601 pseudo_vma.vm_file = file;
602
603 for (index = start; index < end; index++) {
604 /*
605 * This is supposed to be the vaddr where the page is being
606 * faulted in, but we have no vaddr here.
607 */
608 struct page *page;
609 unsigned long addr;
610 int avoid_reserve = 0;
611
612 cond_resched();
613
614 /*
615 * fallocate(2) manpage permits EINTR; we may have been
616 * interrupted because we are using up too much memory.
617 */
618 if (signal_pending(current)) {
619 error = -EINTR;
620 break;
621 }
622
623 /* Set numa allocation policy based on index */
624 hugetlb_set_vma_policy(&pseudo_vma, inode, index);
625
626 /* addr is the offset within the file (zero based) */
627 addr = index * hpage_size;
628
629 /* mutex taken here, fault path and hole punch */
630 hash = hugetlb_fault_mutex_hash(h, mm, &pseudo_vma, mapping,
631 index, addr);
632 mutex_lock(&hugetlb_fault_mutex_table[hash]);
633
634 /* See if already present in mapping to avoid alloc/free */
635 page = find_get_page(mapping, index);
636 if (page) {
637 put_page(page);
638 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
639 hugetlb_drop_vma_policy(&pseudo_vma);
640 continue;
641 }
642
643 /* Allocate page and add to page cache */
644 page = alloc_huge_page(&pseudo_vma, addr, avoid_reserve);
645 hugetlb_drop_vma_policy(&pseudo_vma);
646 if (IS_ERR(page)) {
647 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
648 error = PTR_ERR(page);
649 goto out;
650 }
651 clear_huge_page(page, addr, pages_per_huge_page(h));
652 __SetPageUptodate(page);
653 error = huge_add_to_page_cache(page, mapping, index);
654 if (unlikely(error)) {
655 put_page(page);
656 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
657 goto out;
658 }
659
660 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
661
662 /*
663 * unlock_page because locked by add_to_page_cache()
664 * page_put due to reference from alloc_huge_page()
665 */
666 unlock_page(page);
667 put_page(page);
668 }
669
670 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
671 i_size_write(inode, offset + len);
672 inode->i_ctime = current_time(inode);
673 out:
674 inode_unlock(inode);
675 return error;
676 }
677
678 static int hugetlbfs_setattr(struct dentry *dentry, struct iattr *attr)
679 {
680 struct inode *inode = d_inode(dentry);
681 struct hstate *h = hstate_inode(inode);
682 int error;
683 unsigned int ia_valid = attr->ia_valid;
684 struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
685
686 BUG_ON(!inode);
687
688 error = setattr_prepare(dentry, attr);
689 if (error)
690 return error;
691
692 if (ia_valid & ATTR_SIZE) {
693 loff_t oldsize = inode->i_size;
694 loff_t newsize = attr->ia_size;
695
696 if (newsize & ~huge_page_mask(h))
697 return -EINVAL;
698 /* protected by i_mutex */
699 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
700 (newsize > oldsize && (info->seals & F_SEAL_GROW)))
701 return -EPERM;
702 error = hugetlb_vmtruncate(inode, newsize);
703 if (error)
704 return error;
705 }
706
707 setattr_copy(inode, attr);
708 mark_inode_dirty(inode);
709 return 0;
710 }
711
712 static struct inode *hugetlbfs_get_root(struct super_block *sb,
713 struct hugetlbfs_config *config)
714 {
715 struct inode *inode;
716
717 inode = new_inode(sb);
718 if (inode) {
719 inode->i_ino = get_next_ino();
720 inode->i_mode = S_IFDIR | config->mode;
721 inode->i_uid = config->uid;
722 inode->i_gid = config->gid;
723 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
724 inode->i_op = &hugetlbfs_dir_inode_operations;
725 inode->i_fop = &simple_dir_operations;
726 /* directory inodes start off with i_nlink == 2 (for "." entry) */
727 inc_nlink(inode);
728 lockdep_annotate_inode_mutex_key(inode);
729 }
730 return inode;
731 }
732
733 /*
734 * Hugetlbfs is not reclaimable; therefore its i_mmap_rwsem will never
735 * be taken from reclaim -- unlike regular filesystems. This needs an
736 * annotation because huge_pmd_share() does an allocation under hugetlb's
737 * i_mmap_rwsem.
738 */
739 static struct lock_class_key hugetlbfs_i_mmap_rwsem_key;
740
741 static struct inode *hugetlbfs_get_inode(struct super_block *sb,
742 struct inode *dir,
743 umode_t mode, dev_t dev)
744 {
745 struct inode *inode;
746 struct resv_map *resv_map;
747
748 resv_map = resv_map_alloc();
749 if (!resv_map)
750 return NULL;
751
752 inode = new_inode(sb);
753 if (inode) {
754 struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
755
756 inode->i_ino = get_next_ino();
757 inode_init_owner(inode, dir, mode);
758 lockdep_set_class(&inode->i_mapping->i_mmap_rwsem,
759 &hugetlbfs_i_mmap_rwsem_key);
760 inode->i_mapping->a_ops = &hugetlbfs_aops;
761 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
762 inode->i_mapping->private_data = resv_map;
763 info->seals = F_SEAL_SEAL;
764 switch (mode & S_IFMT) {
765 default:
766 init_special_inode(inode, mode, dev);
767 break;
768 case S_IFREG:
769 inode->i_op = &hugetlbfs_inode_operations;
770 inode->i_fop = &hugetlbfs_file_operations;
771 break;
772 case S_IFDIR:
773 inode->i_op = &hugetlbfs_dir_inode_operations;
774 inode->i_fop = &simple_dir_operations;
775
776 /* directory inodes start off with i_nlink == 2 (for "." entry) */
777 inc_nlink(inode);
778 break;
779 case S_IFLNK:
780 inode->i_op = &page_symlink_inode_operations;
781 inode_nohighmem(inode);
782 break;
783 }
784 lockdep_annotate_inode_mutex_key(inode);
785 } else
786 kref_put(&resv_map->refs, resv_map_release);
787
788 return inode;
789 }
790
791 /*
792 * File creation. Allocate an inode, and we're done..
793 */
794 static int hugetlbfs_mknod(struct inode *dir,
795 struct dentry *dentry, umode_t mode, dev_t dev)
796 {
797 struct inode *inode;
798 int error = -ENOSPC;
799
800 inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);
801 if (inode) {
802 dir->i_ctime = dir->i_mtime = current_time(dir);
803 d_instantiate(dentry, inode);
804 dget(dentry); /* Extra count - pin the dentry in core */
805 error = 0;
806 }
807 return error;
808 }
809
810 static int hugetlbfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
811 {
812 int retval = hugetlbfs_mknod(dir, dentry, mode | S_IFDIR, 0);
813 if (!retval)
814 inc_nlink(dir);
815 return retval;
816 }
817
818 static int hugetlbfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl)
819 {
820 return hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0);
821 }
822
823 static int hugetlbfs_symlink(struct inode *dir,
824 struct dentry *dentry, const char *symname)
825 {
826 struct inode *inode;
827 int error = -ENOSPC;
828
829 inode = hugetlbfs_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0);
830 if (inode) {
831 int l = strlen(symname)+1;
832 error = page_symlink(inode, symname, l);
833 if (!error) {
834 d_instantiate(dentry, inode);
835 dget(dentry);
836 } else
837 iput(inode);
838 }
839 dir->i_ctime = dir->i_mtime = current_time(dir);
840
841 return error;
842 }
843
844 /*
845 * mark the head page dirty
846 */
847 static int hugetlbfs_set_page_dirty(struct page *page)
848 {
849 struct page *head = compound_head(page);
850
851 SetPageDirty(head);
852 return 0;
853 }
854
855 static int hugetlbfs_migrate_page(struct address_space *mapping,
856 struct page *newpage, struct page *page,
857 enum migrate_mode mode)
858 {
859 int rc;
860
861 rc = migrate_huge_page_move_mapping(mapping, newpage, page);
862 if (rc != MIGRATEPAGE_SUCCESS)
863 return rc;
864 if (mode != MIGRATE_SYNC_NO_COPY)
865 migrate_page_copy(newpage, page);
866 else
867 migrate_page_states(newpage, page);
868
869 return MIGRATEPAGE_SUCCESS;
870 }
871
872 static int hugetlbfs_error_remove_page(struct address_space *mapping,
873 struct page *page)
874 {
875 struct inode *inode = mapping->host;
876 pgoff_t index = page->index;
877
878 remove_huge_page(page);
879 if (unlikely(hugetlb_unreserve_pages(inode, index, index + 1, 1)))
880 hugetlb_fix_reserve_counts(inode);
881
882 return 0;
883 }
884
885 /*
886 * Display the mount options in /proc/mounts.
887 */
888 static int hugetlbfs_show_options(struct seq_file *m, struct dentry *root)
889 {
890 struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(root->d_sb);
891 struct hugepage_subpool *spool = sbinfo->spool;
892 unsigned long hpage_size = huge_page_size(sbinfo->hstate);
893 unsigned hpage_shift = huge_page_shift(sbinfo->hstate);
894 char mod;
895
896 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
897 seq_printf(m, ",uid=%u",
898 from_kuid_munged(&init_user_ns, sbinfo->uid));
899 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
900 seq_printf(m, ",gid=%u",
901 from_kgid_munged(&init_user_ns, sbinfo->gid));
902 if (sbinfo->mode != 0755)
903 seq_printf(m, ",mode=%o", sbinfo->mode);
904 if (sbinfo->max_inodes != -1)
905 seq_printf(m, ",nr_inodes=%lu", sbinfo->max_inodes);
906
907 hpage_size /= 1024;
908 mod = 'K';
909 if (hpage_size >= 1024) {
910 hpage_size /= 1024;
911 mod = 'M';
912 }
913 seq_printf(m, ",pagesize=%lu%c", hpage_size, mod);
914 if (spool) {
915 if (spool->max_hpages != -1)
916 seq_printf(m, ",size=%llu",
917 (unsigned long long)spool->max_hpages << hpage_shift);
918 if (spool->min_hpages != -1)
919 seq_printf(m, ",min_size=%llu",
920 (unsigned long long)spool->min_hpages << hpage_shift);
921 }
922 return 0;
923 }
924
925 static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
926 {
927 struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
928 struct hstate *h = hstate_inode(d_inode(dentry));
929
930 buf->f_type = HUGETLBFS_MAGIC;
931 buf->f_bsize = huge_page_size(h);
932 if (sbinfo) {
933 spin_lock(&sbinfo->stat_lock);
934 /* If no limits set, just report 0 for max/free/used
935 * blocks, like simple_statfs() */
936 if (sbinfo->spool) {
937 long free_pages;
938
939 spin_lock(&sbinfo->spool->lock);
940 buf->f_blocks = sbinfo->spool->max_hpages;
941 free_pages = sbinfo->spool->max_hpages
942 - sbinfo->spool->used_hpages;
943 buf->f_bavail = buf->f_bfree = free_pages;
944 spin_unlock(&sbinfo->spool->lock);
945 buf->f_files = sbinfo->max_inodes;
946 buf->f_ffree = sbinfo->free_inodes;
947 }
948 spin_unlock(&sbinfo->stat_lock);
949 }
950 buf->f_namelen = NAME_MAX;
951 return 0;
952 }
953
954 static void hugetlbfs_put_super(struct super_block *sb)
955 {
956 struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);
957
958 if (sbi) {
959 sb->s_fs_info = NULL;
960
961 if (sbi->spool)
962 hugepage_put_subpool(sbi->spool);
963
964 kfree(sbi);
965 }
966 }
967
968 static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
969 {
970 if (sbinfo->free_inodes >= 0) {
971 spin_lock(&sbinfo->stat_lock);
972 if (unlikely(!sbinfo->free_inodes)) {
973 spin_unlock(&sbinfo->stat_lock);
974 return 0;
975 }
976 sbinfo->free_inodes--;
977 spin_unlock(&sbinfo->stat_lock);
978 }
979
980 return 1;
981 }
982
983 static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
984 {
985 if (sbinfo->free_inodes >= 0) {
986 spin_lock(&sbinfo->stat_lock);
987 sbinfo->free_inodes++;
988 spin_unlock(&sbinfo->stat_lock);
989 }
990 }
991
992
993 static struct kmem_cache *hugetlbfs_inode_cachep;
994
995 static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
996 {
997 struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
998 struct hugetlbfs_inode_info *p;
999
1000 if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
1001 return NULL;
1002 p = kmem_cache_alloc(hugetlbfs_inode_cachep, GFP_KERNEL);
1003 if (unlikely(!p)) {
1004 hugetlbfs_inc_free_inodes(sbinfo);
1005 return NULL;
1006 }
1007
1008 /*
1009 * Any time after allocation, hugetlbfs_destroy_inode can be called
1010 * for the inode. mpol_free_shared_policy is unconditionally called
1011 * as part of hugetlbfs_destroy_inode. So, initialize policy here
1012 * in case of a quick call to destroy.
1013 *
1014 * Note that the policy is initialized even if we are creating a
1015 * private inode. This simplifies hugetlbfs_destroy_inode.
1016 */
1017 mpol_shared_policy_init(&p->policy, NULL);
1018
1019 return &p->vfs_inode;
1020 }
1021
1022 static void hugetlbfs_i_callback(struct rcu_head *head)
1023 {
1024 struct inode *inode = container_of(head, struct inode, i_rcu);
1025 kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
1026 }
1027
1028 static void hugetlbfs_destroy_inode(struct inode *inode)
1029 {
1030 hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
1031 mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);
1032 call_rcu(&inode->i_rcu, hugetlbfs_i_callback);
1033 }
1034
1035 static const struct address_space_operations hugetlbfs_aops = {
1036 .write_begin = hugetlbfs_write_begin,
1037 .write_end = hugetlbfs_write_end,
1038 .set_page_dirty = hugetlbfs_set_page_dirty,
1039 .migratepage = hugetlbfs_migrate_page,
1040 .error_remove_page = hugetlbfs_error_remove_page,
1041 };
1042
1043
1044 static void init_once(void *foo)
1045 {
1046 struct hugetlbfs_inode_info *ei = (struct hugetlbfs_inode_info *)foo;
1047
1048 inode_init_once(&ei->vfs_inode);
1049 }
1050
1051 const struct file_operations hugetlbfs_file_operations = {
1052 .read_iter = hugetlbfs_read_iter,
1053 .mmap = hugetlbfs_file_mmap,
1054 .fsync = noop_fsync,
1055 .get_unmapped_area = hugetlb_get_unmapped_area,
1056 .llseek = default_llseek,
1057 .fallocate = hugetlbfs_fallocate,
1058 };
1059
1060 static const struct inode_operations hugetlbfs_dir_inode_operations = {
1061 .create = hugetlbfs_create,
1062 .lookup = simple_lookup,
1063 .link = simple_link,
1064 .unlink = simple_unlink,
1065 .symlink = hugetlbfs_symlink,
1066 .mkdir = hugetlbfs_mkdir,
1067 .rmdir = simple_rmdir,
1068 .mknod = hugetlbfs_mknod,
1069 .rename = simple_rename,
1070 .setattr = hugetlbfs_setattr,
1071 };
1072
1073 static const struct inode_operations hugetlbfs_inode_operations = {
1074 .setattr = hugetlbfs_setattr,
1075 };
1076
1077 static const struct super_operations hugetlbfs_ops = {
1078 .alloc_inode = hugetlbfs_alloc_inode,
1079 .destroy_inode = hugetlbfs_destroy_inode,
1080 .evict_inode = hugetlbfs_evict_inode,
1081 .statfs = hugetlbfs_statfs,
1082 .put_super = hugetlbfs_put_super,
1083 .show_options = hugetlbfs_show_options,
1084 };
1085
1086 enum hugetlbfs_size_type { NO_SIZE, SIZE_STD, SIZE_PERCENT };
1087
1088 /*
1089 * Convert size option passed from command line to number of huge pages
1090 * in the pool specified by hstate. Size option could be in bytes
1091 * (val_type == SIZE_STD) or percentage of the pool (val_type == SIZE_PERCENT).
1092 */
1093 static long
1094 hugetlbfs_size_to_hpages(struct hstate *h, unsigned long long size_opt,
1095 enum hugetlbfs_size_type val_type)
1096 {
1097 if (val_type == NO_SIZE)
1098 return -1;
1099
1100 if (val_type == SIZE_PERCENT) {
1101 size_opt <<= huge_page_shift(h);
1102 size_opt *= h->max_huge_pages;
1103 do_div(size_opt, 100);
1104 }
1105
1106 size_opt >>= huge_page_shift(h);
1107 return size_opt;
1108 }
1109
1110 static int
1111 hugetlbfs_parse_options(char *options, struct hugetlbfs_config *pconfig)
1112 {
1113 char *p, *rest;
1114 substring_t args[MAX_OPT_ARGS];
1115 int option;
1116 unsigned long long max_size_opt = 0, min_size_opt = 0;
1117 enum hugetlbfs_size_type max_val_type = NO_SIZE, min_val_type = NO_SIZE;
1118
1119 if (!options)
1120 return 0;
1121
1122 while ((p = strsep(&options, ",")) != NULL) {
1123 int token;
1124 if (!*p)
1125 continue;
1126
1127 token = match_token(p, tokens, args);
1128 switch (token) {
1129 case Opt_uid:
1130 if (match_int(&args[0], &option))
1131 goto bad_val;
1132 pconfig->uid = make_kuid(current_user_ns(), option);
1133 if (!uid_valid(pconfig->uid))
1134 goto bad_val;
1135 break;
1136
1137 case Opt_gid:
1138 if (match_int(&args[0], &option))
1139 goto bad_val;
1140 pconfig->gid = make_kgid(current_user_ns(), option);
1141 if (!gid_valid(pconfig->gid))
1142 goto bad_val;
1143 break;
1144
1145 case Opt_mode:
1146 if (match_octal(&args[0], &option))
1147 goto bad_val;
1148 pconfig->mode = option & 01777U;
1149 break;
1150
1151 case Opt_size: {
1152 /* memparse() will accept a K/M/G without a digit */
1153 if (!isdigit(*args[0].from))
1154 goto bad_val;
1155 max_size_opt = memparse(args[0].from, &rest);
1156 max_val_type = SIZE_STD;
1157 if (*rest == '%')
1158 max_val_type = SIZE_PERCENT;
1159 break;
1160 }
1161
1162 case Opt_nr_inodes:
1163 /* memparse() will accept a K/M/G without a digit */
1164 if (!isdigit(*args[0].from))
1165 goto bad_val;
1166 pconfig->nr_inodes = memparse(args[0].from, &rest);
1167 break;
1168
1169 case Opt_pagesize: {
1170 unsigned long ps;
1171 ps = memparse(args[0].from, &rest);
1172 pconfig->hstate = size_to_hstate(ps);
1173 if (!pconfig->hstate) {
1174 pr_err("Unsupported page size %lu MB\n",
1175 ps >> 20);
1176 return -EINVAL;
1177 }
1178 break;
1179 }
1180
1181 case Opt_min_size: {
1182 /* memparse() will accept a K/M/G without a digit */
1183 if (!isdigit(*args[0].from))
1184 goto bad_val;
1185 min_size_opt = memparse(args[0].from, &rest);
1186 min_val_type = SIZE_STD;
1187 if (*rest == '%')
1188 min_val_type = SIZE_PERCENT;
1189 break;
1190 }
1191
1192 default:
1193 pr_err("Bad mount option: \"%s\"\n", p);
1194 return -EINVAL;
1195 break;
1196 }
1197 }
1198
1199 /*
1200 * Use huge page pool size (in hstate) to convert the size
1201 * options to number of huge pages. If NO_SIZE, -1 is returned.
1202 */
1203 pconfig->max_hpages = hugetlbfs_size_to_hpages(pconfig->hstate,
1204 max_size_opt, max_val_type);
1205 pconfig->min_hpages = hugetlbfs_size_to_hpages(pconfig->hstate,
1206 min_size_opt, min_val_type);
1207
1208 /*
1209 * If max_size was specified, then min_size must be smaller
1210 */
1211 if (max_val_type > NO_SIZE &&
1212 pconfig->min_hpages > pconfig->max_hpages) {
1213 pr_err("minimum size can not be greater than maximum size\n");
1214 return -EINVAL;
1215 }
1216
1217 return 0;
1218
1219 bad_val:
1220 pr_err("Bad value '%s' for mount option '%s'\n", args[0].from, p);
1221 return -EINVAL;
1222 }
1223
1224 static int
1225 hugetlbfs_fill_super(struct super_block *sb, void *data, int silent)
1226 {
1227 int ret;
1228 struct hugetlbfs_config config;
1229 struct hugetlbfs_sb_info *sbinfo;
1230
1231 config.max_hpages = -1; /* No limit on size by default */
1232 config.nr_inodes = -1; /* No limit on number of inodes by default */
1233 config.uid = current_fsuid();
1234 config.gid = current_fsgid();
1235 config.mode = 0755;
1236 config.hstate = &default_hstate;
1237 config.min_hpages = -1; /* No default minimum size */
1238 ret = hugetlbfs_parse_options(data, &config);
1239 if (ret)
1240 return ret;
1241
1242 sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
1243 if (!sbinfo)
1244 return -ENOMEM;
1245 sb->s_fs_info = sbinfo;
1246 sbinfo->hstate = config.hstate;
1247 spin_lock_init(&sbinfo->stat_lock);
1248 sbinfo->max_inodes = config.nr_inodes;
1249 sbinfo->free_inodes = config.nr_inodes;
1250 sbinfo->spool = NULL;
1251 sbinfo->uid = config.uid;
1252 sbinfo->gid = config.gid;
1253 sbinfo->mode = config.mode;
1254
1255 /*
1256 * Allocate and initialize subpool if maximum or minimum size is
1257 * specified. Any needed reservations (for minimim size) are taken
1258 * taken when the subpool is created.
1259 */
1260 if (config.max_hpages != -1 || config.min_hpages != -1) {
1261 sbinfo->spool = hugepage_new_subpool(config.hstate,
1262 config.max_hpages,
1263 config.min_hpages);
1264 if (!sbinfo->spool)
1265 goto out_free;
1266 }
1267 sb->s_maxbytes = MAX_LFS_FILESIZE;
1268 sb->s_blocksize = huge_page_size(config.hstate);
1269 sb->s_blocksize_bits = huge_page_shift(config.hstate);
1270 sb->s_magic = HUGETLBFS_MAGIC;
1271 sb->s_op = &hugetlbfs_ops;
1272 sb->s_time_gran = 1;
1273 sb->s_root = d_make_root(hugetlbfs_get_root(sb, &config));
1274 if (!sb->s_root)
1275 goto out_free;
1276 return 0;
1277 out_free:
1278 kfree(sbinfo->spool);
1279 kfree(sbinfo);
1280 return -ENOMEM;
1281 }
1282
1283 static struct dentry *hugetlbfs_mount(struct file_system_type *fs_type,
1284 int flags, const char *dev_name, void *data)
1285 {
1286 return mount_nodev(fs_type, flags, data, hugetlbfs_fill_super);
1287 }
1288
1289 static struct file_system_type hugetlbfs_fs_type = {
1290 .name = "hugetlbfs",
1291 .mount = hugetlbfs_mount,
1292 .kill_sb = kill_litter_super,
1293 };
1294
1295 static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];
1296
1297 static int can_do_hugetlb_shm(void)
1298 {
1299 kgid_t shm_group;
1300 shm_group = make_kgid(&init_user_ns, sysctl_hugetlb_shm_group);
1301 return capable(CAP_IPC_LOCK) || in_group_p(shm_group);
1302 }
1303
1304 static int get_hstate_idx(int page_size_log)
1305 {
1306 struct hstate *h = hstate_sizelog(page_size_log);
1307
1308 if (!h)
1309 return -1;
1310 return h - hstates;
1311 }
1312
1313 static const struct dentry_operations anon_ops = {
1314 .d_dname = simple_dname
1315 };
1316
1317 /*
1318 * Note that size should be aligned to proper hugepage size in caller side,
1319 * otherwise hugetlb_reserve_pages reserves one less hugepages than intended.
1320 */
1321 struct file *hugetlb_file_setup(const char *name, size_t size,
1322 vm_flags_t acctflag, struct user_struct **user,
1323 int creat_flags, int page_size_log)
1324 {
1325 struct file *file = ERR_PTR(-ENOMEM);
1326 struct inode *inode;
1327 struct path path;
1328 struct super_block *sb;
1329 struct qstr quick_string;
1330 int hstate_idx;
1331
1332 hstate_idx = get_hstate_idx(page_size_log);
1333 if (hstate_idx < 0)
1334 return ERR_PTR(-ENODEV);
1335
1336 *user = NULL;
1337 if (!hugetlbfs_vfsmount[hstate_idx])
1338 return ERR_PTR(-ENOENT);
1339
1340 if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) {
1341 *user = current_user();
1342 if (user_shm_lock(size, *user)) {
1343 task_lock(current);
1344 pr_warn_once("%s (%d): Using mlock ulimits for SHM_HUGETLB is deprecated\n",
1345 current->comm, current->pid);
1346 task_unlock(current);
1347 } else {
1348 *user = NULL;
1349 return ERR_PTR(-EPERM);
1350 }
1351 }
1352
1353 sb = hugetlbfs_vfsmount[hstate_idx]->mnt_sb;
1354 quick_string.name = name;
1355 quick_string.len = strlen(quick_string.name);
1356 quick_string.hash = 0;
1357 path.dentry = d_alloc_pseudo(sb, &quick_string);
1358 if (!path.dentry)
1359 goto out_shm_unlock;
1360
1361 d_set_d_op(path.dentry, &anon_ops);
1362 path.mnt = mntget(hugetlbfs_vfsmount[hstate_idx]);
1363 file = ERR_PTR(-ENOSPC);
1364 inode = hugetlbfs_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0);
1365 if (!inode)
1366 goto out_dentry;
1367 if (creat_flags == HUGETLB_SHMFS_INODE)
1368 inode->i_flags |= S_PRIVATE;
1369
1370 file = ERR_PTR(-ENOMEM);
1371 if (hugetlb_reserve_pages(inode, 0,
1372 size >> huge_page_shift(hstate_inode(inode)), NULL,
1373 acctflag))
1374 goto out_inode;
1375
1376 d_instantiate(path.dentry, inode);
1377 inode->i_size = size;
1378 clear_nlink(inode);
1379
1380 file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
1381 &hugetlbfs_file_operations);
1382 if (IS_ERR(file))
1383 goto out_dentry; /* inode is already attached */
1384
1385 return file;
1386
1387 out_inode:
1388 iput(inode);
1389 out_dentry:
1390 path_put(&path);
1391 out_shm_unlock:
1392 if (*user) {
1393 user_shm_unlock(size, *user);
1394 *user = NULL;
1395 }
1396 return file;
1397 }
1398
1399 static int __init init_hugetlbfs_fs(void)
1400 {
1401 struct hstate *h;
1402 int error;
1403 int i;
1404
1405 if (!hugepages_supported()) {
1406 pr_info("disabling because there are no supported hugepage sizes\n");
1407 return -ENOTSUPP;
1408 }
1409
1410 error = -ENOMEM;
1411 hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
1412 sizeof(struct hugetlbfs_inode_info),
1413 0, SLAB_ACCOUNT, init_once);
1414 if (hugetlbfs_inode_cachep == NULL)
1415 goto out2;
1416
1417 error = register_filesystem(&hugetlbfs_fs_type);
1418 if (error)
1419 goto out;
1420
1421 i = 0;
1422 for_each_hstate(h) {
1423 char buf[50];
1424 unsigned ps_kb = 1U << (h->order + PAGE_SHIFT - 10);
1425
1426 snprintf(buf, sizeof(buf), "pagesize=%uK", ps_kb);
1427 hugetlbfs_vfsmount[i] = kern_mount_data(&hugetlbfs_fs_type,
1428 buf);
1429
1430 if (IS_ERR(hugetlbfs_vfsmount[i])) {
1431 pr_err("Cannot mount internal hugetlbfs for "
1432 "page size %uK", ps_kb);
1433 error = PTR_ERR(hugetlbfs_vfsmount[i]);
1434 hugetlbfs_vfsmount[i] = NULL;
1435 }
1436 i++;
1437 }
1438 /* Non default hstates are optional */
1439 if (!IS_ERR_OR_NULL(hugetlbfs_vfsmount[default_hstate_idx]))
1440 return 0;
1441
1442 out:
1443 kmem_cache_destroy(hugetlbfs_inode_cachep);
1444 out2:
1445 return error;
1446 }
1447 fs_initcall(init_hugetlbfs_fs)
Linux with added FPC-III support