io_uring/filetable.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/kernel.h>
   3 #include <linux/errno.h>
   4 #include <linux/file.h>
   5 #include <linux/mm.h>
   6 #include <linux/slab.h>
   7 #include <linux/nospec.h>
   8 #include <linux/io_uring.h>
   9
  10 #include <uapi/linux/io_uring.h>
  11
  12 #include "io_uring.h"
  13 #include "rsrc.h"
  14 #include "filetable.h"
  15
  16 static int io_file_bitmap_get(struct io_ring_ctx *ctx)
  17 {
  18         struct io_file_table *table = &ctx->file_table;
  19         unsigned long nr = ctx->file_alloc_end;
  20         int ret;
  21
  22         if (!table->bitmap)
  23                 return -ENFILE;
  24
  25         do {
  26                 ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);
  27                 if (ret != nr)
  28                         return ret;
  29
  30                 if (table->alloc_hint == ctx->file_alloc_start)
  31                         break;
  32                 nr = table->alloc_hint;
  33                 table->alloc_hint = ctx->file_alloc_start;
  34         } while (1);
  35
  36         return -ENFILE;
  37 }
  38
  39 bool io_alloc_file_tables(struct io_ring_ctx *ctx, struct io_file_table *table,
  40                           unsigned nr_files)
  41 {
  42         if (io_rsrc_data_alloc(&table->data, nr_files))
  43                 return false;
  44         table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);
  45         if (table->bitmap)
  46                 return true;
  47         io_rsrc_data_free(ctx, &table->data);
  48         return false;
  49 }
  50
  51 void io_free_file_tables(struct io_ring_ctx *ctx, struct io_file_table *table)
  52 {
  53         io_rsrc_data_free(ctx, &table->data);
  54         bitmap_free(table->bitmap);
  55         table->bitmap = NULL;
  56 }
  57
  58 static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
  59                                  u32 slot_index)
  60         __must_hold(&req->ctx->uring_lock)
  61 {
  62         struct io_rsrc_node *node;
  63
  64         if (io_is_uring_fops(file))
  65                 return -EBADF;
  66         if (!ctx->file_table.data.nr)
  67                 return -ENXIO;
  68         if (slot_index >= ctx->file_table.data.nr)
  69                 return -EINVAL;
  70
  71         node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE);
  72         if (!node)
  73                 return -ENOMEM;
  74
  75         if (!io_reset_rsrc_node(ctx, &ctx->file_table.data, slot_index))
  76                 io_file_bitmap_set(&ctx->file_table, slot_index);
  77
  78         ctx->file_table.data.nodes[slot_index] = node;
  79         io_fixed_file_set(node, file);
  80         return 0;
  81 }
  82
  83 int __io_fixed_fd_install(struct io_ring_ctx *ctx, struct file *file,
  84                           unsigned int file_slot)
  85 {
  86         bool alloc_slot = file_slot == IORING_FILE_INDEX_ALLOC;
  87         int ret;
  88
  89         if (alloc_slot) {
  90                 ret = io_file_bitmap_get(ctx);
  91                 if (unlikely(ret < 0))
  92                         return ret;
  93                 file_slot = ret;
  94         } else {
  95                 file_slot--;
  96         }
  97
  98         ret = io_install_fixed_file(ctx, file, file_slot);
  99         if (!ret && alloc_slot)
 100                 ret = file_slot;
 101         return ret;
 102 }
 103 /*
 104  * Note when io_fixed_fd_install() returns error value, it will ensure
 105  * fput() is called correspondingly.
 106  */
 107 int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags,
 108                         struct file *file, unsigned int file_slot)
 109 {
 110         struct io_ring_ctx *ctx = req->ctx;
 111         int ret;
 112
 113         io_ring_submit_lock(ctx, issue_flags);
 114         ret = __io_fixed_fd_install(ctx, file, file_slot);
 115         io_ring_submit_unlock(ctx, issue_flags);
 116
 117         if (unlikely(ret < 0))
 118                 fput(file);
 119         return ret;
 120 }
 121
 122 int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
 123 {
 124         struct io_rsrc_node *node;
 125
 126         if (unlikely(!ctx->file_table.data.nr))
 127                 return -ENXIO;
 128         if (offset >= ctx->file_table.data.nr)
 129                 return -EINVAL;
 130
 131         node = io_rsrc_node_lookup(&ctx->file_table.data, offset);
 132         if (!node)
 133                 return -EBADF;
 134         io_reset_rsrc_node(ctx, &ctx->file_table.data, offset);
 135         io_file_bitmap_clear(&ctx->file_table, offset);
 136         return 0;
 137 }
 138
 139 int io_register_file_alloc_range(struct io_ring_ctx *ctx,
 140                                  struct io_uring_file_index_range __user *arg)
 141 {
 142         struct io_uring_file_index_range range;
 143         u32 end;
 144
 145         if (copy_from_user(&range, arg, sizeof(range)))
 146                 return -EFAULT;
 147         if (check_add_overflow(range.off, range.len, &end))
 148                 return -EOVERFLOW;
 149         if (range.resv || end > ctx->file_table.data.nr)
 150                 return -EINVAL;
 151
 152         io_file_table_set_alloc_range(ctx, range.off, range.len);
 153         return 0;
 154 }