1 // SPDX-License-Identifier: GPL-2.0
3 * io_uring opcode handling table
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
26 #include "openclose.h"
27 #include "uring_cmd.h"
40 static int io_no_issue(struct io_kiocb
*req
, unsigned int issue_flags
)
46 static __maybe_unused
int io_eopnotsupp_prep(struct io_kiocb
*kiocb
,
47 const struct io_uring_sqe
*sqe
)
52 const struct io_issue_def io_issue_defs
[] = {
61 .unbound_nonreg_file
= 1,
70 .async_size
= sizeof(struct io_async_rw
),
71 .prep
= io_prep_readv
,
74 [IORING_OP_WRITEV
] = {
77 .unbound_nonreg_file
= 1,
85 .async_size
= sizeof(struct io_async_rw
),
86 .prep
= io_prep_writev
,
92 .prep
= io_fsync_prep
,
95 [IORING_OP_READ_FIXED
] = {
97 .unbound_nonreg_file
= 1,
104 .async_size
= sizeof(struct io_async_rw
),
105 .prep
= io_prep_read_fixed
,
108 [IORING_OP_WRITE_FIXED
] = {
111 .unbound_nonreg_file
= 1,
118 .async_size
= sizeof(struct io_async_rw
),
119 .prep
= io_prep_write_fixed
,
122 [IORING_OP_POLL_ADD
] = {
124 .unbound_nonreg_file
= 1,
126 .prep
= io_poll_add_prep
,
127 .issue
= io_poll_add
,
129 [IORING_OP_POLL_REMOVE
] = {
131 .prep
= io_poll_remove_prep
,
132 .issue
= io_poll_remove
,
134 [IORING_OP_SYNC_FILE_RANGE
] = {
138 .issue
= io_sync_file_range
,
140 [IORING_OP_SENDMSG
] = {
142 .unbound_nonreg_file
= 1,
145 #if defined(CONFIG_NET)
146 .async_size
= sizeof(struct io_async_msghdr
),
147 .prep
= io_sendmsg_prep
,
150 .prep
= io_eopnotsupp_prep
,
153 [IORING_OP_RECVMSG
] = {
155 .unbound_nonreg_file
= 1,
159 #if defined(CONFIG_NET)
160 .async_size
= sizeof(struct io_async_msghdr
),
161 .prep
= io_recvmsg_prep
,
164 .prep
= io_eopnotsupp_prep
,
167 [IORING_OP_TIMEOUT
] = {
169 .async_size
= sizeof(struct io_timeout_data
),
170 .prep
= io_timeout_prep
,
173 [IORING_OP_TIMEOUT_REMOVE
] = {
174 /* used by timeout updates' prep() */
176 .prep
= io_timeout_remove_prep
,
177 .issue
= io_timeout_remove
,
179 [IORING_OP_ACCEPT
] = {
181 .unbound_nonreg_file
= 1,
184 .ioprio
= 1, /* used for flags */
185 #if defined(CONFIG_NET)
186 .prep
= io_accept_prep
,
189 .prep
= io_eopnotsupp_prep
,
192 [IORING_OP_ASYNC_CANCEL
] = {
194 .prep
= io_async_cancel_prep
,
195 .issue
= io_async_cancel
,
197 [IORING_OP_LINK_TIMEOUT
] = {
199 .async_size
= sizeof(struct io_timeout_data
),
200 .prep
= io_link_timeout_prep
,
201 .issue
= io_no_issue
,
203 [IORING_OP_CONNECT
] = {
205 .unbound_nonreg_file
= 1,
207 #if defined(CONFIG_NET)
208 .async_size
= sizeof(struct io_async_msghdr
),
209 .prep
= io_connect_prep
,
212 .prep
= io_eopnotsupp_prep
,
215 [IORING_OP_FALLOCATE
] = {
217 .prep
= io_fallocate_prep
,
218 .issue
= io_fallocate
,
220 [IORING_OP_OPENAT
] = {
221 .prep
= io_openat_prep
,
224 [IORING_OP_CLOSE
] = {
225 .prep
= io_close_prep
,
228 [IORING_OP_FILES_UPDATE
] = {
231 .prep
= io_files_update_prep
,
232 .issue
= io_files_update
,
234 [IORING_OP_STATX
] = {
236 .prep
= io_statx_prep
,
241 .unbound_nonreg_file
= 1,
249 .async_size
= sizeof(struct io_async_rw
),
250 .prep
= io_prep_read
,
253 [IORING_OP_WRITE
] = {
256 .unbound_nonreg_file
= 1,
263 .async_size
= sizeof(struct io_async_rw
),
264 .prep
= io_prep_write
,
267 [IORING_OP_FADVISE
] = {
270 .prep
= io_fadvise_prep
,
273 [IORING_OP_MADVISE
] = {
275 .prep
= io_madvise_prep
,
280 .unbound_nonreg_file
= 1,
285 #if defined(CONFIG_NET)
286 .async_size
= sizeof(struct io_async_msghdr
),
287 .prep
= io_sendmsg_prep
,
290 .prep
= io_eopnotsupp_prep
,
295 .unbound_nonreg_file
= 1,
300 #if defined(CONFIG_NET)
301 .async_size
= sizeof(struct io_async_msghdr
),
302 .prep
= io_recvmsg_prep
,
305 .prep
= io_eopnotsupp_prep
,
308 [IORING_OP_OPENAT2
] = {
309 .prep
= io_openat2_prep
,
312 [IORING_OP_EPOLL_CTL
] = {
313 .unbound_nonreg_file
= 1,
315 #if defined(CONFIG_EPOLL)
316 .prep
= io_epoll_ctl_prep
,
317 .issue
= io_epoll_ctl
,
319 .prep
= io_eopnotsupp_prep
,
322 [IORING_OP_SPLICE
] = {
325 .unbound_nonreg_file
= 1,
327 .prep
= io_splice_prep
,
330 [IORING_OP_PROVIDE_BUFFERS
] = {
333 .prep
= io_provide_buffers_prep
,
334 .issue
= io_provide_buffers
,
336 [IORING_OP_REMOVE_BUFFERS
] = {
339 .prep
= io_remove_buffers_prep
,
340 .issue
= io_remove_buffers
,
345 .unbound_nonreg_file
= 1,
350 [IORING_OP_SHUTDOWN
] = {
352 #if defined(CONFIG_NET)
353 .prep
= io_shutdown_prep
,
354 .issue
= io_shutdown
,
356 .prep
= io_eopnotsupp_prep
,
359 [IORING_OP_RENAMEAT
] = {
360 .prep
= io_renameat_prep
,
361 .issue
= io_renameat
,
363 [IORING_OP_UNLINKAT
] = {
364 .prep
= io_unlinkat_prep
,
365 .issue
= io_unlinkat
,
367 [IORING_OP_MKDIRAT
] = {
368 .prep
= io_mkdirat_prep
,
371 [IORING_OP_SYMLINKAT
] = {
372 .prep
= io_symlinkat_prep
,
373 .issue
= io_symlinkat
,
375 [IORING_OP_LINKAT
] = {
376 .prep
= io_linkat_prep
,
379 [IORING_OP_MSG_RING
] = {
382 .prep
= io_msg_ring_prep
,
383 .issue
= io_msg_ring
,
385 [IORING_OP_FSETXATTR
] = {
387 .prep
= io_fsetxattr_prep
,
388 .issue
= io_fsetxattr
,
390 [IORING_OP_SETXATTR
] = {
391 .prep
= io_setxattr_prep
,
392 .issue
= io_setxattr
,
394 [IORING_OP_FGETXATTR
] = {
396 .prep
= io_fgetxattr_prep
,
397 .issue
= io_fgetxattr
,
399 [IORING_OP_GETXATTR
] = {
400 .prep
= io_getxattr_prep
,
401 .issue
= io_getxattr
,
403 [IORING_OP_SOCKET
] = {
405 #if defined(CONFIG_NET)
406 .prep
= io_socket_prep
,
409 .prep
= io_eopnotsupp_prep
,
412 [IORING_OP_URING_CMD
] = {
417 .async_size
= 2 * sizeof(struct io_uring_sqe
),
418 .prep
= io_uring_cmd_prep
,
419 .issue
= io_uring_cmd
,
421 [IORING_OP_SEND_ZC
] = {
423 .unbound_nonreg_file
= 1,
427 #if defined(CONFIG_NET)
428 .async_size
= sizeof(struct io_async_msghdr
),
429 .prep
= io_send_zc_prep
,
432 .prep
= io_eopnotsupp_prep
,
435 [IORING_OP_SENDMSG_ZC
] = {
437 .unbound_nonreg_file
= 1,
440 #if defined(CONFIG_NET)
441 .async_size
= sizeof(struct io_async_msghdr
),
442 .prep
= io_send_zc_prep
,
443 .issue
= io_sendmsg_zc
,
445 .prep
= io_eopnotsupp_prep
,
448 [IORING_OP_READ_MULTISHOT
] = {
450 .unbound_nonreg_file
= 1,
454 .async_size
= sizeof(struct io_async_rw
),
455 .prep
= io_read_mshot_prep
,
456 .issue
= io_read_mshot
,
458 [IORING_OP_WAITID
] = {
459 .async_size
= sizeof(struct io_waitid_async
),
460 .prep
= io_waitid_prep
,
463 [IORING_OP_FUTEX_WAIT
] = {
464 #if defined(CONFIG_FUTEX)
465 .prep
= io_futex_prep
,
466 .issue
= io_futex_wait
,
468 .prep
= io_eopnotsupp_prep
,
471 [IORING_OP_FUTEX_WAKE
] = {
472 #if defined(CONFIG_FUTEX)
473 .prep
= io_futex_prep
,
474 .issue
= io_futex_wake
,
476 .prep
= io_eopnotsupp_prep
,
479 [IORING_OP_FUTEX_WAITV
] = {
480 #if defined(CONFIG_FUTEX)
481 .prep
= io_futexv_prep
,
482 .issue
= io_futexv_wait
,
484 .prep
= io_eopnotsupp_prep
,
487 [IORING_OP_FIXED_FD_INSTALL
] = {
489 .prep
= io_install_fixed_fd_prep
,
490 .issue
= io_install_fixed_fd
,
492 [IORING_OP_FTRUNCATE
] = {
495 .prep
= io_ftruncate_prep
,
496 .issue
= io_ftruncate
,
499 #if defined(CONFIG_NET)
501 .prep
= io_bind_prep
,
503 .async_size
= sizeof(struct io_async_msghdr
),
505 .prep
= io_eopnotsupp_prep
,
508 [IORING_OP_LISTEN
] = {
509 #if defined(CONFIG_NET)
511 .prep
= io_listen_prep
,
513 .async_size
= sizeof(struct io_async_msghdr
),
515 .prep
= io_eopnotsupp_prep
,
520 const struct io_cold_def io_cold_defs
[] = {
524 [IORING_OP_READV
] = {
526 .cleanup
= io_readv_writev_cleanup
,
529 [IORING_OP_WRITEV
] = {
531 .cleanup
= io_readv_writev_cleanup
,
534 [IORING_OP_FSYNC
] = {
537 [IORING_OP_READ_FIXED
] = {
538 .name
= "READ_FIXED",
539 .cleanup
= io_readv_writev_cleanup
,
542 [IORING_OP_WRITE_FIXED
] = {
543 .name
= "WRITE_FIXED",
544 .cleanup
= io_readv_writev_cleanup
,
547 [IORING_OP_POLL_ADD
] = {
550 [IORING_OP_POLL_REMOVE
] = {
551 .name
= "POLL_REMOVE",
553 [IORING_OP_SYNC_FILE_RANGE
] = {
554 .name
= "SYNC_FILE_RANGE",
556 [IORING_OP_SENDMSG
] = {
558 #if defined(CONFIG_NET)
559 .cleanup
= io_sendmsg_recvmsg_cleanup
,
560 .fail
= io_sendrecv_fail
,
563 [IORING_OP_RECVMSG
] = {
565 #if defined(CONFIG_NET)
566 .cleanup
= io_sendmsg_recvmsg_cleanup
,
567 .fail
= io_sendrecv_fail
,
570 [IORING_OP_TIMEOUT
] = {
573 [IORING_OP_TIMEOUT_REMOVE
] = {
574 .name
= "TIMEOUT_REMOVE",
576 [IORING_OP_ACCEPT
] = {
579 [IORING_OP_ASYNC_CANCEL
] = {
580 .name
= "ASYNC_CANCEL",
582 [IORING_OP_LINK_TIMEOUT
] = {
583 .name
= "LINK_TIMEOUT",
585 [IORING_OP_CONNECT
] = {
588 [IORING_OP_FALLOCATE
] = {
591 [IORING_OP_OPENAT
] = {
593 .cleanup
= io_open_cleanup
,
595 [IORING_OP_CLOSE
] = {
598 [IORING_OP_FILES_UPDATE
] = {
599 .name
= "FILES_UPDATE",
601 [IORING_OP_STATX
] = {
603 .cleanup
= io_statx_cleanup
,
607 .cleanup
= io_readv_writev_cleanup
,
610 [IORING_OP_WRITE
] = {
612 .cleanup
= io_readv_writev_cleanup
,
615 [IORING_OP_FADVISE
] = {
618 [IORING_OP_MADVISE
] = {
623 #if defined(CONFIG_NET)
624 .cleanup
= io_sendmsg_recvmsg_cleanup
,
625 .fail
= io_sendrecv_fail
,
630 #if defined(CONFIG_NET)
631 .cleanup
= io_sendmsg_recvmsg_cleanup
,
632 .fail
= io_sendrecv_fail
,
635 [IORING_OP_OPENAT2
] = {
637 .cleanup
= io_open_cleanup
,
639 [IORING_OP_EPOLL_CTL
] = {
642 [IORING_OP_SPLICE
] = {
645 [IORING_OP_PROVIDE_BUFFERS
] = {
646 .name
= "PROVIDE_BUFFERS",
648 [IORING_OP_REMOVE_BUFFERS
] = {
649 .name
= "REMOVE_BUFFERS",
654 [IORING_OP_SHUTDOWN
] = {
657 [IORING_OP_RENAMEAT
] = {
659 .cleanup
= io_renameat_cleanup
,
661 [IORING_OP_UNLINKAT
] = {
663 .cleanup
= io_unlinkat_cleanup
,
665 [IORING_OP_MKDIRAT
] = {
667 .cleanup
= io_mkdirat_cleanup
,
669 [IORING_OP_SYMLINKAT
] = {
671 .cleanup
= io_link_cleanup
,
673 [IORING_OP_LINKAT
] = {
675 .cleanup
= io_link_cleanup
,
677 [IORING_OP_MSG_RING
] = {
679 .cleanup
= io_msg_ring_cleanup
,
681 [IORING_OP_FSETXATTR
] = {
683 .cleanup
= io_xattr_cleanup
,
685 [IORING_OP_SETXATTR
] = {
687 .cleanup
= io_xattr_cleanup
,
689 [IORING_OP_FGETXATTR
] = {
691 .cleanup
= io_xattr_cleanup
,
693 [IORING_OP_GETXATTR
] = {
695 .cleanup
= io_xattr_cleanup
,
697 [IORING_OP_SOCKET
] = {
700 [IORING_OP_URING_CMD
] = {
703 [IORING_OP_SEND_ZC
] = {
705 #if defined(CONFIG_NET)
706 .cleanup
= io_send_zc_cleanup
,
707 .fail
= io_sendrecv_fail
,
710 [IORING_OP_SENDMSG_ZC
] = {
711 .name
= "SENDMSG_ZC",
712 #if defined(CONFIG_NET)
713 .cleanup
= io_send_zc_cleanup
,
714 .fail
= io_sendrecv_fail
,
717 [IORING_OP_READ_MULTISHOT
] = {
718 .name
= "READ_MULTISHOT",
719 .cleanup
= io_readv_writev_cleanup
,
721 [IORING_OP_WAITID
] = {
724 [IORING_OP_FUTEX_WAIT
] = {
725 .name
= "FUTEX_WAIT",
727 [IORING_OP_FUTEX_WAKE
] = {
728 .name
= "FUTEX_WAKE",
730 [IORING_OP_FUTEX_WAITV
] = {
731 .name
= "FUTEX_WAITV",
733 [IORING_OP_FIXED_FD_INSTALL
] = {
734 .name
= "FIXED_FD_INSTALL",
736 [IORING_OP_FTRUNCATE
] = {
742 [IORING_OP_LISTEN
] = {
747 const char *io_uring_get_opcode(u8 opcode
)
749 if (opcode
< IORING_OP_LAST
)
750 return io_cold_defs
[opcode
].name
;
754 bool io_uring_op_supported(u8 opcode
)
756 if (opcode
< IORING_OP_LAST
&&
757 io_issue_defs
[opcode
].prep
!= io_eopnotsupp_prep
)
762 void __init
io_uring_optable_init(void)
766 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs
) != IORING_OP_LAST
);
767 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs
) != IORING_OP_LAST
);
769 for (i
= 0; i
< ARRAY_SIZE(io_issue_defs
); i
++) {
770 BUG_ON(!io_issue_defs
[i
].prep
);
771 if (io_issue_defs
[i
].prep
!= io_eopnotsupp_prep
)
772 BUG_ON(!io_issue_defs
[i
].issue
);
773 WARN_ON_ONCE(!io_cold_defs
[i
].name
);