sys/kern/sys_aio.c

   1 /*      $NetBSD: sys_aio.c,v 1.29 2009/10/21 21:12:06 rmind Exp $       */
   2
   3 /*
   4  * Copyright (c) 2007, Mindaugas Rasiukevicius <rmind at NetBSD org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * Implementation of POSIX asynchronous I/O.
  31  * Defined in the Base Definitions volume of IEEE Std 1003.1-2001.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __KERNEL_RCSID(0, "$NetBSD: sys_aio.c,v 1.29 2009/10/21 21:12:06 rmind Exp $");
  36
  37 #ifdef _KERNEL_OPT
  38 #include "opt_ddb.h"
  39 #endif
  40
  41 #include <sys/param.h>
  42 #include <sys/condvar.h>
  43 #include <sys/file.h>
  44 #include <sys/filedesc.h>
  45 #include <sys/kernel.h>
  46 #include <sys/kmem.h>
  47 #include <sys/lwp.h>
  48 #include <sys/mutex.h>
  49 #include <sys/pool.h>
  50 #include <sys/proc.h>
  51 #include <sys/queue.h>
  52 #include <sys/signal.h>
  53 #include <sys/signalvar.h>
  54 #include <sys/syscall.h>
  55 #include <sys/syscallargs.h>
  56 #include <sys/syscallvar.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/systm.h>
  59 #include <sys/types.h>
  60 #include <sys/vnode.h>
  61 #include <sys/atomic.h>
  62 #include <sys/module.h>
  63 #include <sys/buf.h>
  64
  65 #include <uvm/uvm_extern.h>
  66
  67 MODULE(MODULE_CLASS_MISC, aio, NULL);
  68
  69 /*
  70  * System-wide limits and counter of AIO operations.
  71  */
  72 u_int aio_listio_max = AIO_LISTIO_MAX;
  73 static u_int aio_max = AIO_MAX;
  74 static u_int aio_jobs_count;
  75
  76 static struct pool aio_job_pool;
  77 static struct pool aio_lio_pool;
  78 static void *aio_ehook;
  79
  80 /* Prototypes */
  81 static void aio_worker(void *);
  82 static void aio_process(struct aio_job *);
  83 static void aio_sendsig(struct proc *, struct sigevent *);
  84 static int aio_enqueue_job(int, void *, struct lio_req *);
  85 static void aio_exit(proc_t *, void *);
  86
  87 static const struct syscall_package aio_syscalls[] = {
  88         { SYS_aio_cancel, 0, (sy_call_t *)sys_aio_cancel },
  89         { SYS_aio_error, 0, (sy_call_t *)sys_aio_error },
  90         { SYS_aio_fsync, 0, (sy_call_t *)sys_aio_fsync },
  91         { SYS_aio_read, 0, (sy_call_t *)sys_aio_read },
  92         { SYS_aio_return, 0, (sy_call_t *)sys_aio_return },
  93         { SYS___aio_suspend50, 0, (sy_call_t *)sys___aio_suspend50 },
  94         { SYS_aio_write, 0, (sy_call_t *)sys_aio_write },
  95         { SYS_lio_listio, 0, (sy_call_t *)sys_lio_listio },
  96         { 0, 0, NULL },
  97 };
  98
  99 /*
 100  * Tear down all AIO state.
 101  */
 102 static int
 103 aio_fini(bool interface)
 104 {
 105         int error;
 106         proc_t *p;
 107
 108         if (interface) {
 109                 /* Stop syscall activity. */
 110                 error = syscall_disestablish(NULL, aio_syscalls);
 111                 if (error != 0)
 112                         return error;
 113                 /* Abort if any processes are using AIO. */
 114                 mutex_enter(proc_lock);
 115                 PROCLIST_FOREACH(p, &allproc) {
 116                         if (p->p_aio != NULL)
 117                                 break;
 118                 }
 119                 mutex_exit(proc_lock);
 120                 if (p != NULL) {
 121                         error = syscall_establish(NULL, aio_syscalls);
 122                         KASSERT(error == 0);
 123                         return EBUSY;
 124                 }
 125         }
 126         KASSERT(aio_jobs_count == 0);
 127         exithook_disestablish(aio_ehook);
 128         pool_destroy(&aio_job_pool);
 129         pool_destroy(&aio_lio_pool);
 130         return 0;
 131 }
 132
 133 /*
 134  * Initialize global AIO state.
 135  */
 136 static int
 137 aio_init(void)
 138 {
 139         int error;
 140
 141         pool_init(&aio_job_pool, sizeof(struct aio_job), 0, 0, 0,
 142             "aio_jobs_pool", &pool_allocator_nointr, IPL_NONE);
 143         pool_init(&aio_lio_pool, sizeof(struct lio_req), 0, 0, 0,
 144             "aio_lio_pool", &pool_allocator_nointr, IPL_NONE);
 145         aio_ehook = exithook_establish(aio_exit, NULL);
 146         error = syscall_establish(NULL, aio_syscalls);
 147         if (error != 0)
 148                 aio_fini(false);
 149         return error;
 150 }
 151
 152 /*
 153  * Module interface.
 154  */
 155 static int
 156 aio_modcmd(modcmd_t cmd, void *arg)
 157 {
 158
 159         switch (cmd) {
 160         case MODULE_CMD_INIT:
 161                 return aio_init();
 162         case MODULE_CMD_FINI:
 163                 return aio_fini(true);
 164         default:
 165                 return ENOTTY;
 166         }
 167 }
 168
 169 /*
 170  * Initialize Asynchronous I/O data structures for the process.
 171  */
 172 static int
 173 aio_procinit(struct proc *p)
 174 {
 175         struct aioproc *aio;
 176         struct lwp *l;
 177         int error;
 178         vaddr_t uaddr;
 179
 180         /* Allocate and initialize AIO structure */
 181         aio = kmem_zalloc(sizeof(struct aioproc), KM_SLEEP);
 182         if (aio == NULL)
 183                 return EAGAIN;
 184
 185         /* Initialize queue and their synchronization structures */
 186         mutex_init(&aio->aio_mtx, MUTEX_DEFAULT, IPL_NONE);
 187         cv_init(&aio->aio_worker_cv, "aiowork");
 188         cv_init(&aio->done_cv, "aiodone");
 189         TAILQ_INIT(&aio->jobs_queue);
 190
 191         /*
 192          * Create an AIO worker thread.
 193          * XXX: Currently, AIO thread is not protected against user's actions.
 194          */
 195         uaddr = uvm_uarea_alloc();
 196         if (uaddr == 0) {
 197                 aio_exit(p, aio);
 198                 return EAGAIN;
 199         }
 200         error = lwp_create(curlwp, p, uaddr, 0, NULL, 0, aio_worker,
 201             NULL, &l, curlwp->l_class);
 202         if (error != 0) {
 203                 uvm_uarea_free(uaddr);
 204                 aio_exit(p, aio);
 205                 return error;
 206         }
 207
 208         /* Recheck if we are really first */
 209         mutex_enter(p->p_lock);
 210         if (p->p_aio) {
 211                 mutex_exit(p->p_lock);
 212                 aio_exit(p, aio);
 213                 lwp_exit(l);
 214                 return 0;
 215         }
 216         p->p_aio = aio;
 217
 218         /* Complete the initialization of thread, and run it */
 219         aio->aio_worker = l;
 220         p->p_nrlwps++;
 221         lwp_lock(l);
 222         l->l_stat = LSRUN;
 223         l->l_priority = MAXPRI_USER;
 224         sched_enqueue(l, false);
 225         lwp_unlock(l);
 226         mutex_exit(p->p_lock);
 227
 228         return 0;
 229 }
 230
 231 /*
 232  * Exit of Asynchronous I/O subsystem of process.
 233  */
 234 static void
 235 aio_exit(struct proc *p, void *cookie)
 236 {
 237         struct aio_job *a_job;
 238         struct aioproc *aio;
 239
 240         if (cookie != NULL)
 241                 aio = cookie;
 242         else if ((aio = p->p_aio) == NULL)
 243                 return;
 244
 245         /* Free AIO queue */
 246         while (!TAILQ_EMPTY(&aio->jobs_queue)) {
 247                 a_job = TAILQ_FIRST(&aio->jobs_queue);
 248                 TAILQ_REMOVE(&aio->jobs_queue, a_job, list);
 249                 pool_put(&aio_job_pool, a_job);
 250                 atomic_dec_uint(&aio_jobs_count);
 251         }
 252
 253         /* Destroy and free the entire AIO data structure */
 254         cv_destroy(&aio->aio_worker_cv);
 255         cv_destroy(&aio->done_cv);
 256         mutex_destroy(&aio->aio_mtx);
 257         kmem_free(aio, sizeof(struct aioproc));
 258 }
 259
 260 /*
 261  * AIO worker thread and processor.
 262  */
 263 static void
 264 aio_worker(void *arg)
 265 {
 266         struct proc *p = curlwp->l_proc;
 267         struct aioproc *aio = p->p_aio;
 268         struct aio_job *a_job;
 269         struct lio_req *lio;
 270         sigset_t oss, nss;
 271         int error, refcnt;
 272
 273         /*
 274          * Make an empty signal mask, so it
 275          * handles only SIGKILL and SIGSTOP.
 276          */
 277         sigfillset(&nss);
 278         mutex_enter(p->p_lock);
 279         error = sigprocmask1(curlwp, SIG_SETMASK, &nss, &oss);
 280         mutex_exit(p->p_lock);
 281         KASSERT(error == 0);
 282
 283         for (;;) {
 284                 /*
 285                  * Loop for each job in the queue.  If there
 286                  * are no jobs then sleep.
 287                  */
 288                 mutex_enter(&aio->aio_mtx);
 289                 while ((a_job = TAILQ_FIRST(&aio->jobs_queue)) == NULL) {
 290                         if (cv_wait_sig(&aio->aio_worker_cv, &aio->aio_mtx)) {
 291                                 /*
 292                                  * Thread was interrupted - check for
 293                                  * pending exit or suspend.
 294                                  */
 295                                 mutex_exit(&aio->aio_mtx);
 296                                 lwp_userret(curlwp);
 297                                 mutex_enter(&aio->aio_mtx);
 298                         }
 299                 }
 300
 301                 /* Take the job from the queue */
 302                 aio->curjob = a_job;
 303                 TAILQ_REMOVE(&aio->jobs_queue, a_job, list);
 304
 305                 atomic_dec_uint(&aio_jobs_count);
 306                 aio->jobs_count--;
 307
 308                 mutex_exit(&aio->aio_mtx);
 309
 310                 /* Process an AIO operation */
 311                 aio_process(a_job);
 312
 313                 /* Copy data structure back to the user-space */
 314                 (void)copyout(&a_job->aiocbp, a_job->aiocb_uptr,
 315                     sizeof(struct aiocb));
 316
 317                 mutex_enter(&aio->aio_mtx);
 318                 aio->curjob = NULL;
 319
 320                 /* Decrease a reference counter, if there is a LIO structure */
 321                 lio = a_job->lio;
 322                 refcnt = (lio != NULL ? --lio->refcnt : -1);
 323
 324                 /* Notify all suspenders */
 325                 cv_broadcast(&aio->done_cv);
 326                 mutex_exit(&aio->aio_mtx);
 327
 328                 /* Send a signal, if any */
 329                 aio_sendsig(p, &a_job->aiocbp.aio_sigevent);
 330
 331                 /* Destroy the LIO structure */
 332                 if (refcnt == 0) {
 333                         aio_sendsig(p, &lio->sig);
 334                         pool_put(&aio_lio_pool, lio);
 335                 }
 336
 337                 /* Destroy the job */
 338                 pool_put(&aio_job_pool, a_job);
 339         }
 340
 341         /* NOTREACHED */
 342 }
 343
 344 static void
 345 aio_process(struct aio_job *a_job)
 346 {
 347         struct proc *p = curlwp->l_proc;
 348         struct aiocb *aiocbp = &a_job->aiocbp;
 349         struct file *fp;
 350         int fd = aiocbp->aio_fildes;
 351         int error = 0;
 352
 353         KASSERT(a_job->aio_op != 0);
 354
 355         if ((a_job->aio_op & (AIO_READ | AIO_WRITE)) != 0) {
 356                 struct iovec aiov;
 357                 struct uio auio;
 358
 359                 if (aiocbp->aio_nbytes > SSIZE_MAX) {
 360                         error = EINVAL;
 361                         goto done;
 362                 }
 363
 364                 fp = fd_getfile(fd);
 365                 if (fp == NULL) {
 366                         error = EBADF;
 367                         goto done;
 368                 }
 369
 370                 aiov.iov_base = (void *)(uintptr_t)aiocbp->aio_buf;
 371                 aiov.iov_len = aiocbp->aio_nbytes;
 372                 auio.uio_iov = &aiov;
 373                 auio.uio_iovcnt = 1;
 374                 auio.uio_resid = aiocbp->aio_nbytes;
 375                 auio.uio_vmspace = p->p_vmspace;
 376
 377                 if (a_job->aio_op & AIO_READ) {
 378                         /*
 379                          * Perform a Read operation
 380                          */
 381                         KASSERT((a_job->aio_op & AIO_WRITE) == 0);
 382
 383                         if ((fp->f_flag & FREAD) == 0) {
 384                                 fd_putfile(fd);
 385                                 error = EBADF;
 386                                 goto done;
 387                         }
 388                         auio.uio_rw = UIO_READ;
 389                         error = (*fp->f_ops->fo_read)(fp, &aiocbp->aio_offset,
 390                             &auio, fp->f_cred, FOF_UPDATE_OFFSET);
 391                 } else {
 392                         /*
 393                          * Perform a Write operation
 394                          */
 395                         KASSERT(a_job->aio_op & AIO_WRITE);
 396
 397                         if ((fp->f_flag & FWRITE) == 0) {
 398                                 fd_putfile(fd);
 399                                 error = EBADF;
 400                                 goto done;
 401                         }
 402                         auio.uio_rw = UIO_WRITE;
 403                         error = (*fp->f_ops->fo_write)(fp, &aiocbp->aio_offset,
 404                             &auio, fp->f_cred, FOF_UPDATE_OFFSET);
 405                 }
 406                 fd_putfile(fd);
 407
 408                 /* Store the result value */
 409                 a_job->aiocbp.aio_nbytes -= auio.uio_resid;
 410                 a_job->aiocbp._retval = (error == 0) ?
 411                     a_job->aiocbp.aio_nbytes : -1;
 412
 413         } else if ((a_job->aio_op & (AIO_SYNC | AIO_DSYNC)) != 0) {
 414                 /*
 415                  * Perform a file Sync operation
 416                  */
 417                 struct vnode *vp;
 418
 419                 if ((error = fd_getvnode(fd, &fp)) != 0)
 420                         goto done;
 421
 422                 if ((fp->f_flag & FWRITE) == 0) {
 423                         fd_putfile(fd);
 424                         error = EBADF;
 425                         goto done;
 426                 }
 427
 428                 vp = (struct vnode *)fp->f_data;
 429                 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 430                 if (a_job->aio_op & AIO_DSYNC) {
 431                         error = VOP_FSYNC(vp, fp->f_cred,
 432                             FSYNC_WAIT | FSYNC_DATAONLY, 0, 0);
 433                 } else if (a_job->aio_op & AIO_SYNC) {
 434                         error = VOP_FSYNC(vp, fp->f_cred,
 435                             FSYNC_WAIT, 0, 0);
 436                 }
 437                 VOP_UNLOCK(vp, 0);
 438                 fd_putfile(fd);
 439
 440                 /* Store the result value */
 441                 a_job->aiocbp._retval = (error == 0) ? 0 : -1;
 442
 443         } else
 444                 panic("aio_process: invalid operation code\n");
 445
 446 done:
 447         /* Job is done, set the error, if any */
 448         a_job->aiocbp._errno = error;
 449         a_job->aiocbp._state = JOB_DONE;
 450 }
 451
 452 /*
 453  * Send AIO signal.
 454  */
 455 static void
 456 aio_sendsig(struct proc *p, struct sigevent *sig)
 457 {
 458         ksiginfo_t ksi;
 459
 460         if (sig->sigev_signo == 0 || sig->sigev_notify == SIGEV_NONE)
 461                 return;
 462
 463         KSI_INIT(&ksi);
 464         ksi.ksi_signo = sig->sigev_signo;
 465         ksi.ksi_code = SI_ASYNCIO;
 466         ksi.ksi_value = sig->sigev_value;
 467         mutex_enter(proc_lock);
 468         kpsignal(p, &ksi, NULL);
 469         mutex_exit(proc_lock);
 470 }
 471
 472 /*
 473  * Enqueue the job.
 474  */
 475 static int
 476 aio_enqueue_job(int op, void *aiocb_uptr, struct lio_req *lio)
 477 {
 478         struct proc *p = curlwp->l_proc;
 479         struct aioproc *aio;
 480         struct aio_job *a_job;
 481         struct aiocb aiocbp;
 482         struct sigevent *sig;
 483         int error;
 484
 485         /* Non-accurate check for the limit */
 486         if (aio_jobs_count + 1 > aio_max)
 487                 return EAGAIN;
 488
 489         /* Get the data structure from user-space */
 490         error = copyin(aiocb_uptr, &aiocbp, sizeof(struct aiocb));
 491         if (error)
 492                 return error;
 493
 494         /* Check if signal is set, and validate it */
 495         sig = &aiocbp.aio_sigevent;
 496         if (sig->sigev_signo < 0 || sig->sigev_signo >= NSIG ||
 497             sig->sigev_notify < SIGEV_NONE || sig->sigev_notify > SIGEV_SA)
 498                 return EINVAL;
 499
 500         /* Buffer and byte count */
 501         if (((AIO_SYNC | AIO_DSYNC) & op) == 0)
 502                 if (aiocbp.aio_buf == NULL || aiocbp.aio_nbytes > SSIZE_MAX)
 503                         return EINVAL;
 504
 505         /* Check the opcode, if LIO_NOP - simply ignore */
 506         if (op == AIO_LIO) {
 507                 KASSERT(lio != NULL);
 508                 if (aiocbp.aio_lio_opcode == LIO_WRITE)
 509                         op = AIO_WRITE;
 510                 else if (aiocbp.aio_lio_opcode == LIO_READ)
 511                         op = AIO_READ;
 512                 else
 513                         return (aiocbp.aio_lio_opcode == LIO_NOP) ? 0 : EINVAL;
 514         } else {
 515                 KASSERT(lio == NULL);
 516         }
 517
 518         /*
 519          * Look for already existing job.  If found - the job is in-progress.
 520          * According to POSIX this is invalid, so return the error.
 521          */
 522         aio = p->p_aio;
 523         if (aio) {
 524                 mutex_enter(&aio->aio_mtx);
 525                 if (aio->curjob) {
 526                         a_job = aio->curjob;
 527                         if (a_job->aiocb_uptr == aiocb_uptr) {
 528                                 mutex_exit(&aio->aio_mtx);
 529                                 return EINVAL;
 530                         }
 531                 }
 532                 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) {
 533                         if (a_job->aiocb_uptr != aiocb_uptr)
 534                                 continue;
 535                         mutex_exit(&aio->aio_mtx);
 536                         return EINVAL;
 537                 }
 538                 mutex_exit(&aio->aio_mtx);
 539         }
 540
 541         /*
 542          * Check if AIO structure is initialized, if not - initialize it.
 543          * In LIO case, we did that already.  We will recheck this with
 544          * the lock in aio_procinit().
 545          */
 546         if (lio == NULL && p->p_aio == NULL)
 547                 if (aio_procinit(p))
 548                         return EAGAIN;
 549         aio = p->p_aio;
 550
 551         /*
 552          * Set the state with errno, and copy data
 553          * structure back to the user-space.
 554          */
 555         aiocbp._state = JOB_WIP;
 556         aiocbp._errno = EINPROGRESS;
 557         aiocbp._retval = -1;
 558         error = copyout(&aiocbp, aiocb_uptr, sizeof(struct aiocb));
 559         if (error)
 560                 return error;
 561
 562         /* Allocate and initialize a new AIO job */
 563         a_job = pool_get(&aio_job_pool, PR_WAITOK);
 564         memset(a_job, 0, sizeof(struct aio_job));
 565
 566         /*
 567          * Set the data.
 568          * Store the user-space pointer for searching.  Since we
 569          * are storing only per proc pointers - it is safe.
 570          */
 571         memcpy(&a_job->aiocbp, &aiocbp, sizeof(struct aiocb));
 572         a_job->aiocb_uptr = aiocb_uptr;
 573         a_job->aio_op |= op;
 574         a_job->lio = lio;
 575
 576         /*
 577          * Add the job to the queue, update the counters, and
 578          * notify the AIO worker thread to handle the job.
 579          */
 580         mutex_enter(&aio->aio_mtx);
 581
 582         /* Fail, if the limit was reached */
 583         if (atomic_inc_uint_nv(&aio_jobs_count) > aio_max ||
 584             aio->jobs_count >= aio_listio_max) {
 585                 atomic_dec_uint(&aio_jobs_count);
 586                 mutex_exit(&aio->aio_mtx);
 587                 pool_put(&aio_job_pool, a_job);
 588                 return EAGAIN;
 589         }
 590
 591         TAILQ_INSERT_TAIL(&aio->jobs_queue, a_job, list);
 592         aio->jobs_count++;
 593         if (lio)
 594                 lio->refcnt++;
 595         cv_signal(&aio->aio_worker_cv);
 596
 597         mutex_exit(&aio->aio_mtx);
 598
 599         /*
 600          * One would handle the errors only with aio_error() function.
 601          * This way is appropriate according to POSIX.
 602          */
 603         return 0;
 604 }
 605
 606 /*
 607  * Syscall functions.
 608  */
 609
 610 int
 611 sys_aio_cancel(struct lwp *l, const struct sys_aio_cancel_args *uap,
 612     register_t *retval)
 613 {
 614         /* {
 615                 syscallarg(int) fildes;
 616                 syscallarg(struct aiocb *) aiocbp;
 617         } */
 618         struct proc *p = l->l_proc;
 619         struct aioproc *aio;
 620         struct aio_job *a_job;
 621         struct aiocb *aiocbp_ptr;
 622         struct lio_req *lio;
 623         struct filedesc *fdp = p->p_fd;
 624         unsigned int cn, errcnt, fildes;
 625         fdtab_t *dt;
 626
 627         TAILQ_HEAD(, aio_job) tmp_jobs_list;
 628
 629         /* Check for invalid file descriptor */
 630         fildes = (unsigned int)SCARG(uap, fildes);
 631         dt = fdp->fd_dt;
 632         if (fildes >= dt->dt_nfiles)
 633                 return EBADF;
 634         if (dt->dt_ff[fildes] == NULL || dt->dt_ff[fildes]->ff_file == NULL)
 635                 return EBADF;
 636
 637         /* Check if AIO structure is initialized */
 638         if (p->p_aio == NULL) {
 639                 *retval = AIO_NOTCANCELED;
 640                 return 0;
 641         }
 642
 643         aio = p->p_aio;
 644         aiocbp_ptr = (struct aiocb *)SCARG(uap, aiocbp);
 645
 646         mutex_enter(&aio->aio_mtx);
 647
 648         /* Cancel the jobs, and remove them from the queue */
 649         cn = 0;
 650         TAILQ_INIT(&tmp_jobs_list);
 651         TAILQ_FOREACH(a_job, &aio->jobs_queue, list) {
 652                 if (aiocbp_ptr) {
 653                         if (aiocbp_ptr != a_job->aiocb_uptr)
 654                                 continue;
 655                         if (fildes != a_job->aiocbp.aio_fildes) {
 656                                 mutex_exit(&aio->aio_mtx);
 657                                 return EBADF;
 658                         }
 659                 } else if (a_job->aiocbp.aio_fildes != fildes)
 660                         continue;
 661
 662                 TAILQ_REMOVE(&aio->jobs_queue, a_job, list);
 663                 TAILQ_INSERT_TAIL(&tmp_jobs_list, a_job, list);
 664
 665                 /* Decrease the counters */
 666                 atomic_dec_uint(&aio_jobs_count);
 667                 aio->jobs_count--;
 668                 lio = a_job->lio;
 669                 if (lio != NULL && --lio->refcnt != 0)
 670                         a_job->lio = NULL;
 671
 672                 cn++;
 673                 if (aiocbp_ptr)
 674                         break;
 675         }
 676
 677         /* There are canceled jobs */
 678         if (cn)
 679                 *retval = AIO_CANCELED;
 680
 681         /* We cannot cancel current job */
 682         a_job = aio->curjob;
 683         if (a_job && ((a_job->aiocbp.aio_fildes == fildes) ||
 684             (a_job->aiocb_uptr == aiocbp_ptr)))
 685                 *retval = AIO_NOTCANCELED;
 686
 687         mutex_exit(&aio->aio_mtx);
 688
 689         /* Free the jobs after the lock */
 690         errcnt = 0;
 691         while (!TAILQ_EMPTY(&tmp_jobs_list)) {
 692                 a_job = TAILQ_FIRST(&tmp_jobs_list);
 693                 TAILQ_REMOVE(&tmp_jobs_list, a_job, list);
 694                 /* Set the errno and copy structures back to the user-space */
 695                 a_job->aiocbp._errno = ECANCELED;
 696                 a_job->aiocbp._state = JOB_DONE;
 697                 if (copyout(&a_job->aiocbp, a_job->aiocb_uptr,
 698                     sizeof(struct aiocb)))
 699                         errcnt++;
 700                 /* Send a signal if any */
 701                 aio_sendsig(p, &a_job->aiocbp.aio_sigevent);
 702                 if (a_job->lio) {
 703                         lio = a_job->lio;
 704                         aio_sendsig(p, &lio->sig);
 705                         pool_put(&aio_lio_pool, lio);
 706                 }
 707                 pool_put(&aio_job_pool, a_job);
 708         }
 709
 710         if (errcnt)
 711                 return EFAULT;
 712
 713         /* Set a correct return value */
 714         if (*retval == 0)
 715                 *retval = AIO_ALLDONE;
 716
 717         return 0;
 718 }
 719
 720 int
 721 sys_aio_error(struct lwp *l, const struct sys_aio_error_args *uap,
 722     register_t *retval)
 723 {
 724         /* {
 725                 syscallarg(const struct aiocb *) aiocbp;
 726         } */
 727         struct proc *p = l->l_proc;
 728         struct aioproc *aio = p->p_aio;
 729         struct aiocb aiocbp;
 730         int error;
 731
 732         if (aio == NULL)
 733                 return EINVAL;
 734
 735         error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb));
 736         if (error)
 737                 return error;
 738
 739         if (aiocbp._state == JOB_NONE)
 740                 return EINVAL;
 741
 742         *retval = aiocbp._errno;
 743
 744         return 0;
 745 }
 746
 747 int
 748 sys_aio_fsync(struct lwp *l, const struct sys_aio_fsync_args *uap,
 749     register_t *retval)
 750 {
 751         /* {
 752                 syscallarg(int) op;
 753                 syscallarg(struct aiocb *) aiocbp;
 754         } */
 755         int op = SCARG(uap, op);
 756
 757         if ((op != O_DSYNC) && (op != O_SYNC))
 758                 return EINVAL;
 759
 760         op = O_DSYNC ? AIO_DSYNC : AIO_SYNC;
 761
 762         return aio_enqueue_job(op, SCARG(uap, aiocbp), NULL);
 763 }
 764
 765 int
 766 sys_aio_read(struct lwp *l, const struct sys_aio_read_args *uap,
 767     register_t *retval)
 768 {
 769         /* {
 770                 syscallarg(struct aiocb *) aiocbp;
 771         } */
 772
 773         return aio_enqueue_job(AIO_READ, SCARG(uap, aiocbp), NULL);
 774 }
 775
 776 int
 777 sys_aio_return(struct lwp *l, const struct sys_aio_return_args *uap,
 778     register_t *retval)
 779 {
 780         /* {
 781                 syscallarg(struct aiocb *) aiocbp;
 782         } */
 783         struct proc *p = l->l_proc;
 784         struct aioproc *aio = p->p_aio;
 785         struct aiocb aiocbp;
 786         int error;
 787
 788         if (aio == NULL)
 789                 return EINVAL;
 790
 791         error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb));
 792         if (error)
 793                 return error;
 794
 795         if (aiocbp._errno == EINPROGRESS || aiocbp._state != JOB_DONE)
 796                 return EINVAL;
 797
 798         *retval = aiocbp._retval;
 799
 800         /* Reset the internal variables */
 801         aiocbp._errno = 0;
 802         aiocbp._retval = -1;
 803         aiocbp._state = JOB_NONE;
 804         error = copyout(&aiocbp, SCARG(uap, aiocbp), sizeof(struct aiocb));
 805
 806         return error;
 807 }
 808
 809 int
 810 sys___aio_suspend50(struct lwp *l, const struct sys___aio_suspend50_args *uap,
 811     register_t *retval)
 812 {
 813         /* {
 814                 syscallarg(const struct aiocb *const[]) list;
 815                 syscallarg(int) nent;
 816                 syscallarg(const struct timespec *) timeout;
 817         } */
 818         struct aiocb **list;
 819         struct timespec ts;
 820         int error, nent;
 821
 822         nent = SCARG(uap, nent);
 823         if (nent <= 0 || nent > aio_listio_max)
 824                 return EAGAIN;
 825
 826         if (SCARG(uap, timeout)) {
 827                 /* Convert timespec to ticks */
 828                 error = copyin(SCARG(uap, timeout), &ts,
 829                     sizeof(struct timespec));
 830                 if (error)
 831                         return error;
 832         }
 833         list = kmem_alloc(nent * sizeof(*list), KM_SLEEP);
 834         error = copyin(SCARG(uap, list), list, nent * sizeof(*list));
 835         if (error)
 836                 goto out;
 837         error = aio_suspend1(l, list, nent, SCARG(uap, timeout) ? &ts : NULL);
 838 out:
 839         kmem_free(list, nent * sizeof(*list));
 840         return error;
 841 }
 842
 843 int
 844 aio_suspend1(struct lwp *l, struct aiocb **aiocbp_list, int nent,
 845     struct timespec *ts)
 846 {
 847         struct proc *p = l->l_proc;
 848         struct aioproc *aio;
 849         struct aio_job *a_job;
 850         int i, error, timo;
 851
 852         if (p->p_aio == NULL)
 853                 return EAGAIN;
 854         aio = p->p_aio;
 855
 856         if (ts) {
 857                 timo = mstohz((ts->tv_sec * 1000) + (ts->tv_nsec / 1000000));
 858                 if (timo == 0 && ts->tv_sec == 0 && ts->tv_nsec > 0)
 859                         timo = 1;
 860                 if (timo <= 0)
 861                         return EAGAIN;
 862         } else
 863                 timo = 0;
 864
 865         /* Get the list from user-space */
 866
 867         mutex_enter(&aio->aio_mtx);
 868         for (;;) {
 869
 870                 for (i = 0; i < nent; i++) {
 871
 872                         /* Skip NULL entries */
 873                         if (aiocbp_list[i] == NULL)
 874                                 continue;
 875
 876                         /* Skip current job */
 877                         if (aio->curjob) {
 878                                 a_job = aio->curjob;
 879                                 if (a_job->aiocb_uptr == aiocbp_list[i])
 880                                         continue;
 881                         }
 882
 883                         /* Look for a job in the queue */
 884                         TAILQ_FOREACH(a_job, &aio->jobs_queue, list)
 885                                 if (a_job->aiocb_uptr == aiocbp_list[i])
 886                                         break;
 887
 888                         if (a_job == NULL) {
 889                                 struct aiocb aiocbp;
 890
 891                                 mutex_exit(&aio->aio_mtx);
 892
 893                                 error = copyin(aiocbp_list[i], &aiocbp,
 894                                     sizeof(struct aiocb));
 895                                 if (error == 0 && aiocbp._state != JOB_DONE) {
 896                                         mutex_enter(&aio->aio_mtx);
 897                                         continue;
 898                                 }
 899                                 return error;
 900                         }
 901                 }
 902
 903                 /* Wait for a signal or when timeout occurs */
 904                 error = cv_timedwait_sig(&aio->done_cv, &aio->aio_mtx, timo);
 905                 if (error) {
 906                         if (error == EWOULDBLOCK)
 907                                 error = EAGAIN;
 908                         break;
 909                 }
 910         }
 911         mutex_exit(&aio->aio_mtx);
 912         return error;
 913 }
 914
 915 int
 916 sys_aio_write(struct lwp *l, const struct sys_aio_write_args *uap,
 917     register_t *retval)
 918 {
 919         /* {
 920                 syscallarg(struct aiocb *) aiocbp;
 921         } */
 922
 923         return aio_enqueue_job(AIO_WRITE, SCARG(uap, aiocbp), NULL);
 924 }
 925
 926 int
 927 sys_lio_listio(struct lwp *l, const struct sys_lio_listio_args *uap,
 928     register_t *retval)
 929 {
 930         /* {
 931                 syscallarg(int) mode;
 932                 syscallarg(struct aiocb *const[]) list;
 933                 syscallarg(int) nent;
 934                 syscallarg(struct sigevent *) sig;
 935         } */
 936         struct proc *p = l->l_proc;
 937         struct aioproc *aio;
 938         struct aiocb **aiocbp_list;
 939         struct lio_req *lio;
 940         int i, error, errcnt, mode, nent;
 941
 942         mode = SCARG(uap, mode);
 943         nent = SCARG(uap, nent);
 944
 945         /* Non-accurate checks for the limit and invalid values */
 946         if (nent < 1 || nent > aio_listio_max)
 947                 return EINVAL;
 948         if (aio_jobs_count + nent > aio_max)
 949                 return EAGAIN;
 950
 951         /* Check if AIO structure is initialized, if not - initialize it */
 952         if (p->p_aio == NULL)
 953                 if (aio_procinit(p))
 954                         return EAGAIN;
 955         aio = p->p_aio;
 956
 957         /* Create a LIO structure */
 958         lio = pool_get(&aio_lio_pool, PR_WAITOK);
 959         lio->refcnt = 1;
 960         error = 0;
 961
 962         switch (mode) {
 963         case LIO_WAIT:
 964                 memset(&lio->sig, 0, sizeof(struct sigevent));
 965                 break;
 966         case LIO_NOWAIT:
 967                 /* Check for signal, validate it */
 968                 if (SCARG(uap, sig)) {
 969                         struct sigevent *sig = &lio->sig;
 970
 971                         error = copyin(SCARG(uap, sig), &lio->sig,
 972                             sizeof(struct sigevent));
 973                         if (error == 0 &&
 974                             (sig->sigev_signo < 0 ||
 975                             sig->sigev_signo >= NSIG ||
 976                             sig->sigev_notify < SIGEV_NONE ||
 977                             sig->sigev_notify > SIGEV_SA))
 978                                 error = EINVAL;
 979                 } else
 980                         memset(&lio->sig, 0, sizeof(struct sigevent));
 981                 break;
 982         default:
 983                 error = EINVAL;
 984                 break;
 985         }
 986
 987         if (error != 0) {
 988                 pool_put(&aio_lio_pool, lio);
 989                 return error;
 990         }
 991
 992         /* Get the list from user-space */
 993         aiocbp_list = kmem_alloc(nent * sizeof(*aiocbp_list), KM_SLEEP);
 994         error = copyin(SCARG(uap, list), aiocbp_list,
 995             nent * sizeof(*aiocbp_list));
 996         if (error) {
 997                 mutex_enter(&aio->aio_mtx);
 998                 goto err;
 999         }
1000
1001         /* Enqueue all jobs */
1002         errcnt = 0;
1003         for (i = 0; i < nent; i++) {
1004                 error = aio_enqueue_job(AIO_LIO, aiocbp_list[i], lio);
1005                 /*
1006                  * According to POSIX, in such error case it may
1007                  * fail with other I/O operations initiated.
1008                  */
1009                 if (error)
1010                         errcnt++;
1011         }
1012
1013         mutex_enter(&aio->aio_mtx);
1014
1015         /* Return an error, if any */
1016         if (errcnt) {
1017                 error = EIO;
1018                 goto err;
1019         }
1020
1021         if (mode == LIO_WAIT) {
1022                 /*
1023                  * Wait for AIO completion.  In such case,
1024                  * the LIO structure will be freed here.
1025                  */
1026                 while (lio->refcnt > 1 && error == 0)
1027                         error = cv_wait_sig(&aio->done_cv, &aio->aio_mtx);
1028                 if (error)
1029                         error = EINTR;
1030         }
1031
1032 err:
1033         if (--lio->refcnt != 0)
1034                 lio = NULL;
1035         mutex_exit(&aio->aio_mtx);
1036         if (lio != NULL) {
1037                 aio_sendsig(p, &lio->sig);
1038                 pool_put(&aio_lio_pool, lio);
1039         }
1040         kmem_free(aiocbp_list, nent * sizeof(*aiocbp_list));
1041         return error;
1042 }
1043
1044 /*
1045  * SysCtl
1046  */
1047
1048 static int
1049 sysctl_aio_listio_max(SYSCTLFN_ARGS)
1050 {
1051         struct sysctlnode node;
1052         int error, newsize;
1053
1054         node = *rnode;
1055         node.sysctl_data = &newsize;
1056
1057         newsize = aio_listio_max;
1058         error = sysctl_lookup(SYSCTLFN_CALL(&node));
1059         if (error || newp == NULL)
1060                 return error;
1061
1062         if (newsize < 1 || newsize > aio_max)
1063                 return EINVAL;
1064         aio_listio_max = newsize;
1065
1066         return 0;
1067 }
1068
1069 static int
1070 sysctl_aio_max(SYSCTLFN_ARGS)
1071 {
1072         struct sysctlnode node;
1073         int error, newsize;
1074
1075         node = *rnode;
1076         node.sysctl_data = &newsize;
1077
1078         newsize = aio_max;
1079         error = sysctl_lookup(SYSCTLFN_CALL(&node));
1080         if (error || newp == NULL)
1081                 return error;
1082
1083         if (newsize < 1 || newsize < aio_listio_max)
1084                 return EINVAL;
1085         aio_max = newsize;
1086
1087         return 0;
1088 }
1089
1090 SYSCTL_SETUP(sysctl_aio_setup, "sysctl aio setup")
1091 {
1092
1093         sysctl_createv(clog, 0, NULL, NULL,
1094                 CTLFLAG_PERMANENT,
1095                 CTLTYPE_NODE, "kern", NULL,
1096                 NULL, 0, NULL, 0,
1097                 CTL_KERN, CTL_EOL);
1098         sysctl_createv(clog, 0, NULL, NULL,
1099                 CTLFLAG_PERMANENT | CTLFLAG_IMMEDIATE,
1100                 CTLTYPE_INT, "posix_aio",
1101                 SYSCTL_DESCR("Version of IEEE Std 1003.1 and its "
1102                              "Asynchronous I/O option to which the "
1103                              "system attempts to conform"),
1104                 NULL, _POSIX_ASYNCHRONOUS_IO, NULL, 0,
1105                 CTL_KERN, CTL_CREATE, CTL_EOL);
1106         sysctl_createv(clog, 0, NULL, NULL,
1107                 CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
1108                 CTLTYPE_INT, "aio_listio_max",
1109                 SYSCTL_DESCR("Maximum number of asynchronous I/O "
1110                              "operations in a single list I/O call"),
1111                 sysctl_aio_listio_max, 0, &aio_listio_max, 0,
1112                 CTL_KERN, CTL_CREATE, CTL_EOL);
1113         sysctl_createv(clog, 0, NULL, NULL,
1114                 CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
1115                 CTLTYPE_INT, "aio_max",
1116                 SYSCTL_DESCR("Maximum number of asynchronous I/O "
1117                              "operations"),
1118                 sysctl_aio_max, 0, &aio_max, 0,
1119                 CTL_KERN, CTL_CREATE, CTL_EOL);
1120 }
1121
1122 /*
1123  * Debugging
1124  */
1125 #if defined(DDB)
1126 void
1127 aio_print_jobs(void (*pr)(const char *, ...))
1128 {
1129         struct proc *p = (curlwp == NULL ? NULL : curlwp->l_proc);
1130         struct aioproc *aio;
1131         struct aio_job *a_job;
1132         struct aiocb *aiocbp;
1133
1134         if (p == NULL) {
1135                 (*pr)("AIO: We are not in the processes right now.\n");
1136                 return;
1137         }
1138
1139         aio = p->p_aio;
1140         if (aio == NULL) {
1141                 (*pr)("AIO data is not initialized (PID = %d).\n", p->p_pid);
1142                 return;
1143         }
1144
1145         (*pr)("AIO: PID = %d\n", p->p_pid);
1146         (*pr)("AIO: Global count of the jobs = %u\n", aio_jobs_count);
1147         (*pr)("AIO: Count of the jobs = %u\n", aio->jobs_count);
1148
1149         if (aio->curjob) {
1150                 a_job = aio->curjob;
1151                 (*pr)("\nAIO current job:\n");
1152                 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n",
1153                     a_job->aio_op, a_job->aiocbp._errno,
1154                     a_job->aiocbp._state, a_job->aiocb_uptr);
1155                 aiocbp = &a_job->aiocbp;
1156                 (*pr)("   fd = %d, offset = %u, buf = %p, nbytes = %u\n",
1157                     aiocbp->aio_fildes, aiocbp->aio_offset,
1158                     aiocbp->aio_buf, aiocbp->aio_nbytes);
1159         }
1160
1161         (*pr)("\nAIO queue:\n");
1162         TAILQ_FOREACH(a_job, &aio->jobs_queue, list) {
1163                 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n",
1164                     a_job->aio_op, a_job->aiocbp._errno,
1165                     a_job->aiocbp._state, a_job->aiocb_uptr);
1166                 aiocbp = &a_job->aiocbp;
1167                 (*pr)("   fd = %d, offset = %u, buf = %p, nbytes = %u\n",
1168                     aiocbp->aio_fildes, aiocbp->aio_offset,
1169                     aiocbp->aio_buf, aiocbp->aio_nbytes);
1170         }
1171 }
1172 #endif /* defined(DDB) */