?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
8322 nl: misleading-indentation
[unleashed/tickless.git] / usr / src / uts / common / os / aio_subr.c
blobce8ed143b441b6d0fd51c7317b7e8bafc8aa0b97
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include <sys/types.h>
28 #include <sys/proc.h>
29 #include <sys/file.h>
30 #include <sys/errno.h>
31 #include <sys/param.h>
32 #include <sys/sysmacros.h>
33 #include <sys/cmn_err.h>
34 #include <sys/systm.h>
35 #include <vm/as.h>
36 #include <vm/page.h>
37 #include <sys/uio.h>
38 #include <sys/kmem.h>
39 #include <sys/debug.h>
40 #include <sys/aio_impl.h>
41 #include <sys/epm.h>
42 #include <sys/fs/snode.h>
43 #include <sys/siginfo.h>
44 #include <sys/cpuvar.h>
45 #include <sys/tnf_probe.h>
46 #include <sys/conf.h>
47 #include <sys/sdt.h>
48
49 int aphysio(int (*)(), int (*)(), dev_t, int, void (*)(), struct aio_req *);
50 void aio_done(struct buf *);
51 void aphysio_unlock(aio_req_t *);
52 void aio_cleanup(int);
53 void aio_cleanup_exit(void);
54
55 /*
56 * private functions
57 */
58 static void aio_sigev_send(proc_t *, sigqueue_t *);
59 static void aio_hash_delete(aio_t *, aio_req_t *);
60 static void aio_lio_free(aio_t *, aio_lio_t *);
61 static int aio_cleanup_cleanupq(aio_t *, aio_req_t *, int);
62 static int aio_cleanup_notifyq(aio_t *, aio_req_t *, int);
63 static void aio_cleanup_pollq(aio_t *, aio_req_t *, int);
64 static void aio_cleanup_portq(aio_t *, aio_req_t *, int);
65
66 /*
67 * async version of physio() that doesn't wait synchronously
68 * for the driver's strategy routine to complete.
69 */
70
71 int
72 aphysio(
73 int (*strategy)(struct buf *),
74 int (*cancel)(struct buf *),
75 dev_t dev,
76 int rw,
77 void (*mincnt)(struct buf *),
78 struct aio_req *aio)
79 {
80 struct uio *uio = aio->aio_uio;
81 aio_req_t *reqp = (aio_req_t *)aio->aio_private;
82 struct buf *bp = &reqp->aio_req_buf;
83 struct iovec *iov;
84 struct as *as;
85 char *a;
86 int error;
87 size_t c;
88 struct page **pplist;
89 struct dev_ops *ops = devopsp[getmajor(dev)];
90
91 if (uio->uio_loffset < 0)
92 return (EINVAL);
93 #ifdef _ILP32
94 /*
95 * For 32-bit kernels, check against SPEC_MAXOFFSET_T which represents
96 * the maximum size that can be supported by the IO subsystem.
97 * XXX this code assumes a D_64BIT driver.
98 */
99 if (uio->uio_loffset > SPEC_MAXOFFSET_T)
100 return (EINVAL);
101 #endif /* _ILP32 */
102
103 TNF_PROBE_5(aphysio_start, "kaio", /* CSTYLED */,
104 tnf_opaque, bp, bp,
105 tnf_device, device, dev,
106 tnf_offset, blkno, btodt(uio->uio_loffset),
107 tnf_size, size, uio->uio_iov->iov_len,
108 tnf_bioflags, rw, rw);
109
110 if (rw == B_READ) {
111 CPU_STATS_ADD_K(sys, phread, 1);
112 } else {
113 CPU_STATS_ADD_K(sys, phwrite, 1);
114 }
115
116 iov = uio->uio_iov;
117 sema_init(&bp->b_sem, 0, NULL, SEMA_DEFAULT, NULL);
118 sema_init(&bp->b_io, 0, NULL, SEMA_DEFAULT, NULL);
119
120 bp->b_error = 0;
121 bp->b_flags = B_BUSY | B_PHYS | B_ASYNC | rw;
122 bp->b_edev = dev;
123 bp->b_dev = cmpdev(dev);
124 bp->b_lblkno = btodt(uio->uio_loffset);
125 bp->b_offset = uio->uio_loffset;
126 (void) ops->devo_getinfo(NULL, DDI_INFO_DEVT2DEVINFO,
127 (void *)bp->b_edev, (void **)&bp->b_dip);
128
129 /*
130 * Clustering: Clustering can set the b_iodone, b_forw and
131 * b_proc fields to cluster-specifc values.
132 */
133 if (bp->b_iodone == NULL) {
134 bp->b_iodone = (int (*)()) aio_done;
135 /* b_forw points at an aio_req_t structure */
136 bp->b_forw = (struct buf *)reqp;
137 bp->b_proc = curproc;
138 }
139
140 a = bp->b_un.b_addr = iov->iov_base;
141 c = bp->b_bcount = iov->iov_len;
142
143 (*mincnt)(bp);
144 if (bp->b_bcount != iov->iov_len)
145 return (ENOTSUP);
146
147 as = bp->b_proc->p_as;
148
149 error = as_pagelock(as, &pplist, a,
150 c, rw == B_READ? S_WRITE : S_READ);
151 if (error != 0) {
152 bp->b_flags |= B_ERROR;
153 bp->b_error = error;
154 bp->b_flags &= ~(B_BUSY|B_WANTED|B_PHYS|B_SHADOW);
155 return (error);
156 }
157 reqp->aio_req_flags |= AIO_PAGELOCKDONE;
158 bp->b_shadow = pplist;
159 if (pplist != NULL) {
160 bp->b_flags |= B_SHADOW;
161 }
162
163 if (cancel != anocancel)
164 cmn_err(CE_PANIC,
165 "aphysio: cancellation not supported, use anocancel");
166
167 reqp->aio_req_cancel = cancel;
168
169 DTRACE_IO1(start, struct buf *, bp);
170
171 return ((*strategy)(bp));
172 }
173
174 /*ARGSUSED*/
175 int
176 anocancel(struct buf *bp)
177 {
178 return (ENXIO);
179 }
180
181 /*
182 * Called from biodone().
183 * Notify process that a pending AIO has finished.
184 */
185
186 /*
187 * Clustering: This function is made non-static as it is used
188 * by clustering s/w as contract private interface.
189 */
190
191 void
192 aio_done(struct buf *bp)
193 {
194 proc_t *p;
195 struct as *as;
196 aio_req_t *reqp;
197 aio_lio_t *head = NULL;
198 aio_t *aiop;
199 sigqueue_t *sigev = NULL;
200 sigqueue_t *lio_sigev = NULL;
201 port_kevent_t *pkevp = NULL;
202 port_kevent_t *lio_pkevp = NULL;
203 int fd;
204 int cleanupqflag;
205 int pollqflag;
206 int portevpend;
207 void (*func)();
208 int use_port = 0;
209 int reqp_flags = 0;
210 int send_signal = 0;
211
212 p = bp->b_proc;
213 as = p->p_as;
214 reqp = (aio_req_t *)bp->b_forw;
215 fd = reqp->aio_req_fd;
216
217 TNF_PROBE_5(aphysio_end, "kaio", /* CSTYLED */,
218 tnf_opaque, bp, bp,
219 tnf_device, device, bp->b_edev,
220 tnf_offset, blkno, btodt(reqp->aio_req_uio.uio_loffset),
221 tnf_size, size, reqp->aio_req_uio.uio_iov->iov_len,
222 tnf_bioflags, rw, (bp->b_flags & (B_READ|B_WRITE)));
223
224 /*
225 * mapout earlier so that more kmem is available when aio is
226 * heavily used. bug #1262082
227 */
228 if (bp->b_flags & B_REMAPPED)
229 bp_mapout(bp);
230
231 /* decrement fd's ref count by one, now that aio request is done. */
232 areleasef(fd, P_FINFO(p));
233
234 aiop = p->p_aio;
235 ASSERT(aiop != NULL);
236
237 mutex_enter(&aiop->aio_portq_mutex);
238 mutex_enter(&aiop->aio_mutex);
239 ASSERT(aiop->aio_pending > 0);
240 ASSERT(reqp->aio_req_flags & AIO_PENDING);
241 aiop->aio_pending--;
242 reqp->aio_req_flags &= ~AIO_PENDING;
243 reqp_flags = reqp->aio_req_flags;
244 if ((pkevp = reqp->aio_req_portkev) != NULL) {
245 /* Event port notification is desired for this transaction */
246 if (reqp->aio_req_flags & AIO_CLOSE_PORT) {
247 /*
248 * The port is being closed and it is waiting for
249 * pending asynchronous I/O transactions to complete.
250 */
251 portevpend = --aiop->aio_portpendcnt;
252 aio_deq(&aiop->aio_portpending, reqp);
253 aio_enq(&aiop->aio_portq, reqp, 0);
254 mutex_exit(&aiop->aio_mutex);
255 mutex_exit(&aiop->aio_portq_mutex);
256 port_send_event(pkevp);
257 if (portevpend == 0)
258 cv_broadcast(&aiop->aio_portcv);
259 return;
260 }
261
262 if (aiop->aio_flags & AIO_CLEANUP) {
263 /*
264 * aio_cleanup_thread() is waiting for completion of
265 * transactions.
266 */
267 mutex_enter(&as->a_contents);
268 aio_deq(&aiop->aio_portpending, reqp);
269 aio_enq(&aiop->aio_portcleanupq, reqp, 0);
270 cv_signal(&aiop->aio_cleanupcv);
271 mutex_exit(&as->a_contents);
272 mutex_exit(&aiop->aio_mutex);
273 mutex_exit(&aiop->aio_portq_mutex);
274 return;
275 }
276
277 aio_deq(&aiop->aio_portpending, reqp);
278 aio_enq(&aiop->aio_portq, reqp, 0);
279
280 use_port = 1;
281 } else {
282 /*
283 * when the AIO_CLEANUP flag is enabled for this
284 * process, or when the AIO_POLL bit is set for
285 * this request, special handling is required.
286 * otherwise the request is put onto the doneq.
287 */
288 cleanupqflag = (aiop->aio_flags & AIO_CLEANUP);
289 pollqflag = (reqp->aio_req_flags & AIO_POLL);
290 if (cleanupqflag | pollqflag) {
291
292 if (cleanupqflag)
293 mutex_enter(&as->a_contents);
294
295 /*
296 * requests with their AIO_POLL bit set are put
297 * on the pollq, requests with sigevent structures
298 * or with listio heads are put on the notifyq, and
299 * the remaining requests don't require any special
300 * cleanup handling, so they're put onto the default
301 * cleanupq.
302 */
303 if (pollqflag)
304 aio_enq(&aiop->aio_pollq, reqp, AIO_POLLQ);
305 else if (reqp->aio_req_sigqp || reqp->aio_req_lio)
306 aio_enq(&aiop->aio_notifyq, reqp, AIO_NOTIFYQ);
307 else
308 aio_enq(&aiop->aio_cleanupq, reqp,
309 AIO_CLEANUPQ);
310
311 if (cleanupqflag) {
312 cv_signal(&aiop->aio_cleanupcv);
313 mutex_exit(&as->a_contents);
314 mutex_exit(&aiop->aio_mutex);
315 mutex_exit(&aiop->aio_portq_mutex);
316 } else {
317 ASSERT(pollqflag);
318 /* block aio_cleanup_exit until we're done */
319 aiop->aio_flags |= AIO_DONE_ACTIVE;
320 mutex_exit(&aiop->aio_mutex);
321 mutex_exit(&aiop->aio_portq_mutex);
322 /*
323 * let the cleanup processing happen from an AST
324 * set an AST on all threads in this process
325 */
326 mutex_enter(&p->p_lock);
327 set_proc_ast(p);
328 mutex_exit(&p->p_lock);
329 mutex_enter(&aiop->aio_mutex);
330 /* wakeup anybody waiting in aiowait() */
331 cv_broadcast(&aiop->aio_waitcv);
332
333 /* wakeup aio_cleanup_exit if needed */
334 if (aiop->aio_flags & AIO_CLEANUP)
335 cv_signal(&aiop->aio_cleanupcv);
336 aiop->aio_flags &= ~AIO_DONE_ACTIVE;
337 mutex_exit(&aiop->aio_mutex);
338 }
339 return;
340 }
341
342 /*
343 * save req's sigevent pointer, and check its
344 * value after releasing aio_mutex lock.
345 */
346 sigev = reqp->aio_req_sigqp;
347 reqp->aio_req_sigqp = NULL;
348
349 /* put request on done queue. */
350 aio_enq(&aiop->aio_doneq, reqp, AIO_DONEQ);
351 } /* portkevent */
352
353 /*
354 * when list IO notification is enabled, a notification or
355 * signal is sent only when all entries in the list are done.
356 */
357 if ((head = reqp->aio_req_lio) != NULL) {
358 ASSERT(head->lio_refcnt > 0);
359 if (--head->lio_refcnt == 0) {
360 /*
361 * save lio's sigevent pointer, and check
362 * its value after releasing aio_mutex lock.
363 */
364 lio_sigev = head->lio_sigqp;
365 head->lio_sigqp = NULL;
366 cv_signal(&head->lio_notify);
367 if (head->lio_port >= 0 &&
368 (lio_pkevp = head->lio_portkev) != NULL)
369 head->lio_port = -1;
370 }
371 }
372
373 /*
374 * if AIO_WAITN set then
375 * send signal only when we reached the
376 * required amount of IO's finished
377 * or when all IO's are done
378 */
379 if (aiop->aio_flags & AIO_WAITN) {
380 if (aiop->aio_waitncnt > 0)
381 aiop->aio_waitncnt--;
382 if (aiop->aio_pending == 0 ||
383 aiop->aio_waitncnt == 0)
384 cv_broadcast(&aiop->aio_waitcv);
385 } else {
386 cv_broadcast(&aiop->aio_waitcv);
387 }
388
389 /*
390 * No need to set this flag for pollq, portq, lio requests.
391 * If this is an old Solaris aio request, and the process has
392 * a SIGIO signal handler enabled, then send a SIGIO signal.
393 */
394 if (!sigev && !use_port && head == NULL &&
395 (reqp->aio_req_flags & AIO_SOLARIS) &&
396 (func = PTOU(p)->u_signal[SIGIO - 1]) != SIG_DFL &&
397 (func != SIG_IGN)) {
398 send_signal = 1;
399 reqp->aio_req_flags |= AIO_SIGNALLED;
400 }
401
402 mutex_exit(&aiop->aio_mutex);
403 mutex_exit(&aiop->aio_portq_mutex);
404
405 /*
406 * Could the cleanup thread be waiting for AIO with locked
407 * resources to finish?
408 * Ideally in that case cleanup thread should block on cleanupcv,
409 * but there is a window, where it could miss to see a new aio
410 * request that sneaked in.
411 */
412 mutex_enter(&as->a_contents);
413 if ((reqp_flags & AIO_PAGELOCKDONE) && AS_ISUNMAPWAIT(as))
414 cv_broadcast(&as->a_cv);
415 mutex_exit(&as->a_contents);
416
417 if (sigev)
418 aio_sigev_send(p, sigev);
419 else if (send_signal)
420 psignal(p, SIGIO);
421
422 if (pkevp)
423 port_send_event(pkevp);
424 if (lio_sigev)
425 aio_sigev_send(p, lio_sigev);
426 if (lio_pkevp)
427 port_send_event(lio_pkevp);
428 }
429
430 /*
431 * send a queued signal to the specified process when
432 * the event signal is non-NULL. A return value of 1
433 * will indicate that a signal is queued, and 0 means that
434 * no signal was specified, nor sent.
435 */
436 static void
437 aio_sigev_send(proc_t *p, sigqueue_t *sigev)
438 {
439 ASSERT(sigev != NULL);
440
441 mutex_enter(&p->p_lock);
442 sigaddqa(p, NULL, sigev);
443 mutex_exit(&p->p_lock);
444 }
445
446 /*
447 * special case handling for zero length requests. the aio request
448 * short circuits the normal completion path since all that's required
449 * to complete this request is to copyout a zero to the aio request's
450 * return value.
451 */
452 void
453 aio_zerolen(aio_req_t *reqp)
454 {
455
456 struct buf *bp = &reqp->aio_req_buf;
457
458 reqp->aio_req_flags |= AIO_ZEROLEN;
459
460 bp->b_forw = (struct buf *)reqp;
461 bp->b_proc = curproc;
462
463 bp->b_resid = 0;
464 bp->b_flags = 0;
465
466 aio_done(bp);
467 }
468
469 /*
470 * unlock pages previously locked by as_pagelock
471 */
472 void
473 aphysio_unlock(aio_req_t *reqp)
474 {
475 struct buf *bp;
476 struct iovec *iov;
477 int flags;
478
479 if (reqp->aio_req_flags & AIO_PHYSIODONE)
480 return;
481
482 reqp->aio_req_flags |= AIO_PHYSIODONE;
483
484 if (reqp->aio_req_flags & AIO_ZEROLEN)
485 return;
486
487 bp = &reqp->aio_req_buf;
488 iov = reqp->aio_req_uio.uio_iov;
489 flags = (((bp->b_flags & B_READ) == B_READ) ? S_WRITE : S_READ);
490 if (reqp->aio_req_flags & AIO_PAGELOCKDONE) {
491 as_pageunlock(bp->b_proc->p_as,
492 bp->b_flags & B_SHADOW ? bp->b_shadow : NULL,
493 iov->iov_base, iov->iov_len, flags);
494 reqp->aio_req_flags &= ~AIO_PAGELOCKDONE;
495 }
496 bp->b_flags &= ~(B_BUSY|B_WANTED|B_PHYS|B_SHADOW);
497 bp->b_flags |= B_DONE;
498 }
499
500 /*
501 * deletes a requests id from the hash table of outstanding io.
502 */
503 static void
504 aio_hash_delete(aio_t *aiop, struct aio_req_t *reqp)
505 {
506 long index;
507 aio_result_t *resultp = reqp->aio_req_resultp;
508 aio_req_t *current;
509 aio_req_t **nextp;
510
511 index = AIO_HASH(resultp);
512 nextp = (aiop->aio_hash + index);
513 while ((current = *nextp) != NULL) {
514 if (current->aio_req_resultp == resultp) {
515 *nextp = current->aio_hash_next;
516 return;
517 }
518 nextp = &current->aio_hash_next;
519 }
520 }
521
522 /*
523 * Put a list head struct onto its free list.
524 */
525 static void
526 aio_lio_free(aio_t *aiop, aio_lio_t *head)
527 {
528 ASSERT(MUTEX_HELD(&aiop->aio_mutex));
529
530 if (head->lio_sigqp != NULL)
531 kmem_free(head->lio_sigqp, sizeof (sigqueue_t));
532 head->lio_next = aiop->aio_lio_free;
533 aiop->aio_lio_free = head;
534 }
535
536 /*
537 * Put a reqp onto the freelist.
538 */
539 void
540 aio_req_free(aio_t *aiop, aio_req_t *reqp)
541 {
542 aio_lio_t *liop;
543
544 ASSERT(MUTEX_HELD(&aiop->aio_mutex));
545
546 if (reqp->aio_req_portkev) {
547 port_free_event(reqp->aio_req_portkev);
548 reqp->aio_req_portkev = NULL;
549 }
550
551 if ((liop = reqp->aio_req_lio) != NULL) {
552 if (--liop->lio_nent == 0)
553 aio_lio_free(aiop, liop);
554 reqp->aio_req_lio = NULL;
555 }
556 if (reqp->aio_req_sigqp != NULL) {
557 kmem_free(reqp->aio_req_sigqp, sizeof (sigqueue_t));
558 reqp->aio_req_sigqp = NULL;
559 }
560 reqp->aio_req_next = aiop->aio_free;
561 reqp->aio_req_prev = NULL;
562 aiop->aio_free = reqp;
563 aiop->aio_outstanding--;
564 if (aiop->aio_outstanding == 0)
565 cv_broadcast(&aiop->aio_waitcv);
566 aio_hash_delete(aiop, reqp);
567 }
568
569 /*
570 * Put a reqp onto the freelist.
571 */
572 void
573 aio_req_free_port(aio_t *aiop, aio_req_t *reqp)
574 {
575 ASSERT(MUTEX_HELD(&aiop->aio_mutex));
576
577 reqp->aio_req_next = aiop->aio_free;
578 reqp->aio_req_prev = NULL;
579 aiop->aio_free = reqp;
580 aiop->aio_outstanding--;
581 aio_hash_delete(aiop, reqp);
582 }
583
584
585 /*
586 * Verify the integrity of a queue.
587 */
588 #if defined(DEBUG)
589 static void
590 aio_verify_queue(aio_req_t *head,
591 aio_req_t *entry_present, aio_req_t *entry_missing)
592 {
593 aio_req_t *reqp;
594 int found = 0;
595 int present = 0;
596
597 if ((reqp = head) != NULL) {
598 do {
599 ASSERT(reqp->aio_req_prev->aio_req_next == reqp);
600 ASSERT(reqp->aio_req_next->aio_req_prev == reqp);
601 if (entry_present == reqp)
602 found++;
603 if (entry_missing == reqp)
604 present++;
605 } while ((reqp = reqp->aio_req_next) != head);
606 }
607 ASSERT(entry_present == NULL || found == 1);
608 ASSERT(entry_missing == NULL || present == 0);
609 }
610 #else
611 #define aio_verify_queue(x, y, z)
612 #endif
613
614 /*
615 * Put a request onto the tail of a queue.
616 */
617 void
618 aio_enq(aio_req_t **qhead, aio_req_t *reqp, int qflg_new)
619 {
620 aio_req_t *head;
621 aio_req_t *prev;
622
623 aio_verify_queue(*qhead, NULL, reqp);
624
625 if ((head = *qhead) == NULL) {
626 reqp->aio_req_next = reqp;
627 reqp->aio_req_prev = reqp;
628 *qhead = reqp;
629 } else {
630 reqp->aio_req_next = head;
631 reqp->aio_req_prev = prev = head->aio_req_prev;
632 prev->aio_req_next = reqp;
633 head->aio_req_prev = reqp;
634 }
635 reqp->aio_req_flags |= qflg_new;
636 }
637
638 /*
639 * Remove a request from its queue.
640 */
641 void
642 aio_deq(aio_req_t **qhead, aio_req_t *reqp)
643 {
644 aio_verify_queue(*qhead, reqp, NULL);
645
646 if (reqp->aio_req_next == reqp) {
647 *qhead = NULL;
648 } else {
649 reqp->aio_req_prev->aio_req_next = reqp->aio_req_next;
650 reqp->aio_req_next->aio_req_prev = reqp->aio_req_prev;
651 if (*qhead == reqp)
652 *qhead = reqp->aio_req_next;
653 }
654 reqp->aio_req_next = NULL;
655 reqp->aio_req_prev = NULL;
656 }
657
658 /*
659 * concatenate a specified queue with the cleanupq. the specified
660 * queue is put onto the tail of the cleanupq. all elements on the
661 * specified queue should have their aio_req_flags field cleared.
662 */
663 /*ARGSUSED*/
664 void
665 aio_cleanupq_concat(aio_t *aiop, aio_req_t *q2, int qflg)
666 {
667 aio_req_t *cleanupqhead, *q2tail;
668 aio_req_t *reqp = q2;
669
670 do {
671 ASSERT(reqp->aio_req_flags & qflg);
672 reqp->aio_req_flags &= ~qflg;
673 reqp->aio_req_flags |= AIO_CLEANUPQ;
674 } while ((reqp = reqp->aio_req_next) != q2);
675
676 cleanupqhead = aiop->aio_cleanupq;
677 if (cleanupqhead == NULL)
678 aiop->aio_cleanupq = q2;
679 else {
680 cleanupqhead->aio_req_prev->aio_req_next = q2;
681 q2tail = q2->aio_req_prev;
682 q2tail->aio_req_next = cleanupqhead;
683 q2->aio_req_prev = cleanupqhead->aio_req_prev;
684 cleanupqhead->aio_req_prev = q2tail;
685 }
686 }
687
688 /*
689 * cleanup aio requests that are on the per-process poll queue.
690 */
691 void
692 aio_cleanup(int flag)
693 {
694 aio_t *aiop = curproc->p_aio;
695 aio_req_t *pollqhead, *cleanupqhead, *notifyqhead;
696 aio_req_t *cleanupport;
697 aio_req_t *portq = NULL;
698 void (*func)();
699 int signalled = 0;
700 int qflag = 0;
701 int exitflg;
702
703 ASSERT(aiop != NULL);
704
705 if (flag == AIO_CLEANUP_EXIT)
706 exitflg = AIO_CLEANUP_EXIT;
707 else
708 exitflg = 0;
709
710 /*
711 * We need to get the aio_cleanupq_mutex because we are calling
712 * aio_cleanup_cleanupq()
713 */
714 mutex_enter(&aiop->aio_cleanupq_mutex);
715 /*
716 * take all the requests off the cleanupq, the notifyq,
717 * and the pollq.
718 */
719 mutex_enter(&aiop->aio_mutex);
720 if ((cleanupqhead = aiop->aio_cleanupq) != NULL) {
721 aiop->aio_cleanupq = NULL;
722 qflag++;
723 }
724 if ((notifyqhead = aiop->aio_notifyq) != NULL) {
725 aiop->aio_notifyq = NULL;
726 qflag++;
727 }
728 if ((pollqhead = aiop->aio_pollq) != NULL) {
729 aiop->aio_pollq = NULL;
730 qflag++;
731 }
732 if (flag) {
733 if ((portq = aiop->aio_portq) != NULL)
734 qflag++;
735
736 if ((cleanupport = aiop->aio_portcleanupq) != NULL) {
737 aiop->aio_portcleanupq = NULL;
738 qflag++;
739 }
740 }
741 mutex_exit(&aiop->aio_mutex);
742
743 /*
744 * return immediately if cleanupq, pollq, and
745 * notifyq are all empty. someone else must have
746 * emptied them.
747 */
748 if (!qflag) {
749 mutex_exit(&aiop->aio_cleanupq_mutex);
750 return;
751 }
752
753 /*
754 * do cleanup for the various queues.
755 */
756 if (cleanupqhead)
757 signalled = aio_cleanup_cleanupq(aiop, cleanupqhead, exitflg);
758 mutex_exit(&aiop->aio_cleanupq_mutex);
759 if (notifyqhead)
760 signalled = aio_cleanup_notifyq(aiop, notifyqhead, exitflg);
761 if (pollqhead)
762 aio_cleanup_pollq(aiop, pollqhead, exitflg);
763 if (flag && (cleanupport || portq))
764 aio_cleanup_portq(aiop, cleanupport, exitflg);
765
766 if (exitflg)
767 return;
768
769 /*
770 * If we have an active aio_cleanup_thread it's possible for
771 * this routine to push something on to the done queue after
772 * an aiowait/aiosuspend thread has already decided to block.
773 * This being the case, we need a cv_broadcast here to wake
774 * these threads up. It is simpler and cleaner to do this
775 * broadcast here than in the individual cleanup routines.
776 */
777
778 mutex_enter(&aiop->aio_mutex);
779 /*
780 * If there has never been an old solaris aio request
781 * issued by this process, then do not send a SIGIO signal.
782 */
783 if (!(aiop->aio_flags & AIO_SOLARIS_REQ))
784 signalled = 1;
785 cv_broadcast(&aiop->aio_waitcv);
786 mutex_exit(&aiop->aio_mutex);
787
788 /*
789 * Only if the process wasn't already signalled,
790 * determine if a SIGIO signal should be delievered.
791 */
792 if (!signalled &&
793 (func = PTOU(curproc)->u_signal[SIGIO - 1]) != SIG_DFL &&
794 func != SIG_IGN)
795 psignal(curproc, SIGIO);
796 }
797
798
799 /*
800 * Do cleanup for every element of the port cleanup queue.
801 */
802 static void
803 aio_cleanup_portq(aio_t *aiop, aio_req_t *cleanupq, int exitflag)
804 {
805 aio_req_t *reqp;
806 aio_req_t *next;
807 aio_req_t *headp;
808 aio_lio_t *liop;
809
810 /* first check the portq */
811 if (exitflag || ((aiop->aio_flags & AIO_CLEANUP_PORT) == 0)) {
812 mutex_enter(&aiop->aio_mutex);
813 if (aiop->aio_flags & AIO_CLEANUP)
814 aiop->aio_flags |= AIO_CLEANUP_PORT;
815 mutex_exit(&aiop->aio_mutex);
816
817 /*
818 * It is not allowed to hold locks during aphysio_unlock().
819 * The aio_done() interrupt function will try to acquire
820 * aio_mutex and aio_portq_mutex. Therefore we disconnect
821 * the portq list from the aiop for the duration of the
822 * aphysio_unlock() loop below.
823 */
824 mutex_enter(&aiop->aio_portq_mutex);
825 headp = aiop->aio_portq;
826 aiop->aio_portq = NULL;
827 mutex_exit(&aiop->aio_portq_mutex);
828 if ((reqp = headp) != NULL) {
829 do {
830 next = reqp->aio_req_next;
831 aphysio_unlock(reqp);
832 if (exitflag) {
833 mutex_enter(&aiop->aio_mutex);
834 aio_req_free(aiop, reqp);
835 mutex_exit(&aiop->aio_mutex);
836 }
837 } while ((reqp = next) != headp);
838 }
839
840 if (headp != NULL && exitflag == 0) {
841 /* move unlocked requests back to the port queue */
842 aio_req_t *newq;
843
844 mutex_enter(&aiop->aio_portq_mutex);
845 if ((newq = aiop->aio_portq) != NULL) {
846 aio_req_t *headprev = headp->aio_req_prev;
847 aio_req_t *newqprev = newq->aio_req_prev;
848
849 headp->aio_req_prev = newqprev;
850 newq->aio_req_prev = headprev;
851 headprev->aio_req_next = newq;
852 newqprev->aio_req_next = headp;
853 }
854 aiop->aio_portq = headp;
855 cv_broadcast(&aiop->aio_portcv);
856 mutex_exit(&aiop->aio_portq_mutex);
857 }
858 }
859
860 /* now check the port cleanup queue */
861 if ((reqp = cleanupq) == NULL)
862 return;
863 do {
864 next = reqp->aio_req_next;
865 aphysio_unlock(reqp);
866 if (exitflag) {
867 mutex_enter(&aiop->aio_mutex);
868 aio_req_free(aiop, reqp);
869 mutex_exit(&aiop->aio_mutex);
870 } else {
871 mutex_enter(&aiop->aio_portq_mutex);
872 aio_enq(&aiop->aio_portq, reqp, 0);
873 mutex_exit(&aiop->aio_portq_mutex);
874 port_send_event(reqp->aio_req_portkev);
875 if ((liop = reqp->aio_req_lio) != NULL) {
876 int send_event = 0;
877
878 mutex_enter(&aiop->aio_mutex);
879 ASSERT(liop->lio_refcnt > 0);
880 if (--liop->lio_refcnt == 0) {
881 if (liop->lio_port >= 0 &&
882 liop->lio_portkev) {
883 liop->lio_port = -1;
884 send_event = 1;
885 }
886 }
887 mutex_exit(&aiop->aio_mutex);
888 if (send_event)
889 port_send_event(liop->lio_portkev);
890 }
891 }
892 } while ((reqp = next) != cleanupq);
893 }
894
895 /*
896 * Do cleanup for every element of the cleanupq.
897 */
898 static int
899 aio_cleanup_cleanupq(aio_t *aiop, aio_req_t *qhead, int exitflg)
900 {
901 aio_req_t *reqp, *next;
902 int signalled = 0;
903
904 ASSERT(MUTEX_HELD(&aiop->aio_cleanupq_mutex));
905
906 /*
907 * Since aio_req_done() or aio_req_find() use the HASH list to find
908 * the required requests, they could potentially take away elements
909 * if they are already done (AIO_DONEQ is set).
910 * The aio_cleanupq_mutex protects the queue for the duration of the
911 * loop from aio_req_done() and aio_req_find().
912 */
913 if ((reqp = qhead) == NULL)
914 return (0);
915 do {
916 ASSERT(reqp->aio_req_flags & AIO_CLEANUPQ);
917 ASSERT(reqp->aio_req_portkev == NULL);
918 next = reqp->aio_req_next;
919 aphysio_unlock(reqp);
920 mutex_enter(&aiop->aio_mutex);
921 if (exitflg)
922 aio_req_free(aiop, reqp);
923 else
924 aio_enq(&aiop->aio_doneq, reqp, AIO_DONEQ);
925 if (!exitflg) {
926 if (reqp->aio_req_flags & AIO_SIGNALLED)
927 signalled++;
928 else
929 reqp->aio_req_flags |= AIO_SIGNALLED;
930 }
931 mutex_exit(&aiop->aio_mutex);
932 } while ((reqp = next) != qhead);
933 return (signalled);
934 }
935
936 /*
937 * do cleanup for every element of the notify queue.
938 */
939 static int
940 aio_cleanup_notifyq(aio_t *aiop, aio_req_t *qhead, int exitflg)
941 {
942 aio_req_t *reqp, *next;
943 aio_lio_t *liohead;
944 sigqueue_t *sigev, *lio_sigev = NULL;
945 int signalled = 0;
946
947 if ((reqp = qhead) == NULL)
948 return (0);
949 do {
950 ASSERT(reqp->aio_req_flags & AIO_NOTIFYQ);
951 next = reqp->aio_req_next;
952 aphysio_unlock(reqp);
953 if (exitflg) {
954 mutex_enter(&aiop->aio_mutex);
955 aio_req_free(aiop, reqp);
956 mutex_exit(&aiop->aio_mutex);
957 } else {
958 mutex_enter(&aiop->aio_mutex);
959 aio_enq(&aiop->aio_doneq, reqp, AIO_DONEQ);
960 sigev = reqp->aio_req_sigqp;
961 reqp->aio_req_sigqp = NULL;
962 if ((liohead = reqp->aio_req_lio) != NULL) {
963 ASSERT(liohead->lio_refcnt > 0);
964 if (--liohead->lio_refcnt == 0) {
965 cv_signal(&liohead->lio_notify);
966 lio_sigev = liohead->lio_sigqp;
967 liohead->lio_sigqp = NULL;
968 }
969 }
970 mutex_exit(&aiop->aio_mutex);
971 if (sigev) {
972 signalled++;
973 aio_sigev_send(reqp->aio_req_buf.b_proc,
974 sigev);
975 }
976 if (lio_sigev) {
977 signalled++;
978 aio_sigev_send(reqp->aio_req_buf.b_proc,
979 lio_sigev);
980 }
981 }
982 } while ((reqp = next) != qhead);
983
984 return (signalled);
985 }
986
987 /*
988 * Do cleanup for every element of the poll queue.
989 */
990 static void
991 aio_cleanup_pollq(aio_t *aiop, aio_req_t *qhead, int exitflg)
992 {
993 aio_req_t *reqp, *next;
994
995 /*
996 * As no other threads should be accessing the queue at this point,
997 * it isn't necessary to hold aio_mutex while we traverse its elements.
998 */
999 if ((reqp = qhead) == NULL)
1000 return;
1001 do {
1002 ASSERT(reqp->aio_req_flags & AIO_POLLQ);
1003 next = reqp->aio_req_next;
1004 aphysio_unlock(reqp);
1005 if (exitflg) {
1006 mutex_enter(&aiop->aio_mutex);
1007 aio_req_free(aiop, reqp);
1008 mutex_exit(&aiop->aio_mutex);
1009 } else {
1010 aio_copyout_result(reqp);
1011 mutex_enter(&aiop->aio_mutex);
1012 aio_enq(&aiop->aio_doneq, reqp, AIO_DONEQ);
1013 mutex_exit(&aiop->aio_mutex);
1014 }
1015 } while ((reqp = next) != qhead);
1016 }
1017
1018 /*
1019 * called by exit(). waits for all outstanding kaio to finish
1020 * before the kaio resources are freed.
1021 */
1022 void
1023 aio_cleanup_exit(void)
1024 {
1025 proc_t *p = curproc;
1026 aio_t *aiop = p->p_aio;
1027 aio_req_t *reqp, *next, *head;
1028 aio_lio_t *nxtlio, *liop;
1029
1030 /*
1031 * wait for all outstanding kaio to complete. process
1032 * is now single-threaded; no other kaio requests can
1033 * happen once aio_pending is zero.
1034 */
1035 mutex_enter(&aiop->aio_mutex);
1036 aiop->aio_flags |= AIO_CLEANUP;
1037 while ((aiop->aio_pending != 0) || (aiop->aio_flags & AIO_DONE_ACTIVE))
1038 cv_wait(&aiop->aio_cleanupcv, &aiop->aio_mutex);
1039 mutex_exit(&aiop->aio_mutex);
1040
1041 /* cleanup the cleanup-thread queues. */
1042 aio_cleanup(AIO_CLEANUP_EXIT);
1043
1044 /*
1045 * Although this process is now single-threaded, we
1046 * still need to protect ourselves against a race with
1047 * aio_cleanup_dr_delete_memory().
1048 */
1049 mutex_enter(&p->p_lock);
1050
1051 /*
1052 * free up the done queue's resources.
1053 */
1054 if ((head = aiop->aio_doneq) != NULL) {
1055 aiop->aio_doneq = NULL;
1056 reqp = head;
1057 do {
1058 next = reqp->aio_req_next;
1059 aphysio_unlock(reqp);
1060 kmem_free(reqp, sizeof (struct aio_req_t));
1061 } while ((reqp = next) != head);
1062 }
1063 /*
1064 * release aio request freelist.
1065 */
1066 for (reqp = aiop->aio_free; reqp != NULL; reqp = next) {
1067 next = reqp->aio_req_next;
1068 kmem_free(reqp, sizeof (struct aio_req_t));
1069 }
1070
1071 /*
1072 * release io list head freelist.
1073 */
1074 for (liop = aiop->aio_lio_free; liop != NULL; liop = nxtlio) {
1075 nxtlio = liop->lio_next;
1076 kmem_free(liop, sizeof (aio_lio_t));
1077 }
1078
1079 if (aiop->aio_iocb)
1080 kmem_free(aiop->aio_iocb, aiop->aio_iocbsz);
1081
1082 mutex_destroy(&aiop->aio_mutex);
1083 mutex_destroy(&aiop->aio_portq_mutex);
1084 mutex_destroy(&aiop->aio_cleanupq_mutex);
1085 p->p_aio = NULL;
1086 mutex_exit(&p->p_lock);
1087 kmem_free(aiop, sizeof (struct aio));
1088 }
1089
1090 /*
1091 * copy out aio request's result to a user-level result_t buffer.
1092 */
1093 void
1094 aio_copyout_result(aio_req_t *reqp)
1095 {
1096 struct buf *bp;
1097 struct iovec *iov;
1098 void *resultp;
1099 int error;
1100 size_t retval;
1101
1102 if (reqp->aio_req_flags & AIO_COPYOUTDONE)
1103 return;
1104
1105 reqp->aio_req_flags |= AIO_COPYOUTDONE;
1106
1107 iov = reqp->aio_req_uio.uio_iov;
1108 bp = &reqp->aio_req_buf;
1109 /* "resultp" points to user-level result_t buffer */
1110 resultp = (void *)reqp->aio_req_resultp;
1111 if (bp->b_flags & B_ERROR) {
1112 if (bp->b_error)
1113 error = bp->b_error;
1114 else
1115 error = EIO;
1116 retval = (size_t)-1;
1117 } else {
1118 error = 0;
1119 retval = iov->iov_len - bp->b_resid;
1120 }
1121 #ifdef _SYSCALL32_IMPL
1122 if (get_udatamodel() == DATAMODEL_NATIVE) {
1123 (void) sulword(&((aio_result_t *)resultp)->aio_return, retval);
1124 (void) suword32(&((aio_result_t *)resultp)->aio_errno, error);
1125 } else {
1126 (void) suword32(&((aio_result32_t *)resultp)->aio_return,
1127 (int)retval);
1128 (void) suword32(&((aio_result32_t *)resultp)->aio_errno, error);
1129 }
1130 #else
1131 (void) suword32(&((aio_result_t *)resultp)->aio_return, retval);
1132 (void) suword32(&((aio_result_t *)resultp)->aio_errno, error);
1133 #endif
1134 }
1135
1136
1137 void
1138 aio_copyout_result_port(struct iovec *iov, struct buf *bp, void *resultp)
1139 {
1140 int errno;
1141 size_t retval;
1142
1143 if (bp->b_flags & B_ERROR) {
1144 if (bp->b_error)
1145 errno = bp->b_error;
1146 else
1147 errno = EIO;
1148 retval = (size_t)-1;
1149 } else {
1150 errno = 0;
1151 retval = iov->iov_len - bp->b_resid;
1152 }
1153 #ifdef _SYSCALL32_IMPL
1154 if (get_udatamodel() == DATAMODEL_NATIVE) {
1155 (void) sulword(&((aio_result_t *)resultp)->aio_return, retval);
1156 (void) suword32(&((aio_result_t *)resultp)->aio_errno, errno);
1157 } else {
1158 (void) suword32(&((aio_result32_t *)resultp)->aio_return,
1159 (int)retval);
1160 (void) suword32(&((aio_result32_t *)resultp)->aio_errno, errno);
1161 }
1162 #else
1163 (void) suword32(&((aio_result_t *)resultp)->aio_return, retval);
1164 (void) suword32(&((aio_result_t *)resultp)->aio_errno, errno);
1165 #endif
1166 }
1167
1168 /*
1169 * This function is used to remove a request from the done queue.
1170 */
1171
1172 void
1173 aio_req_remove_portq(aio_t *aiop, aio_req_t *reqp)
1174 {
1175 ASSERT(MUTEX_HELD(&aiop->aio_portq_mutex));
1176 while (aiop->aio_portq == NULL) {
1177 /*
1178 * aio_portq is set to NULL when aio_cleanup_portq()
1179 * is working with the event queue.
1180 * The aio_cleanup_thread() uses aio_cleanup_portq()
1181 * to unlock all AIO buffers with completed transactions.
1182 * Wait here until aio_cleanup_portq() restores the
1183 * list of completed transactions in aio_portq.
1184 */
1185 cv_wait(&aiop->aio_portcv, &aiop->aio_portq_mutex);
1186 }
1187 aio_deq(&aiop->aio_portq, reqp);
1188 }
1189
1190 /* ARGSUSED */
1191 void
1192 aio_close_port(void *arg, int port, pid_t pid, int lastclose)
1193 {
1194 aio_t *aiop;
1195 aio_req_t *reqp;
1196 aio_req_t *next;
1197 aio_req_t *headp;
1198 int counter;
1199
1200 if (arg == NULL)
1201 aiop = curproc->p_aio;
1202 else
1203 aiop = (aio_t *)arg;
1204
1205 /*
1206 * The PORT_SOURCE_AIO source is always associated with every new
1207 * created port by default.
1208 * If no asynchronous I/O transactions were associated with the port
1209 * then the aiop pointer will still be set to NULL.
1210 */
1211 if (aiop == NULL)
1212 return;
1213
1214 /*
1215 * Within a process event ports can be used to collect events other
1216 * than PORT_SOURCE_AIO events. At the same time the process can submit
1217 * asynchronous I/Os transactions which are not associated with the
1218 * current port.
1219 * The current process oriented model of AIO uses a sigle queue for
1220 * pending events. On close the pending queue (queue of asynchronous
1221 * I/O transactions using event port notification) must be scanned
1222 * to detect and handle pending I/Os using the current port.
1223 */
1224 mutex_enter(&aiop->aio_portq_mutex);
1225 mutex_enter(&aiop->aio_mutex);
1226 counter = 0;
1227 if ((headp = aiop->aio_portpending) != NULL) {
1228 reqp = headp;
1229 do {
1230 if (reqp->aio_req_portkev &&
1231 reqp->aio_req_port == port) {
1232 reqp->aio_req_flags |= AIO_CLOSE_PORT;
1233 counter++;
1234 }
1235 } while ((reqp = reqp->aio_req_next) != headp);
1236 }
1237 if (counter == 0) {
1238 /* no AIOs pending */
1239 mutex_exit(&aiop->aio_mutex);
1240 mutex_exit(&aiop->aio_portq_mutex);
1241 return;
1242 }
1243 aiop->aio_portpendcnt += counter;
1244 mutex_exit(&aiop->aio_mutex);
1245 while (aiop->aio_portpendcnt)
1246 cv_wait(&aiop->aio_portcv, &aiop->aio_portq_mutex);
1247
1248 /*
1249 * all pending AIOs are completed.
1250 * check port doneq
1251 */
1252 headp = NULL;
1253 if ((reqp = aiop->aio_portq) != NULL) {
1254 do {
1255 next = reqp->aio_req_next;
1256 if (reqp->aio_req_port == port) {
1257 /* dequeue request and discard event */
1258 aio_req_remove_portq(aiop, reqp);
1259 port_free_event(reqp->aio_req_portkev);
1260 /* put request in temporary queue */
1261 reqp->aio_req_next = headp;
1262 headp = reqp;
1263 }
1264 } while ((reqp = next) != aiop->aio_portq);
1265 }
1266 mutex_exit(&aiop->aio_portq_mutex);
1267
1268 /* headp points to the list of requests to be discarded */
1269 for (reqp = headp; reqp != NULL; reqp = next) {
1270 next = reqp->aio_req_next;
1271 aphysio_unlock(reqp);
1272 mutex_enter(&aiop->aio_mutex);
1273 aio_req_free_port(aiop, reqp);
1274 mutex_exit(&aiop->aio_mutex);
1275 }
1276
1277 if (aiop->aio_flags & AIO_CLEANUP)
1278 cv_broadcast(&aiop->aio_waitcv);
1279 }
1280
1281 /*
1282 * aio_cleanup_dr_delete_memory is used by dr's delete_memory_thread
1283 * to kick start the aio_cleanup_thread for the give process to do the
1284 * necessary cleanup.
1285 * This is needed so that delete_memory_thread can obtain writer locks
1286 * on pages that need to be relocated during a dr memory delete operation,
1287 * otherwise a deadly embrace may occur.
1288 */
1289 int
1290 aio_cleanup_dr_delete_memory(proc_t *procp)
1291 {
1292 struct aio *aiop = procp->p_aio;
1293 struct as *as = procp->p_as;
1294 int ret = 0;
1295
1296 ASSERT(MUTEX_HELD(&procp->p_lock));
1297
1298 mutex_enter(&as->a_contents);
1299
1300 if (aiop != NULL) {
1301 aiop->aio_rqclnup = 1;
1302 cv_broadcast(&as->a_cv);
1303 ret = 1;
1304 }
1305 mutex_exit(&as->a_contents);
1306 return (ret);
1307 }
unleashed repo fork