sys/rump/librump/rumpuser/rumpuser_pth.c

   1 /*      $NetBSD: rumpuser_pth.c,v 1.39 2009/11/19 14:44:58 pooka Exp $  */
   2
   3 /*
   4  * Copyright (c) 2007 Antti Kantee.  All Rights Reserved.
   5  *
   6  * Development of this software was supported by the
   7  * Finnish Cultural Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
  19  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  20  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  21  * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  24  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 #if !defined(lint)
  33 __RCSID("$NetBSD: rumpuser_pth.c,v 1.39 2009/11/19 14:44:58 pooka Exp $");
  34 #endif /* !lint */
  35
  36 #ifdef __linux__
  37 #define _XOPEN_SOURCE 500
  38 #define _BSD_SOURCE
  39 #define _FILE_OFFSET_BITS 64
  40 #endif
  41
  42 #include <assert.h>
  43 #include <errno.h>
  44 #include <pthread.h>
  45 #include <stdlib.h>
  46 #include <stdio.h>
  47 #include <string.h>
  48 #include <stdint.h>
  49 #include <unistd.h>
  50
  51 #include <rump/rumpuser.h>
  52
  53 #include "rumpuser_int.h"
  54
  55 static pthread_key_t curlwpkey;
  56
  57 #define NOFAIL(a) do {if (!(a)) abort();} while (/*CONSTCOND*/0)
  58 #define NOFAIL_ERRNO(a)                                                 \
  59 do {                                                                    \
  60         int fail_rv = (a);                                              \
  61         if (fail_rv) {                                                  \
  62                 printf("panic: rumpuser fatal failure %d (%s)\n",       \
  63                     fail_rv, strerror(fail_rv));                        \
  64                     abort();                                            \
  65         }                                                               \
  66 } while (/*CONSTCOND*/0)
  67
  68 #define RUMTX_INCRECURSION(mtx) ((mtx)->recursion++)
  69 #define RUMTX_DECRECURSION(mtx) ((mtx)->recursion--)
  70 struct rumpuser_mtx {
  71         pthread_mutex_t pthmtx;
  72         pthread_t owner;
  73         unsigned recursion;
  74 };
  75
  76 #define RURW_AMWRITER(rw) (pthread_equal(rw->writer, pthread_self())    \
  77                                 && rw->readers == -1)
  78 #define RURW_HASREAD(rw)  (rw->readers > 0)
  79
  80 #define RURW_SETWRITE(rw)                                               \
  81 do {                                                                    \
  82         assert(rw->readers == 0);                                       \
  83         rw->writer = pthread_self();                                    \
  84         rw->readers = -1;                                               \
  85 } while (/*CONSTCOND*/0)
  86 #define RURW_CLRWRITE(rw)                                               \
  87 do {                                                                    \
  88         assert(rw->readers == -1 && RURW_AMWRITER(rw));                 \
  89         rw->readers = 0;                                                \
  90 } while (/*CONSTCOND*/0)
  91 #define RURW_INCREAD(rw)                                                \
  92 do {                                                                    \
  93         pthread_spin_lock(&rw->spin);                                   \
  94         assert(rw->readers >= 0);                                       \
  95         ++(rw)->readers;                                                \
  96         pthread_spin_unlock(&rw->spin);                                 \
  97 } while (/*CONSTCOND*/0)
  98 #define RURW_DECREAD(rw)                                                \
  99 do {                                                                    \
 100         pthread_spin_lock(&rw->spin);                                   \
 101         assert(rw->readers > 0);                                        \
 102         --(rw)->readers;                                                \
 103         pthread_spin_unlock(&rw->spin);                                 \
 104 } while (/*CONSTCOND*/0)
 105
 106 struct rumpuser_rw {
 107         pthread_rwlock_t pthrw;
 108         pthread_spinlock_t spin;
 109         int readers;
 110         pthread_t writer;
 111 };
 112
 113 struct rumpuser_cv {
 114         pthread_cond_t pthcv;
 115         int nwaiters;
 116 };
 117
 118 struct rumpuser_mtx rumpuser_aio_mtx;
 119 struct rumpuser_cv rumpuser_aio_cv;
 120 int rumpuser_aio_head, rumpuser_aio_tail;
 121 struct rumpuser_aio rumpuser_aios[N_AIOS];
 122
 123 kernel_lockfn   rumpuser__klock;
 124 kernel_unlockfn rumpuser__kunlock;
 125 int             rumpuser__wantthreads;
 126
 127 void
 128 /*ARGSUSED*/
 129 rumpuser_biothread(void *arg)
 130 {
 131         struct rumpuser_aio *rua;
 132         rump_biodone_fn biodone = arg;
 133         ssize_t rv;
 134         int error, dummy;
 135
 136         /* unschedule from CPU.  we reschedule before running the interrupt */
 137         rumpuser__kunlock(0, &dummy);
 138         assert(dummy == 0);
 139
 140         NOFAIL_ERRNO(pthread_mutex_lock(&rumpuser_aio_mtx.pthmtx));
 141         for (;;) {
 142                 while (rumpuser_aio_head == rumpuser_aio_tail) {
 143                         NOFAIL_ERRNO(pthread_cond_wait(&rumpuser_aio_cv.pthcv,
 144                             &rumpuser_aio_mtx.pthmtx));
 145                 }
 146
 147                 rua = &rumpuser_aios[rumpuser_aio_tail];
 148                 assert(rua->rua_bp != NULL);
 149                 pthread_mutex_unlock(&rumpuser_aio_mtx.pthmtx);
 150
 151                 if (rua->rua_op & RUA_OP_READ) {
 152                         error = 0;
 153                         rv = pread(rua->rua_fd, rua->rua_data,
 154                             rua->rua_dlen, rua->rua_off);
 155                         if (rv < 0) {
 156                                 rv = 0;
 157                                 error = errno;
 158                         }
 159                 } else {
 160                         error = 0;
 161                         rv = pwrite(rua->rua_fd, rua->rua_data,
 162                             rua->rua_dlen, rua->rua_off);
 163                         if (rv < 0) {
 164                                 rv = 0;
 165                                 error = errno;
 166                         } else if (rua->rua_op & RUA_OP_SYNC) {
 167 #ifdef __NetBSD__
 168                                 fsync_range(rua->rua_fd, FDATASYNC,
 169                                     rua->rua_off, rua->rua_dlen);
 170 #else
 171                                 fsync(rua->rua_fd);
 172 #endif
 173                         }
 174                 }
 175                 rumpuser__klock(0);
 176                 biodone(rua->rua_bp, (size_t)rv, error);
 177                 rumpuser__kunlock(0, &dummy);
 178
 179                 rua->rua_bp = NULL;
 180
 181                 NOFAIL_ERRNO(pthread_mutex_lock(&rumpuser_aio_mtx.pthmtx));
 182                 rumpuser_aio_tail = (rumpuser_aio_tail+1) % N_AIOS;
 183                 pthread_cond_signal(&rumpuser_aio_cv.pthcv);
 184         }
 185
 186         /*NOTREACHED*/
 187         fprintf(stderr, "error: rumpuser_biothread reached unreachable\n");
 188         abort();
 189 }
 190
 191 void
 192 rumpuser_thrinit(kernel_lockfn lockfn, kernel_unlockfn unlockfn, int threads)
 193 {
 194
 195         pthread_mutex_init(&rumpuser_aio_mtx.pthmtx, NULL);
 196         pthread_cond_init(&rumpuser_aio_cv.pthcv, NULL);
 197
 198         pthread_key_create(&curlwpkey, NULL);
 199
 200         rumpuser__klock = lockfn;
 201         rumpuser__kunlock = unlockfn;
 202         rumpuser__wantthreads = threads;
 203 }
 204
 205 #if 0
 206 void
 207 rumpuser__thrdestroy(void)
 208 {
 209
 210         pthread_key_delete(curlwpkey);
 211 }
 212 #endif
 213
 214 int
 215 rumpuser_thread_create(void *(*f)(void *), void *arg, const char *thrname)
 216 {
 217         pthread_t ptid;
 218         int rv;
 219
 220         rv = pthread_create(&ptid, NULL, f, arg);
 221 #ifdef __NetBSD__
 222         if (rv == 0 && thrname)
 223                 pthread_setname_np(ptid, thrname, NULL);
 224 #endif
 225
 226         return rv;
 227 }
 228
 229 __dead void
 230 rumpuser_thread_exit(void)
 231 {
 232
 233         pthread_exit(NULL);
 234 }
 235
 236 void
 237 rumpuser_mutex_init(struct rumpuser_mtx **mtx)
 238 {
 239         pthread_mutexattr_t att;
 240
 241         NOFAIL(*mtx = malloc(sizeof(struct rumpuser_mtx)));
 242
 243         pthread_mutexattr_init(&att);
 244         pthread_mutexattr_settype(&att, PTHREAD_MUTEX_ERRORCHECK);
 245         NOFAIL_ERRNO(pthread_mutex_init(&((*mtx)->pthmtx), &att));
 246         pthread_mutexattr_destroy(&att);
 247
 248         (*mtx)->owner = NULL;
 249         (*mtx)->recursion = 0;
 250 }
 251
 252 void
 253 rumpuser_mutex_recursive_init(struct rumpuser_mtx **mtx)
 254 {
 255         pthread_mutexattr_t mattr;
 256
 257         pthread_mutexattr_init(&mattr);
 258         pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_RECURSIVE);
 259
 260         NOFAIL(*mtx = malloc(sizeof(struct rumpuser_mtx)));
 261         NOFAIL_ERRNO(pthread_mutex_init(&((*mtx)->pthmtx), &mattr));
 262         (*mtx)->owner = NULL;
 263         (*mtx)->recursion = 0;
 264
 265         pthread_mutexattr_destroy(&mattr);
 266 }
 267
 268 static void
 269 mtxenter(struct rumpuser_mtx *mtx)
 270 {
 271
 272         if (mtx->recursion++ == 0) {
 273                 assert(mtx->owner == NULL);
 274                 mtx->owner = pthread_self();
 275         } else {
 276                 assert(pthread_equal(mtx->owner, pthread_self()));
 277         }
 278 }
 279
 280 static void
 281 mtxexit(struct rumpuser_mtx *mtx)
 282 {
 283
 284         assert(mtx->owner != NULL);
 285         if (--mtx->recursion == 0)
 286                 mtx->owner = NULL;
 287 }
 288
 289 void
 290 rumpuser_mutex_enter(struct rumpuser_mtx *mtx)
 291 {
 292
 293         if (pthread_mutex_trylock(&mtx->pthmtx) != 0)
 294                 KLOCK_WRAP(NOFAIL_ERRNO(pthread_mutex_lock(&mtx->pthmtx)));
 295         mtxenter(mtx);
 296 }
 297
 298 void
 299 rumpuser_mutex_enter_nowrap(struct rumpuser_mtx *mtx)
 300 {
 301
 302         NOFAIL_ERRNO(pthread_mutex_lock(&mtx->pthmtx));
 303         mtxenter(mtx);
 304 }
 305
 306 int
 307 rumpuser_mutex_tryenter(struct rumpuser_mtx *mtx)
 308 {
 309         int rv;
 310
 311         rv = pthread_mutex_trylock(&mtx->pthmtx);
 312         if (rv == 0) {
 313                 mtxenter(mtx);
 314         }
 315
 316         return rv == 0;
 317 }
 318
 319 void
 320 rumpuser_mutex_exit(struct rumpuser_mtx *mtx)
 321 {
 322
 323         mtxexit(mtx);
 324         NOFAIL_ERRNO(pthread_mutex_unlock(&mtx->pthmtx));
 325 }
 326
 327 void
 328 rumpuser_mutex_destroy(struct rumpuser_mtx *mtx)
 329 {
 330
 331         NOFAIL_ERRNO(pthread_mutex_destroy(&mtx->pthmtx));
 332         free(mtx);
 333 }
 334
 335 int
 336 rumpuser_mutex_held(struct rumpuser_mtx *mtx)
 337 {
 338
 339         return mtx->recursion && pthread_equal(mtx->owner, pthread_self());
 340 }
 341
 342 void
 343 rumpuser_rw_init(struct rumpuser_rw **rw)
 344 {
 345
 346         NOFAIL(*rw = malloc(sizeof(struct rumpuser_rw)));
 347         NOFAIL_ERRNO(pthread_rwlock_init(&((*rw)->pthrw), NULL));
 348         NOFAIL_ERRNO(pthread_spin_init(&((*rw)->spin), PTHREAD_PROCESS_SHARED));
 349         (*rw)->readers = 0;
 350         (*rw)->writer = NULL;
 351 }
 352
 353 void
 354 rumpuser_rw_enter(struct rumpuser_rw *rw, int iswrite)
 355 {
 356
 357         if (iswrite) {
 358                 if (pthread_rwlock_trywrlock(&rw->pthrw) != 0)
 359                         KLOCK_WRAP(NOFAIL_ERRNO(
 360                             pthread_rwlock_wrlock(&rw->pthrw)));
 361                 RURW_SETWRITE(rw);
 362         } else {
 363                 if (pthread_rwlock_tryrdlock(&rw->pthrw) != 0)
 364                         KLOCK_WRAP(NOFAIL_ERRNO(
 365                             pthread_rwlock_rdlock(&rw->pthrw)));
 366                 RURW_INCREAD(rw);
 367         }
 368 }
 369
 370 int
 371 rumpuser_rw_tryenter(struct rumpuser_rw *rw, int iswrite)
 372 {
 373         int rv;
 374
 375         if (iswrite) {
 376                 rv = pthread_rwlock_trywrlock(&rw->pthrw);
 377                 if (rv == 0)
 378                         RURW_SETWRITE(rw);
 379         } else {
 380                 rv = pthread_rwlock_tryrdlock(&rw->pthrw);
 381                 if (rv == 0)
 382                         RURW_INCREAD(rw);
 383         }
 384
 385         return rv == 0;
 386 }
 387
 388 void
 389 rumpuser_rw_exit(struct rumpuser_rw *rw)
 390 {
 391
 392         if (RURW_HASREAD(rw))
 393                 RURW_DECREAD(rw);
 394         else
 395                 RURW_CLRWRITE(rw);
 396         NOFAIL_ERRNO(pthread_rwlock_unlock(&rw->pthrw));
 397 }
 398
 399 void
 400 rumpuser_rw_destroy(struct rumpuser_rw *rw)
 401 {
 402
 403         NOFAIL_ERRNO(pthread_rwlock_destroy(&rw->pthrw));
 404         NOFAIL_ERRNO(pthread_spin_destroy(&rw->spin));
 405         free(rw);
 406 }
 407
 408 int
 409 rumpuser_rw_held(struct rumpuser_rw *rw)
 410 {
 411
 412         return rw->readers != 0;
 413 }
 414
 415 int
 416 rumpuser_rw_rdheld(struct rumpuser_rw *rw)
 417 {
 418
 419         return RURW_HASREAD(rw);
 420 }
 421
 422 int
 423 rumpuser_rw_wrheld(struct rumpuser_rw *rw)
 424 {
 425
 426         return RURW_AMWRITER(rw);
 427 }
 428
 429 void
 430 rumpuser_cv_init(struct rumpuser_cv **cv)
 431 {
 432
 433         NOFAIL(*cv = malloc(sizeof(struct rumpuser_cv)));
 434         NOFAIL_ERRNO(pthread_cond_init(&((*cv)->pthcv), NULL));
 435         (*cv)->nwaiters = 0;
 436 }
 437
 438 void
 439 rumpuser_cv_destroy(struct rumpuser_cv *cv)
 440 {
 441
 442         NOFAIL_ERRNO(pthread_cond_destroy(&cv->pthcv));
 443         free(cv);
 444 }
 445
 446 void
 447 rumpuser_cv_wait(struct rumpuser_cv *cv, struct rumpuser_mtx *mtx)
 448 {
 449
 450         cv->nwaiters++;
 451         assert(mtx->recursion == 1);
 452         mtxexit(mtx);
 453         KLOCK_WRAP(NOFAIL_ERRNO(pthread_cond_wait(&cv->pthcv, &mtx->pthmtx)));
 454         mtxenter(mtx);
 455         cv->nwaiters--;
 456 }
 457
 458 void
 459 rumpuser_cv_wait_nowrap(struct rumpuser_cv *cv, struct rumpuser_mtx *mtx)
 460 {
 461
 462         cv->nwaiters++;
 463         assert(mtx->recursion == 1);
 464         mtxexit(mtx);
 465         NOFAIL_ERRNO(pthread_cond_wait(&cv->pthcv, &mtx->pthmtx));
 466         mtxenter(mtx);
 467         cv->nwaiters--;
 468 }
 469
 470 int
 471 rumpuser_cv_timedwait(struct rumpuser_cv *cv, struct rumpuser_mtx *mtx,
 472         int64_t sec, int64_t nsec)
 473 {
 474         struct timespec ts;
 475         int rv;
 476
 477         /* LINTED */
 478         ts.tv_sec = sec; ts.tv_nsec = nsec;
 479
 480         cv->nwaiters++;
 481         mtxexit(mtx);
 482         KLOCK_WRAP(rv = pthread_cond_timedwait(&cv->pthcv, &mtx->pthmtx, &ts));
 483         mtxenter(mtx);
 484         cv->nwaiters--;
 485         if (rv != 0 && rv != ETIMEDOUT)
 486                 abort();
 487
 488         return rv == ETIMEDOUT;
 489 }
 490
 491 void
 492 rumpuser_cv_signal(struct rumpuser_cv *cv)
 493 {
 494
 495         NOFAIL_ERRNO(pthread_cond_signal(&cv->pthcv));
 496 }
 497
 498 void
 499 rumpuser_cv_broadcast(struct rumpuser_cv *cv)
 500 {
 501
 502         NOFAIL_ERRNO(pthread_cond_broadcast(&cv->pthcv));
 503 }
 504
 505 int
 506 rumpuser_cv_has_waiters(struct rumpuser_cv *cv)
 507 {
 508
 509         return cv->nwaiters;
 510 }
 511
 512 /*
 513  * curlwp
 514  */
 515
 516 void
 517 rumpuser_set_curlwp(struct lwp *l)
 518 {
 519
 520         assert(pthread_getspecific(curlwpkey) == NULL || l == NULL);
 521         pthread_setspecific(curlwpkey, l);
 522 }
 523
 524 struct lwp *
 525 rumpuser_get_curlwp(void)
 526 {
 527
 528         return pthread_getspecific(curlwpkey);
 529 }