sys/dev/lockstat.c

   1 /*      $NetBSD: lockstat.c,v 1.14 2008/04/28 15:36:01 ad Exp $ */
   2
   3 /*-
   4  * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
   5  * All rights reserved.
   6  *
   7  * This code is derived from software contributed to The NetBSD Foundation
   8  * by Andrew Doran.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  29  * POSSIBILITY OF SUCH DAMAGE.
  30  */
  31
  32 /*
  33  * Lock statistics driver, providing kernel support for the lockstat(8)
  34  * command.
  35  *
  36  * We use a global lock word (lockstat_lock) to track device opens.
  37  * Only one thread can hold the device at a time, providing a global lock.
  38  *
  39  * XXX Timings for contention on sleep locks are currently incorrect.
  40  */
  41
  42 #include <sys/cdefs.h>
  43 __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.14 2008/04/28 15:36:01 ad Exp $");
  44
  45 #include <sys/types.h>
  46 #include <sys/param.h>
  47 #include <sys/proc.h>
  48 #include <sys/resourcevar.h>
  49 #include <sys/systm.h>
  50 #include <sys/kernel.h>
  51 #include <sys/kmem.h>
  52 #include <sys/conf.h>
  53 #include <sys/syslog.h>
  54 #include <sys/atomic.h>
  55
  56 #include <dev/lockstat.h>
  57
  58 #include <machine/lock.h>
  59
  60 #ifndef __HAVE_CPU_COUNTER
  61 #error CPU counters not available
  62 #endif
  63
  64 #if LONG_BIT == 64
  65 #define LOCKSTAT_HASH_SHIFT     3
  66 #elif LONG_BIT == 32
  67 #define LOCKSTAT_HASH_SHIFT     2
  68 #endif
  69
  70 #define LOCKSTAT_MINBUFS        1000
  71 #define LOCKSTAT_DEFBUFS        10000
  72 #define LOCKSTAT_MAXBUFS        50000
  73
  74 #define LOCKSTAT_HASH_SIZE      128
  75 #define LOCKSTAT_HASH_MASK      (LOCKSTAT_HASH_SIZE - 1)
  76 #define LOCKSTAT_HASH(key)      \
  77         ((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK)
  78
  79 typedef struct lscpu {
  80         SLIST_HEAD(, lsbuf)     lc_free;
  81         u_int                   lc_overflow;
  82         LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE];
  83 } lscpu_t;
  84
  85 typedef struct lslist lslist_t;
  86
  87 void    lockstatattach(int);
  88 void    lockstat_start(lsenable_t *);
  89 int     lockstat_alloc(lsenable_t *);
  90 void    lockstat_init_tables(lsenable_t *);
  91 int     lockstat_stop(lsdisable_t *);
  92 void    lockstat_free(void);
  93
  94 dev_type_open(lockstat_open);
  95 dev_type_close(lockstat_close);
  96 dev_type_read(lockstat_read);
  97 dev_type_ioctl(lockstat_ioctl);
  98
  99 volatile u_int  lockstat_enabled;
 100 uintptr_t       lockstat_csstart;
 101 uintptr_t       lockstat_csend;
 102 uintptr_t       lockstat_csmask;
 103 uintptr_t       lockstat_lamask;
 104 uintptr_t       lockstat_lockstart;
 105 uintptr_t       lockstat_lockend;
 106 __cpu_simple_lock_t lockstat_lock;
 107 lwp_t           *lockstat_lwp;
 108 lsbuf_t         *lockstat_baseb;
 109 size_t          lockstat_sizeb;
 110 int             lockstat_busy;
 111 struct timespec lockstat_stime;
 112
 113 const struct cdevsw lockstat_cdevsw = {
 114         lockstat_open, lockstat_close, lockstat_read, nowrite, lockstat_ioctl,
 115         nostop, notty, nopoll, nommap, nokqfilter, D_OTHER | D_MPSAFE
 116 };
 117
 118 /*
 119  * Called when the pseudo-driver is attached.
 120  */
 121 void
 122 lockstatattach(int nunits)
 123 {
 124
 125         (void)nunits;
 126
 127         __cpu_simple_lock_init(&lockstat_lock);
 128 }
 129
 130 /*
 131  * Prepare the per-CPU tables for use, or clear down tables when tracing is
 132  * stopped.
 133  */
 134 void
 135 lockstat_init_tables(lsenable_t *le)
 136 {
 137         int i, per, slop, cpuno;
 138         CPU_INFO_ITERATOR cii;
 139         struct cpu_info *ci;
 140         lscpu_t *lc;
 141         lsbuf_t *lb;
 142
 143         KASSERT(!lockstat_enabled);
 144
 145         for (CPU_INFO_FOREACH(cii, ci)) {
 146                 if (ci->ci_lockstat != NULL) {
 147                         kmem_free(ci->ci_lockstat, sizeof(lscpu_t));
 148                         ci->ci_lockstat = NULL;
 149                 }
 150         }
 151
 152         if (le == NULL)
 153                 return;
 154
 155         lb = lockstat_baseb;
 156         per = le->le_nbufs / ncpu;
 157         slop = le->le_nbufs - (per * ncpu);
 158         cpuno = 0;
 159         for (CPU_INFO_FOREACH(cii, ci)) {
 160                 lc = kmem_alloc(sizeof(*lc), KM_SLEEP);
 161                 lc->lc_overflow = 0;
 162                 ci->ci_lockstat = lc;
 163
 164                 SLIST_INIT(&lc->lc_free);
 165                 for (i = 0; i < LOCKSTAT_HASH_SIZE; i++)
 166                         LIST_INIT(&lc->lc_hash[i]);
 167
 168                 for (i = per; i != 0; i--, lb++) {
 169                         lb->lb_cpu = (uint16_t)cpuno;
 170                         SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
 171                 }
 172                 if (--slop > 0) {
 173                         lb->lb_cpu = (uint16_t)cpuno;
 174                         SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
 175                         lb++;
 176                 }
 177                 cpuno++;
 178         }
 179 }
 180
 181 /*
 182  * Start collecting lock statistics.
 183  */
 184 void
 185 lockstat_start(lsenable_t *le)
 186 {
 187
 188         KASSERT(!lockstat_enabled);
 189
 190         lockstat_init_tables(le);
 191
 192         if ((le->le_flags & LE_CALLSITE) != 0)
 193                 lockstat_csmask = (uintptr_t)-1LL;
 194         else
 195                 lockstat_csmask = 0;
 196
 197         if ((le->le_flags & LE_LOCK) != 0)
 198                 lockstat_lamask = (uintptr_t)-1LL;
 199         else
 200                 lockstat_lamask = 0;
 201
 202         lockstat_csstart = le->le_csstart;
 203         lockstat_csend = le->le_csend;
 204         lockstat_lockstart = le->le_lockstart;
 205         lockstat_lockstart = le->le_lockstart;
 206         lockstat_lockend = le->le_lockend;
 207         membar_sync();
 208         getnanotime(&lockstat_stime);
 209         lockstat_enabled = le->le_mask;
 210         membar_producer();
 211 }
 212
 213 /*
 214  * Stop collecting lock statistics.
 215  */
 216 int
 217 lockstat_stop(lsdisable_t *ld)
 218 {
 219         CPU_INFO_ITERATOR cii;
 220         struct cpu_info *ci;
 221         u_int cpuno, overflow;
 222         struct timespec ts;
 223         int error;
 224         lwp_t *l;
 225
 226         KASSERT(lockstat_enabled);
 227
 228         /*
 229          * Set enabled false, force a write barrier, and wait for other CPUs
 230          * to exit lockstat_event().
 231          */
 232         lockstat_enabled = 0;
 233         membar_producer();
 234         getnanotime(&ts);
 235         tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10));
 236
 237         /*
 238          * Did we run out of buffers while tracing?
 239          */
 240         overflow = 0;
 241         for (CPU_INFO_FOREACH(cii, ci))
 242                 overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow;
 243
 244         if (overflow != 0) {
 245                 error = EOVERFLOW;
 246                 log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n",
 247                     overflow);
 248         } else
 249                 error = 0;
 250
 251         lockstat_init_tables(NULL);
 252
 253         /* Run through all LWPs and clear the slate for the next run. */
 254         mutex_enter(proc_lock);
 255         LIST_FOREACH(l, &alllwp, l_list) {
 256                 l->l_pfailaddr = 0;
 257                 l->l_pfailtime = 0;
 258                 l->l_pfaillock = 0;
 259         }
 260         mutex_exit(proc_lock);
 261
 262         if (ld == NULL)
 263                 return error;
 264
 265         /*
 266          * Fill out the disable struct for the caller.
 267          */
 268         timespecsub(&ts, &lockstat_stime, &ld->ld_time);
 269         ld->ld_size = lockstat_sizeb;
 270
 271         cpuno = 0;
 272         for (CPU_INFO_FOREACH(cii, ci)) {
 273                 if (cpuno > sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) {
 274                         log(LOG_WARNING, "lockstat: too many CPUs\n");
 275                         break;
 276                 }
 277                 ld->ld_freq[cpuno++] = cpu_frequency(ci);
 278         }
 279
 280         return error;
 281 }
 282
 283 /*
 284  * Allocate buffers for lockstat_start().
 285  */
 286 int
 287 lockstat_alloc(lsenable_t *le)
 288 {
 289         lsbuf_t *lb;
 290         size_t sz;
 291
 292         KASSERT(!lockstat_enabled);
 293         lockstat_free();
 294
 295         sz = sizeof(*lb) * le->le_nbufs;
 296
 297         lb = kmem_zalloc(sz, KM_SLEEP);
 298         if (lb == NULL)
 299                 return (ENOMEM);
 300
 301         KASSERT(!lockstat_enabled);
 302         KASSERT(lockstat_baseb == NULL);
 303         lockstat_sizeb = sz;
 304         lockstat_baseb = lb;
 305
 306         return (0);
 307 }
 308
 309 /*
 310  * Free allocated buffers after tracing has stopped.
 311  */
 312 void
 313 lockstat_free(void)
 314 {
 315
 316         KASSERT(!lockstat_enabled);
 317
 318         if (lockstat_baseb != NULL) {
 319                 kmem_free(lockstat_baseb, lockstat_sizeb);
 320                 lockstat_baseb = NULL;
 321         }
 322 }
 323
 324 /*
 325  * Main entry point from lock primatives.
 326  */
 327 void
 328 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count,
 329                uint64_t cycles)
 330 {
 331         lslist_t *ll;
 332         lscpu_t *lc;
 333         lsbuf_t *lb;
 334         u_int event;
 335         int s;
 336
 337         if ((flags & lockstat_enabled) != flags || count == 0)
 338                 return;
 339         if (lock < lockstat_lockstart || lock > lockstat_lockend)
 340                 return;
 341         if (callsite < lockstat_csstart || callsite > lockstat_csend)
 342                 return;
 343
 344         callsite &= lockstat_csmask;
 345         lock &= lockstat_lamask;
 346
 347         /*
 348          * Find the table for this lock+callsite pair, and try to locate a
 349          * buffer with the same key.
 350          */
 351         s = splhigh();
 352         lc = curcpu()->ci_lockstat;
 353         ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)];
 354         event = (flags & LB_EVENT_MASK) - 1;
 355
 356         LIST_FOREACH(lb, ll, lb_chain.list) {
 357                 if (lb->lb_lock == lock && lb->lb_callsite == callsite)
 358                         break;
 359         }
 360
 361         if (lb != NULL) {
 362                 /*
 363                  * We found a record.  Move it to the front of the list, as
 364                  * we're likely to hit it again soon.
 365                  */
 366                 if (lb != LIST_FIRST(ll)) {
 367                         LIST_REMOVE(lb, lb_chain.list);
 368                         LIST_INSERT_HEAD(ll, lb, lb_chain.list);
 369                 }
 370                 lb->lb_counts[event] += count;
 371                 lb->lb_times[event] += cycles;
 372         } else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) {
 373                 /*
 374                  * Pinch a new buffer and fill it out.
 375                  */
 376                 SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist);
 377                 LIST_INSERT_HEAD(ll, lb, lb_chain.list);
 378                 lb->lb_flags = (uint16_t)flags;
 379                 lb->lb_lock = lock;
 380                 lb->lb_callsite = callsite;
 381                 lb->lb_counts[event] = count;
 382                 lb->lb_times[event] = cycles;
 383         } else {
 384                 /*
 385                  * We didn't find a buffer and there were none free.
 386                  * lockstat_stop() will notice later on and report the
 387                  * error.
 388                  */
 389                  lc->lc_overflow++;
 390         }
 391
 392         splx(s);
 393 }
 394
 395 /*
 396  * Accept an open() on /dev/lockstat.
 397  */
 398 int
 399 lockstat_open(dev_t dev, int flag, int mode, lwp_t *l)
 400 {
 401
 402         if (!__cpu_simple_lock_try(&lockstat_lock))
 403                 return EBUSY;
 404         lockstat_lwp = curlwp;
 405         return 0;
 406 }
 407
 408 /*
 409  * Accept the last close() on /dev/lockstat.
 410  */
 411 int
 412 lockstat_close(dev_t dev, int flag, int mode, lwp_t *l)
 413 {
 414
 415         lockstat_lwp = NULL;
 416         __cpu_simple_unlock(&lockstat_lock);
 417         return 0;
 418 }
 419
 420 /*
 421  * Handle control operations.
 422  */
 423 int
 424 lockstat_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l)
 425 {
 426         lsenable_t *le;
 427         int error;
 428
 429         if (lockstat_lwp != curlwp)
 430                 return EBUSY;
 431
 432         switch (cmd) {
 433         case IOC_LOCKSTAT_GVERSION:
 434                 *(int *)data = LS_VERSION;
 435                 error = 0;
 436                 break;
 437
 438         case IOC_LOCKSTAT_ENABLE:
 439                 le = (lsenable_t *)data;
 440
 441                 if (!cpu_hascounter()) {
 442                         error = ENODEV;
 443                         break;
 444                 }
 445                 if (lockstat_enabled) {
 446                         error = EBUSY;
 447                         break;
 448                 }
 449
 450                 /*
 451                  * Sanitize the arguments passed in and set up filtering.
 452                  */
 453                 if (le->le_nbufs == 0)
 454                         le->le_nbufs = LOCKSTAT_DEFBUFS;
 455                 else if (le->le_nbufs > LOCKSTAT_MAXBUFS ||
 456                     le->le_nbufs < LOCKSTAT_MINBUFS) {
 457                         error = EINVAL;
 458                         break;
 459                 }
 460                 if ((le->le_flags & LE_ONE_CALLSITE) == 0) {
 461                         le->le_csstart = 0;
 462                         le->le_csend = le->le_csstart - 1;
 463                 }
 464                 if ((le->le_flags & LE_ONE_LOCK) == 0) {
 465                         le->le_lockstart = 0;
 466                         le->le_lockend = le->le_lockstart - 1;
 467                 }
 468                 if ((le->le_mask & LB_EVENT_MASK) == 0)
 469                         return EINVAL;
 470                 if ((le->le_mask & LB_LOCK_MASK) == 0)
 471                         return EINVAL;
 472
 473                 /*
 474                  * Start tracing.
 475                  */
 476                 if ((error = lockstat_alloc(le)) == 0)
 477                         lockstat_start(le);
 478                 break;
 479
 480         case IOC_LOCKSTAT_DISABLE:
 481                 if (!lockstat_enabled)
 482                         error = EINVAL;
 483                 else
 484                         error = lockstat_stop((lsdisable_t *)data);
 485                 break;
 486
 487         default:
 488                 error = ENOTTY;
 489                 break;
 490         }
 491
 492         return error;
 493 }
 494
 495 /*
 496  * Copy buffers out to user-space.
 497  */
 498 int
 499 lockstat_read(dev_t dev, struct uio *uio, int flag)
 500 {
 501
 502         if (curlwp != lockstat_lwp || lockstat_enabled)
 503                 return EBUSY;
 504         return uiomove(lockstat_baseb, lockstat_sizeb, uio);
 505 }