include/sys/cyclic_impl.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26 #ifndef _SYS_CYCLIC_IMPL_H
  27 #define _SYS_CYCLIC_IMPL_H
  28
  29 #ifdef  __cplusplus
  30 extern "C" {
  31 #endif
  32
  33 #include <sys/cyclic.h>
  34 #include <sys/rwlock.h>
  35
  36 /*
  37  *  Cyclic Subsystem Backend-supplied Interfaces
  38  *  --------------------------------------------
  39  *
  40  *  0  Background
  41  *
  42  *    The design, implementation and interfaces of the cyclic subsystem are
  43  *    covered in detail in block comments in the implementation.  This
  44  *    comment covers the interface from the cyclic subsystem into the cyclic
  45  *    backend.  The backend is specified by a structure of function pointers
  46  *    defined below.
  47  *
  48  *  1  Overview
  49  *
  50  *      cyb_configure()      <-- Configures the backend on the specified CPU
  51  *      cyb_unconfigure()    <-- Unconfigures the backend
  52  *      cyb_enable()         <-- Enables the CY_HIGH_LEVEL interrupt source
  53  *      cyb_disable()        <-- Disables the CY_HIGH_LEVEL interrupt source
  54  *      cyb_reprogram()      <-- Reprograms the CY_HIGH_LEVEL interrupt source
  55  *      cyb_softint()        <-- Generates a soft interrupt
  56  *      cyb_set_level()      <-- Sets the programmable interrupt level
  57  *      cyb_restore_level()  <-- Restores the programmable interrupt level
  58  *      cyb_xcall()          <-- Cross calls to the specified CPU
  59  *      cyb_suspend()        <-- Suspends the backend
  60  *      cyb_resume()         <-- Resumes the backend
  61  *
  62  *  2  cyb_arg_t cyb_configure(cpu_t *)
  63  *
  64  *  2.1  Overview
  65  *
  66  *    cyb_configure() should configure the specified CPU for cyclic operation.
  67  *
  68  *  2.2  Arguments and notes
  69  *
  70  *    cyb_configure() should initialize any backend-specific per-CPU
  71  *    structures for the specified CPU.  cyb_configure() will be called for
  72  *    each CPU (including the boot CPU) during boot.  If the platform
  73  *    supports dynamic reconfiguration, cyb_configure() will be called for
  74  *    new CPUs as they are configured into the system.
  75  *
  76  *  2.3  Return value
  77  *
  78  *    cyb_configure() is expected to return a cookie (a cyb_arg_t, which is
  79  *    of type void *) which will be used as the first argument for all future
  80  *    cyclic calls into the backend on the specified CPU.
  81  *
  82  *  2.4  Caller's context
  83  *
  84  *    cpu_lock will be held.  The caller's CPU is unspecified, and may or
  85  *    may not be the CPU specified to cyb_configure().
  86  *
  87  *  3  void cyb_unconfigure(cyb_arg_t arg)
  88  *
  89  *  3.1  Overview
  90  *
  91  *    cyb_unconfigure() should unconfigure the specified backend.
  92  *
  93  *  3.2  Arguments and notes
  94  *
  95  *    The only argument to cyb_unconfigure() is a cookie as returned from
  96  *    cyb_configure().
  97  *
  98  *    cyb_unconfigure() should free any backend-specific per-CPU structures
  99  *    for the specified backend.  cyb_unconfigure() will _only_ be called on
 100  *    platforms which support dynamic reconfiguration.  If the platform does
 101  *    not support dynamic reconfiguration, cyb_unconfigure() may panic.
 102  *
 103  *    After cyb_unconfigure() returns, the backend must not call cyclic_fire()
 104  *    on the corresponding CPU; doing so will result in a bad trap.
 105  *
 106  *  3.3  Return value
 107  *
 108  *    None.
 109  *
 110  *  3.4  Caller's context
 111  *
 112  *    cpu_lock will be held.  The caller's CPU is unspecified, and may or
 113  *    may not be the CPU specified to cyb_unconfigure().  The specified
 114  *    CPU is guaranteed to exist at the time cyb_unconfigure() is called.
 115  *    The cyclic subsystem is guaranteed to be suspended when cyb_unconfigure()
 116  *    is called, and interrupts are guaranteed to be disabled.
 117  *
 118  *  4  void cyb_enable(cyb_arg_t arg)
 119  *
 120  *  4.1  Overview
 121  *
 122  *    cyb_enable() should enable the CY_HIGH_LEVEL interrupt source on
 123  *    the specified backend.
 124  *
 125  *  4.2  Arguments and notes
 126  *
 127  *    The only argument to cyb_enable() is a backend cookie as returned from
 128  *    cyb_configure().
 129  *
 130  *    cyb_enable() will only be called if a) the specified backend has never
 131  *    been enabled or b) the specified backend has been explicitly disabled with
 132  *    cyb_disable().  In either case, cyb_enable() will only be called if
 133  *    the cyclic subsystem wishes to add a cyclic to the CPU corresponding
 134  *    to the specified backend.  cyb_enable() will be called before
 135  *    cyb_reprogram() for a given backend.
 136  *
 137  *    cyclic_fire() should not be called on a CPU which has not had its backend
 138  *    explicitly cyb_enable()'d, but to do so does not constitute fatal error.
 139  *
 140  *  4.3  Return value
 141  *
 142  *    None.
 143  *
 144  *  4.4  Caller's context
 145  *
 146  *    cyb_enable() will only be called from CY_HIGH_LEVEL context on the CPU
 147  *    corresponding to the specified backend.
 148  *
 149  *  5  void cyb_disable(cyb_arg_t arg)
 150  *
 151  *  5.1  Overview
 152  *
 153  *    cyb_disable() should disable the CY_HIGH_LEVEL interrupt source on
 154  *    the specified backend.
 155  *
 156  *  5.2  Arguments and notes
 157  *
 158  *    The only argument to cyb_disable() is a backend cookie as returned from
 159  *    cyb_configure().
 160  *
 161  *    cyb_disable() will only be called on backends which have been previously
 162  *    been cyb_enable()'d.  cyb_disable() will be called when all cyclics have
 163  *    been juggled away or removed from a cyb_enable()'d CPU.
 164  *
 165  *    cyclic_fire() should not be called on a CPU which has had its backend
 166  *    explicitly cyb_disable()'d, but to do so does not constitute fatal
 167  *    error.  cyb_disable() is thus not required to check for a pending
 168  *    CY_HIGH_LEVEL interrupt.
 169  *
 170  *  5.3  Return value
 171  *
 172  *    None.
 173  *
 174  *  5.4  Caller's context
 175  *
 176  *    cyb_disable() will only be called from CY_HIGH_LEVEL context on the CPU
 177  *    corresponding to the specified backend.
 178  *
 179  *  6  void cyb_reprogram(cyb_arg_t arg, hrtime_t time)
 180  *
 181  *  6.1  Overview
 182  *
 183  *    cyb_reprogram() should reprogram the CY_HIGH_LEVEL interrupt source
 184  *    to fire at the absolute time specified.
 185  *
 186  *  6.2  Arguments and notes
 187  *
 188  *    The first argument to cyb_reprogram() is a backend cookie as returned from
 189  *    cyb_configure().
 190  *
 191  *    The second argument is an absolute time at which the CY_HIGH_LEVEL
 192  *    interrupt should fire.  The specified time _may_ be in the past (albeit
 193  *    the very recent past).  If this is the case, the backend should generate
 194  *    a CY_HIGH_LEVEL interrupt as soon as possible.
 195  *
 196  *    The platform should not assume that cyb_reprogram() will be called with
 197  *    monotonically increasing values.
 198  *
 199  *    If the platform does not allow for interrupts at arbitrary times in the
 200  *    future, cyb_reprogram() may do nothing -- as long as cyclic_fire() is
 201  *    called periodically at CY_HIGH_LEVEL.  While this is clearly suboptimal
 202  *    (cyclic granularity will be bounded by the length of the period between
 203  *    cyclic_fire()'s), it allows the cyclic subsystem to be implemented on
 204  *    inferior hardware.
 205  *
 206  *  6.3  Return value
 207  *
 208  *     None.
 209  *
 210  *  6.4  Caller's context
 211  *
 212  *    cyb_reprogram() will only be called from CY_HIGH_LEVEL context on the CPU
 213  *    corresponding to the specified backend.
 214  *
 215  *  7  void cyb_softint(cyb_arg_t arg, cyc_level_t level)
 216  *
 217  *  7.1  Overview
 218  *
 219  *    cyb_softint() should generate a software interrupt on the specified
 220  *    backend at the specified level.
 221  *
 222  *  7.2  Arguments and notes
 223  *
 224  *    The first argument to cyb_softint() is a backend cookie as returned from
 225  *    cyb_configure().  The second argument is the interrupt level at which
 226  *    the software interrupt should be generated; it will be either
 227  *    CY_LOCK_LEVEL or CY_LOW_LEVEL.
 228  *
 229  *    The software interrupt _must_ be generated on the CPU corresponding
 230  *    to the specified backend; platforms are _required_ to have a per-CPU
 231  *    notion of a software interrupt.
 232  *
 233  *    Unless a software interrupt is already pending at the specified level,
 234  *    the software interrupt _must_ be generated.  Once cyclic_softint()
 235  *    has been called at a given level, the software interrupt at that level
 236  *    should no longer be considered pending; an intervening CY_HIGH_LEVEL
 237  *    interrupt and subsequent cyb_softint() must generate another software
 238  *    interrupt.
 239  *
 240  *  7.3  Return value
 241  *
 242  *    None.
 243  *
 244  *  7.4  Caller's context
 245  *
 246  *    cyb_softint() will only be called at a level higher than the one
 247  *    specified:  if CY_LOCK_LEVEL is specified, the caller will be at
 248  *    CY_HIGH_LEVEL; if CY_LOW_LEVEL is specified, the caller will be at
 249  *    either CY_HIGH_LEVEL or CY_LOCK_LEVEL.  cyb_softint() will only be
 250  *    called on the CPU corresponding to the specified backend.
 251  *
 252  *  8  cyb_set_level(cyb_arg_t arg, cyc_level_t level)
 253  *
 254  *  8.1  Overview
 255  *
 256  *    cyb_set_level() should set the programmable interrupt level to the
 257  *    level specified.
 258  *
 259  *  8.2  Arguments and notes
 260  *
 261  *    The first argument to cyb_set_level() is a backend cookie as returned
 262  *    from cyb_configure().  The second argument is the level to which
 263  *    the programmable interrupt level should be set; it will be one of
 264  *    CY_HIGH_LEVEL, CY_LOCK_LEVEL or CY_LOW_LEVEL.
 265  *
 266  *    After cyb_set_level() returns, the CPU associated with the specified
 267  *    backend should accept no interrupt at a level greater than or equal to
 268  *    the specified level.  This will generally be a wrapper around splx().
 269  *
 270  *    The cyclic subsystem will never call cyb_set_level() twice consecutively
 271  *    on the same backend; there will always be an intervening
 272  *    cyb_restore_level();
 273  *
 274  *  8.3  Return value
 275  *
 276  *    cyb_set_level() should return a cookie to be passed back to
 277  *    cyb_restore_level().  On most implementations, this cookie will be
 278  *    the spl at the time of cyb_set_level().
 279  *
 280  *  8.4  Caller's context
 281  *
 282  *    cyb_set_level() is unique in that it is the only backend-provided
 283  *    interface which may be called in cross call context (see cyb_xcall(),
 284  *    below).  cyb_set_level() may also be called from any of the cyclic
 285  *
 286  *  9  cyb_restore_level(cyb_arg_t arg, cyc_cookie_t cookie)
 287  *
 288  *  9.1  Overview
 289  *
 290  *    cyb_restore_level() should restore the programmable interrupt level
 291  *    based upon the specified cookie.
 292  *
 293  *  9.2  Arguments and notes
 294  *
 295  *    The first argument to cyb_restore_level() is a backend cookie as returned
 296  *    from cyb_configure().  The second argument is a cookie as returned from
 297  *    cyb_set_level().
 298  *
 299  *    cyb_restore_level() should restore the programmable interrupt level
 300  *    to its value when cyb_set_level() was called; the cookie is used
 301  *    to provide a hint to the backend.  cyb_restore_level() will not be
 302  *    called without a proceeding call to cyb_set_level(), and
 303  *    cyb_restore_level() will never be called twice consecutively on the
 304  *    same backend.
 305  *
 306  *  9.3  Return value
 307  *
 308  *    None.
 309  *
 310  *  9.4  Caller's context
 311  *
 312  *    The constraints outlined in 5.9.2 imply that cyb_restore_level() can
 313  *    only be called from CY_HIGH_LEVEL, CY_LOCK_LEVEL or CY_LOW_LEVEL context.
 314  *    cyb_restore_level() is always called on the CPU associated with the
 315  *    specified backend.
 316  *
 317  *  10  cyb_xcall(cyb_arg_t arg, cpu_t *, void(*func)(void *), void *farg)
 318  *
 319  *  10.1  Overview
 320  *
 321  *    cyb_xcall() should execute the specified function on the specified CPU.
 322  *
 323  *  10.2  Arguments and notes
 324  *
 325  *    The first argument to cyb_restore_level() is a backend cookie as returned
 326  *    from cyb_configure().  The second argument is a CPU on which the third
 327  *    argument, a function pointer, should be executed.  The fourth argument,
 328  *    a void *, should be passed as the argument to the specified function.
 329  *
 330  *    cyb_xcall() must provide exactly-once semantics.  If the specified
 331  *    function is called more than once, or not at all, the cyclic subsystem
 332  *    will become internally inconsistent.  The specified function must be
 333  *    be executed on the specified CPU, but may be executed in any context
 334  *    (any interrupt context or kernel context).
 335  *
 336  *    cyb_xcall() cannot block.  Any resources which cyb_xcall() needs to
 337  *    acquire must thus be protected by synchronization primitives which
 338  *    never require the caller to block.
 339  *
 340  *  10.3  Return value
 341  *
 342  *    None.
 343  *
 344  *  10.4  Caller's context
 345  *
 346  *    cpu_lock will be held and kernel preemption may be disabled.  The caller
 347  *    may be unable to block, giving rise to the constraint outlined in
 348  *    10.2, above.
 349  *
 350  *  11  cyb_suspend(cyb_arg_t arg)
 351  *
 352  *  11.1  Overview
 353  *
 354  *    cyb_suspend() should suspend the specified backend.
 355  *
 356  *  11.2  Arguments and notes
 357  *
 358  *    The only argument to cyb_suspend() is a backend cookie as returned from
 359  *    cyb_configure().
 360  *
 361  *    cyb_suspend() will never be called on enabled backends.  The backend
 362  *    should assume that the machine may be subsequently powered off; any
 363  *    volatile hardware state should be preserved and restored in cyb_resume().
 364  *    However, the backend should not _assume_ that the machine will be
 365  *    powered off; cyb_suspend() may also be called as part of dynamic
 366  *    reconfiguration.
 367  *
 368  *    cyb_suspend() will be called on the corresponding backend of each
 369  *    CPU in the system in succession, regardless of CPU state (P_ONLINE,
 370  *    P_OFFLINE, P_NOINTR).  The cyclic subsystem will not suspend only a
 371  *    fraction of the CPUs.
 372  *
 373  *  11.3  Return value
 374  *
 375  *    None.
 376  *
 377  *  11.4  Caller's context
 378  *
 379  *    cyb_suspend() will be called in cross call context on the CPU associated
 380  *    with the specified backend.
 381  *
 382  *  12  cyb_resume(cyb_arg_t arg)
 383  *
 384  *  12.1  Overview
 385  *
 386  *    cyb_resume() should resume the specified backend.
 387  *
 388  *  12.2  Arguments and notes
 389  *
 390  *    The only argument to cyb_resume() is a backend cookie as returned from
 391  *    cyb_resume().
 392  *
 393  *    Calls to cyb_resume() will always have been proceeded by corresponding
 394  *    calls to cyb_suspend().  The machine may have been powered off between
 395  *    cyb_suspend() and the call to cyb_resume().  cyb_resume() may decide
 396  *    to restore hardware to its state at the time cyb_suspend() was called.
 397  *
 398  *    The cyclic subsystem will make no calls into the backend between
 399  *    cyb_suspend() and cyb_resume().
 400  *
 401  *  12.3  Return value
 402  *
 403  *    None.
 404  *
 405  *  12.4  Caller's context
 406  *
 407  *    cyb_resume() will be called in cross call context on the CPU associated
 408  *    with the specified backend.
 409  */
 410 typedef struct cyc_backend {
 411         cyb_arg_t (*cyb_configure)(cpu_t *);
 412         void (*cyb_unconfigure)(cyb_arg_t);
 413         void (*cyb_enable)(cyb_arg_t);
 414         void (*cyb_disable)(cyb_arg_t);
 415         void (*cyb_reprogram)(cyb_arg_t, hrtime_t);
 416         void (*cyb_softint)(cyb_arg_t, cyc_level_t);
 417         cyc_cookie_t (*cyb_set_level)(cyb_arg_t, cyc_level_t);
 418         void (*cyb_restore_level)(cyb_arg_t, cyc_cookie_t);
 419         void (*cyb_xcall)(cyb_arg_t, cpu_t *, cyc_func_t, void *);
 420         void (*cyb_suspend)(cyb_arg_t);
 421         void (*cyb_resume)(cyb_arg_t);
 422         cyb_arg_t cyb_arg;
 423 } cyc_backend_t;
 424
 425 extern void cyclic_init(cyc_backend_t *be, hrtime_t resolution);
 426 extern void cyclic_mp_init();
 427
 428 #ifdef DEBUG
 429 #define CYCLIC_TRACE
 430 #endif
 431
 432 typedef enum {
 433         CYS_ONLINE,
 434         CYS_OFFLINE,
 435         CYS_EXPANDING,
 436         CYS_REMOVING,
 437         CYS_SUSPENDED
 438 } cyc_state_t;
 439
 440 #define CYF_FREE                0x0001
 441 #define CYF_CPU_BOUND           0x0002
 442 #define CYF_PART_BOUND          0x0004
 443
 444 typedef struct cyclic {
 445         hrtime_t cy_expire;
 446         hrtime_t cy_interval;
 447         void (*cy_handler)(void *);
 448         void *cy_arg;
 449         uint32_t cy_pend;
 450         uint16_t cy_flags;
 451         cyc_level_t cy_level;
 452 } cyclic_t;
 453
 454 typedef struct cyc_pcbuffer {
 455         cyc_index_t *cypc_buf;
 456         int cypc_prodndx;
 457         int cypc_consndx;
 458         int cypc_sizemask;
 459 } cyc_pcbuffer_t;
 460
 461 typedef struct cyc_softbuf {
 462         uchar_t cys_hard;               /* Can only be zero or one */
 463         uchar_t cys_soft;               /* Can only be zero or one */
 464         cyc_pcbuffer_t cys_buf[2];
 465 } cyc_softbuf_t;
 466
 467 #define CY_NTRACEREC            512
 468
 469 typedef struct cyc_tracerec {
 470         hrtime_t cyt_tstamp;
 471         char *cyt_why;
 472         uint64_t cyt_arg0;
 473         uint64_t cyt_arg1;
 474 } cyc_tracerec_t;
 475
 476 typedef struct cyc_tracebuf {
 477         int cyt_ndx;
 478         cyc_tracerec_t cyt_buf[CY_NTRACEREC];
 479 } cyc_tracebuf_t;
 480
 481 #define CY_NCOVERAGE    127
 482
 483 typedef struct cyc_coverage {
 484         char *cyv_why;
 485         int cyv_passive_count;
 486         int cyv_count[CY_LEVELS];
 487         uint64_t cyv_arg0;
 488         uint64_t cyv_arg1;
 489 } cyc_coverage_t;
 490
 491 typedef struct cyc_cpu {
 492         cpu_t *cyp_cpu;
 493         cyc_index_t *cyp_heap;
 494         cyclic_t *cyp_cyclics;
 495         cyc_index_t cyp_nelems;
 496         cyc_index_t cyp_size;
 497         cyc_state_t cyp_state;
 498         cyc_softbuf_t cyp_softbuf[CY_SOFT_LEVELS];
 499         cyc_backend_t *cyp_backend;
 500         ksema_t cyp_modify_wait;
 501         uint32_t cyp_modify_levels;
 502         uint32_t cyp_rpend;
 503 #ifdef CYCLIC_TRACE
 504         cyc_tracebuf_t cyp_trace[CY_LEVELS];
 505 #endif
 506 } cyc_cpu_t;
 507
 508 typedef struct cyc_omni_cpu {
 509         cyc_cpu_t *cyo_cpu;
 510         cyc_index_t cyo_ndx;
 511         void *cyo_arg;
 512         struct cyc_omni_cpu *cyo_next;
 513 } cyc_omni_cpu_t;
 514
 515 typedef struct cyc_id {
 516         krwlock_t cyi_lock;
 517         cyc_cpu_t *cyi_cpu;
 518         cyc_index_t cyi_ndx;
 519         struct cyc_id *cyi_prev;
 520         struct cyc_id *cyi_next;
 521         cyc_omni_handler_t cyi_omni_hdlr;
 522         cyc_omni_cpu_t *cyi_omni_list;
 523 } cyc_id_t;
 524
 525 typedef struct cyc_xcallarg {
 526         cyc_cpu_t *cyx_cpu;
 527         cyc_handler_t *cyx_hdlr;
 528         cyc_time_t *cyx_when;
 529         cyc_index_t cyx_ndx;
 530         cyc_index_t *cyx_heap;
 531         cyclic_t *cyx_cyclics;
 532         cyc_index_t cyx_size;
 533         uint16_t cyx_flags;
 534         int cyx_wait;
 535 } cyc_xcallarg_t;
 536
 537 #define CY_DEFAULT_PERCPU       1
 538 #define CY_PASSIVE_LEVEL        -1
 539
 540 #define CY_WAIT                 0
 541 #define CY_NOWAIT               1
 542
 543 #define CYC_HEAP_PARENT(ndx)            (((ndx) - 1) >> 1)
 544 #define CYC_HEAP_RIGHT(ndx)             (((ndx) + 1) << 1)
 545 #define CYC_HEAP_LEFT(ndx)              ((((ndx) + 1) << 1) - 1)
 546
 547 #ifdef  __cplusplus
 548 }
 549 #endif
 550
 551 #endif /* _SYS_CYCLIC_IMPL_H */