include/sys/rsm/rsm_in.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License, Version 1.0 only
   6  * (the "License").  You may not use this file except in compliance
   7  * with the License.
   8  *
   9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10  * or http://www.opensolaris.org/os/licensing.
  11  * See the License for the specific language governing permissions
  12  * and limitations under the License.
  13  *
  14  * When distributing Covered Code, include this CDDL HEADER in each
  15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16  * If applicable, add the following below this CDDL HEADER, with the
  17  * fields enclosed by brackets "[]" replaced with your own identifying
  18  * information: Portions Copyright [yyyy] [name of copyright owner]
  19  *
  20  * CDDL HEADER END
  21  */
  22 /*
  23  * Copyright 1999-2003 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26
  27 #ifndef _RSM_IN_H
  28 #define _RSM_IN_H
  29
  30 #pragma ident   "%Z%%M% %I%     %E% SMI"
  31
  32 #ifdef  __cplusplus
  33 extern "C" {
  34 #endif
  35
  36 #include <sys/ddi.h>
  37 #include <sys/sunddi.h>
  38 #include <sys/rsm/rsm.h>
  39 #include <sys/rsm/rsmpi.h>
  40
  41 #define DRIVER_NAME     "rsm"
  42
  43
  44 #define RSM_DRIVER_MINOR        0
  45
  46 #define RSM_CNUM        8
  47
  48 #define RSMIPC_SZ       10      /* number of outstanding requests, max: 256 */
  49
  50 #define RSMIPC_MAX_MESSAGES     64 /* max msgs that receiver can buffer */
  51 #define RSMIPC_LOTSFREE_MSGBUFS 16 /* chunks of credits sent to sender  */
  52
  53 /*
  54  * The base for Sun RSMAPI Kernel Agent service idenitifiers is RSM_INTR_T_KA
  55  * as defined below. This is as per the RSMPI specification. Thus,
  56  * in the kernel agent, we need to use this value as the service identifier
  57  * while registering the service handlers.
  58  */
  59 #define RSM_INTR_T_KA   0x88
  60 #define RSM_SERVICE     RSM_INTR_T_KA
  61
  62 #define RSM_PRI         2
  63 #define RSM_QUEUE_SZ    256
  64
  65 #define RSM_LOCK        0
  66 #define RSM_NOLOCK      1
  67
  68 #define RSM_MAX_NUM_SEG 4095    /* default value for max imp and exp segs */
  69
  70 #define RSM_MAX_NODE    64      /* maximum number of nodes in the cluster */
  71
  72 #define RSM_MAX_CTRL    32      /* maximum number of controllers per node */
  73
  74 /*
  75  * The following defines UINT_MAX rounded down to a page aligned value.
  76  */
  77 #define RSM_MAXSZ_PAGE_ALIGNED (UINT_MAX & PAGEMASK)
  78 /*
  79  * Define TRASHSIZE as the maximum possible size which is page aligned
  80  * This value cannot be 0xffffffffffffe000 since this is taken as a
  81  * negative value in the devmap_umem_remap call, thus causing the call
  82  * to fail.
  83  */
  84 #define TRASHSIZE 0x7fffffffffffe000
  85
  86 #define RSM_ACCESS_READ                         0444
  87 #define RSM_ACCESS_WRITE                        0222
  88 #define RSM_ACCESS_TRUSTED                      0666
  89
  90 /* flag values for rsmseg_unload */
  91 #define DISCONNECT      1
  92 #define NO_DISCONNECT   0
  93
  94 struct rsm_driver_data {
  95         kmutex_t        drv_lock;
  96         kcondvar_t      drv_cv;
  97         int             drv_state;      /* RSM_DRV_YYYY states */
  98         int             drv_memdel_cnt; /* number of memdel callbacks */
  99 };
 100
 101 /* rsm driver state */
 102 #define RSM_DRV_NEW                     0
 103 #define RSM_DRV_OK                      1
 104 #define RSM_DRV_PREDEL_STARTED          2
 105 #define RSM_DRV_PREDEL_COMPLETED        3
 106 #define RSM_DRV_POSTDEL_IN_PROGRESS     4
 107 #define RSM_DRV_DR_IN_PROGRESS          5
 108 #define RSM_DRV_REG_PROCESSING          6
 109 #define RSM_DRV_UNREG_PROCESSING        7
 110
 111 /* internal flags */
 112 #define RSM_DR_QUIESCE          0
 113 #define RSM_DR_UNQUIESCE        1
 114
 115 typedef enum {
 116         RSM_STATE_NEW = 0,
 117         RSM_STATE_NEW_QUIESCED,
 118         RSM_STATE_BIND,
 119         RSM_STATE_BIND_QUIESCED,
 120         RSM_STATE_EXPORT,
 121         RSM_STATE_EXPORT_QUIESCING,
 122         RSM_STATE_EXPORT_QUIESCED,
 123         RSM_STATE_ZOMBIE,
 124         RSM_STATE_CONNECTING,
 125         RSM_STATE_ABORT_CONNECT,
 126         RSM_STATE_CONNECT,
 127         RSM_STATE_CONN_QUIESCE,
 128         RSM_STATE_MAPPING,
 129         RSM_STATE_ACTIVE,
 130         RSM_STATE_MAP_QUIESCE,
 131         RSM_STATE_DISCONNECT,
 132         RSM_STATE_END
 133 } rsm_resource_state_t;
 134
 135 typedef enum {
 136         RSM_RESOURCE_EXPORT_SEGMENT,
 137         RSM_RESOURCE_IMPORT_SEGMENT,
 138         RSM_RESOURCE_BAR
 139 }rsm_resource_type_t;
 140
 141 /*
 142  * All resources have the only common info. whether it is a segment or
 143  * a notification queue.
 144  */
 145 typedef struct rsm_resource {
 146         kmutex_t                rsmrc_lock;     /* sync on resource */
 147         minor_t                 rsmrc_num;      /* (minor) number */
 148         rsm_memseg_id_t         rsmrc_key;      /* user key */
 149         mode_t                  rsmrc_mode;     /* access permission */
 150         struct adapter          *rsmrc_adapter; /* controller number */
 151         rsm_node_id_t           rsmrc_node;     /*  nodeid */
 152         rsm_resource_type_t     rsmrc_type;     /* type of this resource */
 153         rsm_resource_state_t    rsmrc_state;    /* segment state */
 154         struct rsm_resource     *rsmrc_next;
 155 } rsmresource_t;
 156
 157 #define RSMRC_BLKSZ     16
 158 #define RSMRC_RESERVED  ((rsmresource_t *)0x1)
 159
 160 #define RSM_HASHSZ      128
 161
 162 #define RSM_USER_MEMORY         0x1
 163 #define RSM_KERNEL_MEMORY       0x2
 164 #define RSM_EXPORT_WAIT         0x4
 165 #define RSM_SEGMENT_POLL        0x8
 166 #define RSM_FORCE_DISCONNECT    0x10
 167 #define RSM_IMPORT_DUMMY        0x20
 168 /*
 169  * The following macro is used within the kernel agent to indicate that
 170  * rebind/unbind is allowed for an exported segment. It is a part of the
 171  * segment's s_flags field.
 172  */
 173 #define RSMKA_ALLOW_UNBIND_REBIND       0x40
 174 #define RSM_REPUBLISH_WAIT      0x80
 175 #define RSM_DR_INPROGRESS       0x100
 176 #define RSM_FORCE_DESTROY_WAIT  0x200
 177 #define RSMKA_SET_RESOURCE_DONTWAIT     0x400
 178
 179 #define RSMRC_LOCK(p)   mutex_enter(&(p)->rsmrc_lock)
 180 #define RSMRC_UNLOCK(p) mutex_exit(&(p)->rsmrc_lock)
 181 #define RSMRC_HELD(p)   MUTEX_HELD(&(p)->rsmrc_lock)
 182 #define RSMRC_TRY(p)    mutex_tryenter(&(p)->rsmrc_lock)
 183
 184 typedef struct rsm_region {
 185         caddr_t         r_vaddr;        /* exported virtual address */
 186         size_t          r_len;          /* length of export region */
 187         offset_t        r_off;          /* offset of this region in segment */
 188         struct as       *r_asp;
 189         struct rsm_region *r_next;      /* next region of segment */
 190 }rsm_region;
 191
 192 typedef struct rsm_cookie {
 193         devmap_cookie_t         c_dhp;          /* devmap cookie handle */
 194         offset_t                c_off;          /* offset of mapping    */
 195         size_t                  c_len;          /* len of mapping       */
 196         struct rsm_cookie       *c_next;        /* next handle          */
 197 }rsmcookie_t;
 198
 199 typedef struct rsm_mapinfo {
 200         dev_info_t      *dip;
 201         uint_t          dev_register;
 202         off_t           dev_offset;
 203         off_t           start_offset;
 204         size_t          individual_len;
 205         struct rsm_mapinfo *next;
 206 } rsm_mapinfo_t;
 207
 208
 209
 210 /*
 211  * Shared Importer data structure
 212  *
 213  */
 214 typedef struct rsm_import_share {
 215         kmutex_t        rsmsi_lock;     /* lock for shared importers    */
 216         kcondvar_t      rsmsi_cv;       /* condvar to wait at           */
 217         rsm_node_id_t   rsmsi_node;
 218         rsm_memseg_id_t rsmsi_segid;
 219         size_t          rsmsi_seglen;
 220         rsm_memseg_import_handle_t      rsmsi_handle; /* RSMPI handle */
 221         uint_t          rsmsi_state;
 222 #define RSMSI_STATE_NEW                 0x0001
 223 #define RSMSI_STATE_CONNECTING          0x0002
 224 #define RSMSI_STATE_ABORT_CONNECT       0x0004
 225 #define RSMSI_STATE_CONNECTED           0x0008
 226 #define RSMSI_STATE_CONN_QUIESCE        0x0010
 227 #define RSMSI_STATE_MAPPED              0x0020
 228 #define RSMSI_STATE_MAP_QUIESCE         0x0040
 229 #define RSMSI_STATE_DISCONNECTED        0x0080
 230
 231         uint_t          rsmsi_refcnt;   /* ref count of importers       */
 232         uint_t          rsmsi_mapcnt;   /* count of mapped importers    */
 233         mode_t          rsmsi_mode;     /* mode of last (re)publish     */
 234         uid_t           rsmsi_uid;
 235         gid_t           rsmsi_gid;
 236         rsm_mapinfo_t   *rsmsi_mapinfo; /* register, offset, len values */
 237         uint_t          rsmsi_flags;    /* flags                        */
 238 #define RSMSI_FLAGS_ABORTDONE   0x0001  /* NOT_IMPORTING msg for abort conn */
 239                                         /* has been sent                    */
 240         void            *rsmsi_cookie;  /* cookie of the first seg connect */
 241 } rsm_import_share_t;
 242
 243 #define RSMSI_LOCK(sharep)      mutex_enter(&(sharep)->rsmsi_lock)
 244 #define RSMSI_UNLOCK(sharep)    mutex_exit(&(sharep)->rsmsi_lock)
 245 #define RSMSI_HELD(sharep)      MUTEX_HELD(&(sharep)->rsmsi_lock)
 246 #define RSMSI_TRY(sharep)       mutex_tryenter(&(sharep)->rsmsi_lock)
 247
 248 typedef struct rsm_seginfo {
 249         rsmresource_t           s_hdr;          /* resource hdr */
 250 #define s_state s_hdr.rsmrc_state       /* segment state */
 251 #define s_adapter s_hdr.rsmrc_adapter
 252 #define s_node  s_hdr.rsmrc_node
 253 #define s_lock  s_hdr.rsmrc_lock
 254 #define s_minor s_hdr.rsmrc_num         /* minor # of segment */
 255 #define s_key   s_hdr.rsmrc_key         /* user segment key */
 256 #define s_mode  s_hdr.rsmrc_mode        /* user segment mode */
 257 #define s_type  s_hdr.rsmrc_type        /* segment type */
 258         uid_t                   s_uid;          /* owner id */
 259         gid_t                   s_gid;          /* owner id */
 260
 261         size_t                  s_len;          /* total segment size */
 262         rsm_region              s_region;       /* regions of segment */
 263
 264         int                     s_flags;
 265         int                     s_pollflag;     /* indicates poll status */
 266
 267         kcondvar_t              s_cv;           /* condition to wait on */
 268
 269         rsm_memseg_id_t         s_segid;        /* NIC segment id */
 270
 271         int             s_acl_len;              /* length of access list */
 272         rsmapi_access_entry_t *s_acl;           /* access list */
 273         rsm_access_entry_t *s_acl_in;           /* access list with hwaddr */
 274
 275         struct pollhead s_poll;
 276         uint32_t        s_pollevent;
 277         pid_t           s_pid;
 278
 279         rsmcookie_t     *s_ckl;         /* list of devmap cookie */
 280
 281         size_t          s_total_maplen;
 282         rsm_mapinfo_t   *s_mapinfo;     /* register, offset, len  */
 283
 284         union {
 285                 rsm_memseg_import_handle_t      in;
 286                 rsm_memseg_export_handle_t      out;
 287         } s_handle;                     /* NIC handle for segment */
 288
 289         /*
 290          * This field is used to indicate the cookie returned by the
 291          * ddi_umem_lock when binding pages for an export segment.
 292          * Also, for importers on the same node as the export segment,
 293          * this field indicates the cookie used during import mapping.
 294          */
 295         ddi_umem_cookie_t       s_cookie;
 296         rsm_import_share_t      *s_share;       /* shared importer data     */
 297         /*
 298          * This field in an import segments indicates the number of
 299          * putv/getv operations in progress and in an export segment
 300          * it is the number of putv/getv ops currently using it as
 301          * a handle in the iovec.
 302          */
 303         uint_t                  s_rdmacnt;
 304         struct proc             *s_proc;
 305 } rsmseg_t;
 306
 307 #define rsmseglock_acquire(p)   RSMRC_LOCK((rsmresource_t *)(p))
 308 #define rsmseglock_release(p)   RSMRC_UNLOCK((rsmresource_t *)(p))
 309 #define rsmseglock_held(p)      RSMRC_HELD((rsmresource_t *)(p))
 310 #define rsmseglock_try(p)       RSMRC_TRY((rsmresource_t *)(p))
 311
 312 #define rsmsharelock_acquire(p) RSMSI_LOCK(p->s_share)
 313 #define rsmsharelock_release(p) RSMSI_UNLOCK(p->s_share)
 314 #define rsmsharelock_held(p)    RSMSI_HELD(p->s_share)
 315 #define rsmsharelock_try(p)     RSMSI_TRY(p->s_share)
 316
 317 /*
 318  * Resource elements structure
 319  */
 320 typedef struct {
 321         int             rsmrcblk_avail;
 322         rsmresource_t   *rsmrcblk_blks[RSMRC_BLKSZ];
 323 }rsmresource_blk_t;
 324
 325 struct rsmresource_table {
 326         krwlock_t       rsmrc_lock;
 327         int             rsmrc_len;
 328         int             rsmrc_sz;
 329         rsmresource_blk_t **rsmrc_root;
 330 };
 331
 332 /*
 333  * Struct for advertised resource list
 334  */
 335 /*
 336  * Hashtable structs
 337  * bucket points to an array of pointers, each entry in the bucket array
 338  * points to a linked list of resource items.
 339  * bucket index = bucket_address%RSM_HASHSZ
 340  */
 341 typedef struct rsmhash_table {
 342         krwlock_t               rsmhash_rw;
 343         rsmresource_t           **bucket;
 344 } rsmhash_table_t;
 345
 346 /*
 347  * Remote messaging related structure
 348  */
 349
 350 /*
 351  * Flags for ipc slot
 352  */
 353 #define RSMIPC_FREE     0x1                     /* slot is free */
 354 #define RSMIPC_PENDING  0x2                     /* slot has pending request */
 355
 356 #define RSMIPC_SET(x, v)        ((x)->rsmipc_flags |= (v))
 357 #define RSMIPC_GET(x, v)        ((x)->rsmipc_flags & (v))
 358 #define RSMIPC_CLEAR(x, v)      ((x)->rsmipc_flags &= ~(v))
 359
 360 typedef struct rsmipc_slot {
 361         kmutex_t        rsmipc_lock;            /* lock for remote msgs */
 362         kcondvar_t      rsmipc_cv;              /* condition var to wait on */
 363         int             rsmipc_flags;
 364         rsmipc_cookie_t rsmipc_cookie;          /* cookie of request in wire */
 365         void            *rsmipc_data;           /* ptr to data to copy */
 366 }rsmipc_slot_t;
 367
 368 /*
 369  * Messaging struc
 370  */
 371 typedef struct {
 372         kmutex_t        lock;
 373         kcondvar_t      cv;
 374         int             count;
 375         int             wanted;
 376         int             sequence;
 377         rsmipc_slot_t   slots[RSMIPC_SZ];
 378 }rsm_ipc_t;
 379
 380 /*
 381  * These tokens are used for building the list of remote node importers
 382  * of a segment exported from the local node
 383  */
 384 typedef struct importing_token {
 385         struct importing_token  *next;
 386         rsm_memseg_id_t         key;
 387         rsm_node_id_t           importing_node;
 388         void                    *import_segment_cookie;
 389         rsm_addr_t              importing_adapter_hwaddr;
 390 } importing_token_t;
 391
 392 typedef struct {
 393         kmutex_t                lock;
 394         importing_token_t       **bucket;
 395 } importers_table_t;
 396
 397 /*
 398  * Used by the rsm_send_republish() fn
 399  */
 400 typedef struct republish_token {
 401         struct republish_token  *next;
 402         rsm_memseg_id_t         key;
 403         rsm_node_id_t           importing_node;
 404         rsm_permission_t        permission;
 405 } republish_token_t;
 406
 407 /*
 408  * data strucuture for list manipulation
 409  */
 410 typedef struct list_element {
 411         struct list_element     *next;
 412         rsm_node_id_t           nodeid;
 413         uint32_t                flags;
 414 #define RSM_SUSPEND_ACKPENDING  0x01
 415 #define RSM_SUSPEND_NODEDEAD    0x02
 416 } list_element_t;
 417
 418 typedef struct list_head {
 419         struct list_element     *list_head;
 420         kmutex_t                list_lock;
 421 } list_head_t;
 422
 423 #ifdef  __cplusplus
 424 }
 425 #endif
 426
 427 #endif  /* _RSM_IN_H */