include/sys/ib/adapters/hermon/hermon_fm.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25
  26 #ifndef _SYS_IB_ADAPTERS_HERMON_FM_H
  27 #define _SYS_IB_ADAPTERS_HERMON_FM_H
  28
  29 /*
  30  * hermon_fm.h
  31  */
  32 #include <sys/ddifm.h>
  33 #include <sys/fm/protocol.h>
  34 #include <sys/fm/util.h>
  35 #include <sys/fm/io/ddi.h>
  36
  37 #ifdef __cplusplus
  38 extern "C" {
  39 #endif
  40
  41 /*
  42  * HCA FMA compile note.
  43  *
  44  * FMA_TEST is used for HCA function tests, and
  45  * the macro can be on by changing Makefile.
  46  *
  47  * in case of DEBUG
  48  *      FMA_TEST is on
  49  *
  50  * in case of non-DEBUG (DEBUG is off)
  51  *      FMA_TEST is off
  52  */
  53
  54 /*
  55  * HCA FM common data structure
  56  */
  57
  58 /*
  59  * HCA FM Structure
  60  * This structure is used to catch HCA HW errors.
  61  */
  62 struct i_hca_fm {
  63         uint32_t ref_cnt;       /* the number of instances referring to this */
  64         kmutex_t lock;          /* protection for last_err & polling thread */
  65         struct i_hca_acc_handle *hdl;   /* HCA FM acc handle structure */
  66         struct kmem_cache *fm_acc_cache; /* HCA acc handle cache */
  67
  68 };
  69
  70 /*
  71  * HCA FM acc handle structure
  72  * This structure is holding ddi_acc_handle_t and other members
  73  * to deal with HCA PIO FM.
  74  */
  75 struct i_hca_acc_handle {
  76         struct i_hca_acc_handle *next;  /* next structure */
  77         ddi_acc_handle_t save_hdl;      /* acc handle */
  78         kmutex_t lock;                  /* mutex lock for thread count */
  79         uint32_t thread_cnt;            /* number of threads issuing PIOs */
  80 };
  81 _NOTE(SCHEME_PROTECTS_DATA("safe sharing", i_hca_acc_handle::save_hdl))
  82 #define fm_acc_hdl(hdl) (((struct i_hca_acc_handle *)(hdl))->save_hdl)
  83 #define FM_POLL_INTERVAL (10000000)     /* 10ms (nano) */
  84
  85 /*
  86  * HCA FM function test structure
  87  * This structure can be used to test the basic fm function test for HCA.
  88  * The test code is included if the FMA_TEST macro is defined.
  89  */
  90 struct i_hca_fm_test {
  91         int num;                /* serial numner */
  92         int type;               /* PIO or Hermon specific errors */
  93 #define HCA_TEST_PIO    0x1
  94 #define HCA_TEST_IBA    0x2
  95         int trigger;            /* how to trigger a HW error */
  96 #define HCA_TEST_TRANSIENT              0x0001
  97 #define HCA_TEST_PERSISTENT             0x0002
  98 #define HCA_TEST_ATTACH                 0x0010
  99 #define HCA_TEST_START                  0x0100
 100 #define HCA_TEST_END                    0x0200
 101         void (*pio_injection)(struct i_hca_fm_test *, ddi_fm_error_t *);
 102         int errcnt;             /* how many transient error occurs */
 103         int line_num;           /* line number in the source code */
 104         char *file_name;        /* source filename */
 105         char *hash_key;         /* hash table for test items */
 106         void *private;          /* private data */
 107 };
 108
 109 /*
 110  * Hermon FM data structure
 111  */
 112 typedef struct i_hca_fm hermon_hca_fm_t;
 113 typedef struct i_hca_acc_handle hermon_acc_handle_t;
 114 typedef struct i_hca_fm_test hermon_test_t;
 115
 116 /*
 117  * The following defines are to supplement device error reporting.
 118  * At each place where the planned FMA error matrix specifies that
 119  * an ereport will be generated, for now there is a HERMON_FMANOTE()
 120  * call generating an appropriate message string.
 121  *
 122  * This has been revised since it has been realized that FMA is only
 123  * to be used for hardware errors.  HERMON_FMANOTE() is used to report
 124  * errors that are likely to be hardware, but possibly are not.
 125  */
 126 #define HERMON_FMANOTE(state, string)                                   \
 127         cmn_err(CE_WARN, "hermon%d: Device Error: %s",                  \
 128                 (state)->hs_instance, string)
 129
 130 /* CQE Syndrome errors - see hermon_cq.c */
 131
 132 #define HERMON_FMA_LOCLEN       "CQE local length error"
 133 #define HERMON_FMA_LOCQPOP      "CQE local qp operation error"
 134 #define HERMON_FMA_LOCPROT      "CQE local protection error"
 135 #define HERMON_FMA_WQFLUSH      "CQE wqe flushed in error"
 136 #define HERMON_FMA_MWBIND       "CQE memory window bind error"
 137 #define HERMON_FMA_RESP         "CQE bad response"
 138 #define HERMON_FMA_LOCACC       "CQE local access error"
 139 #define HERMON_FMA_REMREQ       "CQE remote invalid request error"
 140 #define HERMON_FMA_REMACC       "CQE remote access error"
 141 #define HERMON_FMA_REMOP        "CQE remote operation error"
 142 #define HERMON_FMA_XPORTCNT     "CQE transport retry counter exceeded"
 143 #define HERMON_FMA_RNRCNT       "CQE RNR retry counter exceeded"
 144 #define HERMON_FMA_REMABRT      "CQE remote aborted error"
 145 #define HERMON_FMA_UNKN         "CQE unknown/reserved error returned"
 146
 147 /* event errors - see hermon_event.c */
 148 #define HERMON_FMA_OVERRUN      "EQE cq overrun or protection error"
 149 #define HERMON_FMA_LOCCAT       "EQE local work queue catastrophic error"
 150 #define HERMON_FMA_QPCAT        "EQE local queue pair catastrophic error"
 151 #define HERMON_FMA_PATHMIG      "EQE path migration failed"
 152 #define HERMON_FMA_LOCINV       "EQE invalid request - local work queue"
 153 #define HERMON_FMA_LOCACEQ      "EQE local access violation"
 154 #define HERMON_FMA_SRQCAT       "EQE shared received queue catastrophic"
 155 #define HERMON_FMA_INTERNAL     "EQE hca internal error"
 156
 157 /* HCR device failure returns - see hermon_cmd.c */
 158 #define HERMON_FMA_HCRINT       "HCR internal error processing command"
 159 #define HERMON_FMA_NVMEM        "HCR NVRAM checksum/CRC failure"
 160 #define HERMON_FMA_TOTOG        "HCR Timeout waiting for command toggle"
 161 #define HERMON_FMA_GOBIT        "HCR Timeout waiting for command go bit"
 162 #define HERMON_FMA_RSRC         "HCR Command insufficient resources"
 163 #define HERMON_FMA_CMDINV       "HCR Command invalid status returned"
 164
 165 /* HCA initialization errors - see hermon.c */
 166 #define HERMON_FMA_FWVER        "HCA firmware not at minimum version"
 167 #define HERMON_FMA_PCIID        "HCA PCIe devid not supported"
 168 #define HERMON_FMA_MAINT        "HCA device set to memory controller mode"
 169 #define HERMON_FMA_BADNVMEM     "HCR bad NVMEM error"
 170
 171 /*
 172  * HCA FM constants
 173  */
 174
 175 /* HCA FM state */
 176 #define HCA_NO_FM               0x0000  /* HCA FM is not supported */
 177 /* HCA FM state flags */
 178 #define HCA_PIO_FM              0x0001  /* PIO is fma-protected */
 179 #define HCA_DMA_FM              0x0002  /* DMA is fma-protected */
 180 #define HCA_EREPORT_FM          0x0004  /* FMA ereport is available */
 181 #define HCA_ERRCB_FM            0x0010  /* FMA error callback is supported */
 182
 183 #define HCA_ATTCH_FM            0x0100  /* HCA FM attach mode */
 184 #define HCA_RUNTM_FM            0x0200  /* HCA FM runtime mode */
 185
 186 /* HCA ererport type */
 187 #define HCA_SYS_ERR             0x001   /* HW error reported by Solaris FMA */
 188 #define HCA_IBA_ERR             0x002   /* IB specific HW error */
 189
 190 /* HCA ereport detail */
 191 #define HCA_ERR_TRANSIENT       0x010   /* HCA temporary error */
 192 #define HCA_ERR_NON_FATAL       0x020   /* HCA persistent error */
 193 #define HCA_ERR_SRV_LOST        0x040   /* HCA attach failure */
 194 #define HCA_ERR_DEGRADED        0x080   /* HCA maintenance mode */
 195 #define HCA_ERR_FATAL           0x100   /* HCA critical situation */
 196 #define HCA_ERR_IOCTL           0x200   /* EIO */
 197
 198 /* Ignore HCA HW error check */
 199 #define HCA_SKIP_HW_CHK         (-1)
 200
 201 /* HCA FM pio retry operation state */
 202 #define HCA_PIO_OK              (0)     /* No HW errors */
 203 #define HCA_PIO_TRANSIENT       (1)     /* transient error */
 204 #define HCA_PIO_PERSISTENT      (2)     /* persistent error */
 205 #define HCA_PIO_RETRY_CNT       (3)
 206
 207 /* HCA firmware faults */
 208 #define HCA_FW_MISC             0x1     /* firmware misc faults */
 209 #define HCA_FW_CORRUPT          0x2     /* firmware corruption */
 210 #define HCA_FW_MISMATCH         0x3     /* firmware version mismatch */
 211
 212 /*
 213  * Hermon FM macros
 214  */
 215
 216 #ifdef FMA_TEST
 217 #define TEST_DECLARE(tst)               hermon_test_t *tst;
 218 #define REGISTER_PIO_TEST(st, tst)                                      \
 219     tst = hermon_test_register(st, __FILE__, __LINE__, HCA_TEST_PIO)
 220 #define PIO_START(st, hdl, tst)         hermon_PIO_start(st, hdl, tst)
 221 #define PIO_END(st, hdl, cnt, tst)      hermon_PIO_end(st, hdl, &cnt, tst)
 222 #else
 223 #define TEST_DECLARE(tst)
 224 #define REGISTER_PIO_TEST(st, tst)
 225 #define PIO_START(st, hdl, tst)         hermon_PIO_start(st, hdl, NULL)
 226 #define PIO_END(st, hdl, cnt, tst)      hermon_PIO_end(st, hdl, &cnt, NULL)
 227 #endif /* FMA_TEST */
 228
 229 /*
 230  * hermon_pio_init() is a macro initializing variables.
 231  */
 232 #define hermon_pio_init(cnt, status, tst)                               \
 233         TEST_DECLARE(tst)                                               \
 234         int     status = HCA_PIO_OK;                                    \
 235         int     cnt = HCA_PIO_RETRY_CNT
 236
 237 /*
 238  * hermon_pio_start() is one of a pair of macros checking HW errors
 239  * at PIO requests, which should be called before the requests are issued.
 240  */
 241 #define hermon_pio_start(st, hdl, label, cnt, status, tst)              \
 242         if (st->hs_fm_state & HCA_PIO_FM) {                             \
 243                 if (st->hs_fm_async_fatal) {                            \
 244                         hermon_fm_ereport(st, HCA_SYS_ERR,              \
 245                             HCA_ERR_NON_FATAL);                         \
 246                         goto label;                                     \
 247                 } else {                                                \
 248                         REGISTER_PIO_TEST(st, tst);                     \
 249                         cnt = HCA_PIO_RETRY_CNT;                        \
 250                         if (PIO_START(st, hdl, tst) ==                  \
 251                             HCA_PIO_PERSISTENT) {                       \
 252                                 goto label;                             \
 253                         }                                               \
 254                 }                                                       \
 255         } else {                                                        \
 256                 status = HCA_SKIP_HW_CHK;                               \
 257         }                                                               \
 258         do {
 259
 260 /*
 261  * hermon_pio_end() is the other of a pair of macros checking HW errors
 262  * at PIO requests, which should be called after the requests end.
 263  * If a HW error is detected and can be isolated well, these macros
 264  * retry the operation to determine if the error is persistent or not.
 265  */
 266 #define hermon_pio_end(st, hdl, label, cnt, status, tst)                \
 267         if (status != HCA_SKIP_HW_CHK) {                                \
 268                 if (st->hs_fm_async_fatal) {                            \
 269                         hermon_fm_ereport(st, HCA_SYS_ERR,              \
 270                             HCA_ERR_NON_FATAL);                         \
 271                         goto label;                                     \
 272                 }                                                       \
 273                 if ((status = PIO_END(st, hdl, cnt, tst)) ==            \
 274                     HCA_PIO_PERSISTENT) {                               \
 275                         goto label;                                     \
 276                 } else if (status == HCA_PIO_TRANSIENT) {               \
 277                         hermon_fm_ereport(st, HCA_SYS_ERR,              \
 278                             HCA_ERR_TRANSIENT);                         \
 279                 }                                                       \
 280         }                                                               \
 281         } while (status == HCA_PIO_TRANSIENT)
 282
 283 extern void hermon_fm_init(hermon_state_t *);
 284 extern void hermon_fm_fini(hermon_state_t *);
 285 extern int hermon_fm_ereport_init(hermon_state_t *);
 286 extern void hermon_fm_ereport_fini(hermon_state_t *);
 287 extern int hermon_get_state(hermon_state_t *);
 288 extern boolean_t hermon_init_failure(hermon_state_t *);
 289 extern boolean_t hermon_cmd_retry_ok(hermon_cmd_post_t *, int);
 290 extern void hermon_fm_ereport(hermon_state_t *, int, int);
 291 extern int hermon_regs_map_setup(hermon_state_t *, uint_t, caddr_t *, offset_t,
 292     offset_t, ddi_device_acc_attr_t *, ddi_acc_handle_t *);
 293 extern void hermon_regs_map_free(hermon_state_t *, ddi_acc_handle_t *);
 294 extern int hermon_pci_config_setup(hermon_state_t *, ddi_acc_handle_t *);
 295 extern void hermon_pci_config_teardown(hermon_state_t *, ddi_acc_handle_t *);
 296 extern ushort_t hermon_devacc_attr_version(hermon_state_t *);
 297 extern uchar_t hermon_devacc_attr_access(hermon_state_t *);
 298 extern int hermon_PIO_start(hermon_state_t *, ddi_acc_handle_t,
 299     hermon_test_t *);
 300 extern int hermon_PIO_end(hermon_state_t *, ddi_acc_handle_t, int *,
 301     hermon_test_t *);
 302 extern ddi_acc_handle_t hermon_rsrc_alloc_uarhdl(hermon_state_t *);
 303 extern ddi_acc_handle_t hermon_get_uarhdl(hermon_state_t *);
 304 extern ddi_acc_handle_t hermon_get_cmdhdl(hermon_state_t *);
 305 extern ddi_acc_handle_t hermon_get_msix_tblhdl(hermon_state_t *);
 306 extern ddi_acc_handle_t hermon_get_msix_pbahdl(hermon_state_t *);
 307 extern ddi_acc_handle_t hermon_get_pcihdl(hermon_state_t *);
 308 extern void hermon_clr_state_nolock(hermon_state_t *, int);
 309 extern void hermon_inter_err_chk(void *);
 310
 311 #ifdef FMA_TEST
 312 extern hermon_test_t *hermon_test_register(hermon_state_t *, char *, int, int);
 313 extern void hermon_test_deregister(void);
 314 extern int hermon_test_num;
 315 #endif /* FMA_TEST */
 316
 317 #ifdef __cplusplus
 318 }
 319 #endif
 320
 321 #endif  /* _SYS_IB_ADAPTERS_HERMON_FM_H */