arch/x86/kernel/platform/i86pc/include/sys/ioat.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26
  27 #ifndef _SYS_IOAT_H
  28 #define _SYS_IOAT_H
  29
  30 #ifdef __cplusplus
  31 extern "C" {
  32 #endif
  33
  34 #include <sys/types.h>
  35 #include <sys/dcopy.h>
  36 #include <sys/dcopy_device.h>
  37
  38
  39 /* ioat ioctls */
  40 #define IOATIOC                 ('T'<< 8)
  41 typedef enum {
  42         IOAT_IOCTL_WRITE_REG    = (IOATIOC | 0x0),
  43         IOAT_IOCTL_READ_REG     = (IOATIOC | 0x1),
  44         IOAT_IOCTL_TEST         = (IOATIOC | 0x2)
  45 } ioat_ioctl_enum_t;
  46
  47 typedef struct ioat_ioctl_reg_s {
  48         uint_t          size;
  49         uint_t          addr;
  50         uint64_t        data;
  51 } ioat_ioctl_reg_t;
  52 typedef ioat_ioctl_reg_t ioat_ioctl_wrreg_t;
  53 typedef ioat_ioctl_reg_t ioat_ioctl_rdreg_t;
  54
  55 #ifdef _KERNEL
  56 /* *** Driver Private Below *** */
  57
  58 /* IOAT_DMACAPABILITY flags */
  59 #define IOAT_DMACAP_PAGEBREAK   0x1
  60 #define IOAT_DMACAP_CRC         0x2
  61 #define IOAT_DMACAP_MARKERSKIP  0x4
  62 #define IOAT_DMACAP_XOR         0x8
  63 #define IOAT_DMACAP_DCA         0x10
  64
  65 /* IOAT_INTRCTL bits */
  66 #define IOAT_INTRCTL_MASTER_EN  0x1
  67 #define IOAT_INTRCTL_INTR_STAT  0x2
  68
  69 /* MMIO Registers */
  70 #define IOAT_CHANCNT            0x0     /* 8-bit */
  71 #define IOAT_XFERCAP            0x1     /* 8-bit */
  72 #define IOAT_GENCTRL            0x2     /* 8-bit */
  73 #define IOAT_INTRCTL            0x3     /* 8-bit */
  74 #define IOAT_ATTNSTATUS         0x4     /* 32-bit */
  75 #define IOAT_CBVER              0x8     /* 8-bit */
  76 #define IOAT_PERPORT_OFF        0xA     /* 16-bit */
  77 #define IOAT_INTRDELAY          0xC     /* 16-bit */
  78 #define IOAT_CSSTATUS           0xE     /* 16-bit */
  79 #define IOAT_DMACAPABILITY      0x10    /* 32-bit */
  80
  81 #define IOAT_CHANNELREG_OFFSET  0x80
  82
  83 /* Channel Registers */
  84 #define IOAT_CHAN_CTL           0x0     /* 16-bit */
  85 #define IOAT_CHAN_COMP          0x2     /* 16-bit */
  86 #define IOAT_CHAN_CMPL_LO       0x18    /* 32-bit */
  87 #define IOAT_CHAN_CMPL_HI       0x1C    /* 32-bit */
  88 #define IOAT_CHAN_ERR           0x28    /* 32-bit */
  89 #define IOAT_CHAN_ERRMASK       0x2C    /* 32-bit */
  90 #define IOAT_CHAN_DCACTRL       0x30    /* 32-bit */
  91
  92 #define IOAT_V1_CHAN_STS_LO     0x4     /* 32-bit */
  93 #define IOAT_V1_CHAN_STS_HI     0x8     /* 32-bit */
  94 #define IOAT_V1_CHAN_ADDR_LO    0x0C    /* 32-bit */
  95 #define IOAT_V1_CHAN_ADDR_HI    0x10    /* 32-bit */
  96 #define IOAT_V1_CHAN_CMD        0x14    /* 8-bit */
  97
  98 #define IOAT_V2_CHAN_CMD        0x4     /* 8-bit */
  99 #define IOAT_V2_CHAN_CNT        0x6     /* 16-bit */
 100 #define IOAT_V2_CHAN_STS_LO     0x8     /* 32-bit */
 101 #define IOAT_V2_CHAN_STS_HI     0xC     /* 32-bit */
 102 #define IOAT_V2_CHAN_ADDR_LO    0x10    /* 32-bit */
 103 #define IOAT_V2_CHAN_ADDR_HI    0x14    /* 32-bit */
 104
 105 #define IOAT_CHAN_STS_ADDR_MASK         0xFFFFFFFFFFFFFFC0
 106 #define IOAT_CHAN_STS_XFER_MASK         0x3F
 107 #define IOAT_CHAN_STS_FAIL_MASK         0x6
 108 #define IOAT_CMPL_INDEX(channel)        \
 109         (((*channel->ic_cmpl & IOAT_CHAN_STS_ADDR_MASK) - \
 110         ring->cr_phys_desc) >> 6)
 111 #define IOAT_CMPL_FAILED(channel)       \
 112         (*channel->ic_cmpl & IOAT_CHAN_STS_FAIL_MASK)
 113
 114
 115 typedef struct ioat_chan_desc_s {
 116         uint32_t        dd_res0;
 117         uint32_t        dd_ctrl;
 118         uint64_t        dd_res1;
 119         uint64_t        dd_res2;
 120         uint64_t        dd_next_desc;
 121         uint64_t        dd_res4;
 122         uint64_t        dd_res5;
 123         uint64_t        dd_res6;
 124         uint64_t        dd_res7;
 125 } ioat_chan_desc_t;
 126
 127 /* dca dd_ctrl bits */
 128 #define IOAT_DESC_CTRL_OP_CNTX  ((uint32_t)0xFF << 24)
 129 #define IOAT_DESC_CTRL_CNTX_CHNG        0x1
 130 typedef struct ioat_chan_dca_desc_s {
 131         uint32_t        dd_cntx;
 132         uint32_t        dd_ctrl;
 133         uint64_t        dd_res1;
 134         uint64_t        dd_res2;
 135         uint64_t        dd_next_desc;
 136         uint64_t        dd_res4;
 137         uint64_t        dd_res5;
 138         uint64_t        dd_res6;
 139         uint64_t        dd_res7;
 140 } ioat_chan_dca_desc_t;
 141
 142 /* dma dd_ctrl bits */
 143 #define IOAT_DESC_CTRL_OP_DMA   (0x0 << 24)
 144 #define IOAT_DESC_DMACTRL_NULL  0x20
 145 #define IOAT_DESC_CTRL_FENCE    0x10
 146 #define IOAT_DESC_CTRL_CMPL     0x8
 147 #define IOAT_DESC_CTRL_NODSTSNP 0x4
 148 #define IOAT_DESC_CTRL_NOSRCSNP 0x2
 149 #define IOAT_DESC_CTRL_INTR     0x1
 150 typedef struct ioat_chan_dma_desc_s {
 151         uint32_t        dd_size;
 152         uint32_t        dd_ctrl;
 153         uint64_t        dd_src_paddr;
 154         uint64_t        dd_dest_paddr;
 155         uint64_t        dd_next_desc;
 156         uint64_t        dd_next_src_paddr;      /* v2 only */
 157         uint64_t        dd_next_dest_paddr;     /* v2 only */
 158         uint64_t        dd_res6;
 159         uint64_t        dd_res7;
 160 } ioat_chan_dma_desc_t;
 161
 162
 163 typedef enum {
 164         IOAT_CBv1,
 165         IOAT_CBv2
 166 } ioat_version_t;
 167
 168 /* ioat private data per command */
 169 typedef struct ioat_cmd_private_s {
 170         uint64_t        ip_generation;
 171         uint64_t        ip_index;
 172         uint64_t        ip_start;
 173         dcopy_cmd_t     ip_next;
 174 } ioat_cmd_private_t;
 175
 176 /* descriptor ring state */
 177 typedef struct ioat_channel_ring_s {
 178         /* protects cr_cmpl_gen & cr_cmpl_last */
 179         kmutex_t                cr_cmpl_mutex;
 180
 181         /* desc ring generation for the last completion we saw */
 182         uint64_t                cr_cmpl_gen;
 183
 184         /* last descriptor index we saw complete */
 185         uint64_t                cr_cmpl_last;
 186
 187         /* protects cr_desc_* */
 188         kmutex_t                cr_desc_mutex;
 189
 190         /*
 191          * last descriptor posted. used to update its next pointer when we
 192          * add a new desc. Also used to tack the completion (See comment for
 193          * cr_desc_gen_prev).
 194          */
 195         uint64_t                cr_desc_prev;
 196
 197         /* where to put the next descriptor */
 198         uint64_t                cr_desc_next;
 199
 200         /* what the current desc ring generation is */
 201         uint64_t                cr_desc_gen;
 202
 203         /*
 204          * used during cmd_post to track the last desc posted. cr_desc_next
 205          * and cr_desc_gen will be pointing to the next free desc after
 206          * writing the descriptor to the ring. But we want to track the
 207          * completion for the last descriptor posted.
 208          */
 209         uint64_t                cr_desc_gen_prev;
 210
 211         /* the last desc in the ring (for wrap) */
 212         uint64_t                cr_desc_last;
 213
 214         /* pointer to the head of the ring */
 215         ioat_chan_desc_t        *cr_desc;
 216
 217         /* physical address of the head of the ring */
 218         uint64_t                cr_phys_desc;
 219
 220         /* back pointer to the channel state */
 221         struct ioat_channel_s   *cr_chan;
 222
 223         /* for CB v2, number of desc posted (written to IOAT_V2_CHAN_CNT) */
 224         uint_t                  cr_post_cnt;
 225 } ioat_channel_ring_t;
 226
 227 /* track channel state so we can handle a failure */
 228 typedef enum {
 229         IOAT_CHANNEL_OK = 0,
 230         IOAT_CHANNEL_IN_FAILURE = 1
 231 } ic_channel_state_t;
 232
 233 typedef struct ioat_channel_s *ioat_channel_t;
 234 struct ioat_channel_s {
 235         /* channel's ring state */
 236         ioat_channel_ring_t     *ic_ring;
 237
 238         /* IOAT_CBv1 || IOAT_CBv2 */
 239         ioat_version_t          ic_ver;
 240
 241         /*
 242          * state to determine if it's OK to post the the channel and if all
 243          * future polls should return failure.
 244          */
 245         ic_channel_state_t      ic_channel_state;
 246
 247         /* channel command cache (*_cmd_alloc, *_cmd_free, etc) */
 248         kmem_cache_t            *ic_cmd_cache;
 249
 250         /* dcopy state for dcopy_device_channel_notify() call */
 251         dcopy_handle_t          ic_dcopy_handle;
 252
 253         /* location in memory where completions are DMA'ed into */
 254         volatile uint64_t       *ic_cmpl;
 255
 256         /* channel specific registers */
 257         uint8_t                 *ic_regs;
 258
 259         /* if this channel is using DCA */
 260         boolean_t               ic_dca_active;
 261
 262         /* DCA ID the channel is currently pointing to */
 263         uint32_t                ic_dca_current;
 264
 265         /* devices channel number */
 266         uint_t                  ic_chan_num;
 267
 268         /* number of descriptors in ring */
 269         uint_t                  ic_chan_desc_cnt;
 270
 271         /* descriptor ring alloc state */
 272         ddi_dma_handle_t        ic_desc_dma_handle;
 273         size_t                  ic_desc_alloc_size;
 274         ddi_acc_handle_t        ic_desc_handle;
 275         ddi_dma_cookie_t        ic_desc_cookies;
 276
 277         /* completion buffer alloc state */
 278         ddi_dma_handle_t        ic_cmpl_dma_handle;
 279         size_t                  ic_cmpl_alloc_size;
 280         ddi_acc_handle_t        ic_cmpl_handle;
 281         ddi_dma_cookie_t        ic_cmpl_cookie;
 282         uint64_t                ic_phys_cmpl;
 283
 284         /* if inuse, we need to re-init the channel during resume */
 285         boolean_t               ic_inuse;
 286
 287         /* backpointer to driver state */
 288         struct ioat_state_s     *ic_state;
 289 };
 290
 291 typedef struct ioat_rs_s *ioat_rs_hdl_t;
 292
 293 /* driver state */
 294 typedef struct ioat_state_s {
 295         dev_info_t              *is_dip;
 296         int                     is_instance;
 297
 298         kmutex_t                is_mutex;
 299
 300         /* register handle and pointer to registers */
 301         ddi_acc_handle_t        is_reg_handle;
 302         uint8_t                 *is_genregs;
 303
 304         /* IOAT_CBv1 || IOAT_CBv2 */
 305         ioat_version_t          is_ver;
 306
 307         /* channel state */
 308         ioat_channel_t          is_channel;
 309         size_t                  is_chansize;
 310         ioat_rs_hdl_t           is_channel_rs;
 311
 312         ddi_iblock_cookie_t     is_iblock_cookie;
 313
 314         /* device info */
 315         uint_t                  is_chanoff;
 316         uint_t                  is_num_channels;
 317         uint_t                  is_maxxfer;
 318         uint_t                  is_cbver;
 319         uint_t                  is_intrdelay;
 320         uint_t                  is_status;
 321         uint_t                  is_capabilities;
 322
 323         /* dcopy_device_register()/dcopy_device_unregister() state */
 324         dcopy_device_handle_t   is_device_handle;
 325         dcopy_device_info_t     is_deviceinfo;
 326 } ioat_state_t;
 327
 328
 329 int ioat_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred,
 330     int *rval);
 331
 332 void ioat_rs_init(ioat_state_t *state, uint_t min_val, uint_t max_val,
 333     ioat_rs_hdl_t *handle);
 334 void ioat_rs_fini(ioat_rs_hdl_t *handle);
 335 int ioat_rs_alloc(ioat_rs_hdl_t handle, uint_t *rs);
 336 void ioat_rs_free(ioat_rs_hdl_t handle, uint_t rs);
 337
 338 int ioat_channel_init(ioat_state_t *state);
 339 void ioat_channel_fini(ioat_state_t *state);
 340 void ioat_channel_suspend(ioat_state_t *state);
 341 int ioat_channel_resume(ioat_state_t *state);
 342 void ioat_channel_quiesce(ioat_state_t *);
 343
 344 int ioat_channel_alloc(void *device_private, dcopy_handle_t handle, int flags,
 345     uint_t size, dcopy_query_channel_t *info, void *channel_private);
 346 void ioat_channel_free(void *channel_private);
 347 void ioat_channel_intr(ioat_channel_t channel);
 348 int ioat_cmd_alloc(void *channel, int flags, dcopy_cmd_t *cmd);
 349 void ioat_cmd_free(void *channel, dcopy_cmd_t *cmd);
 350 int ioat_cmd_post(void *channel, dcopy_cmd_t cmd);
 351 int ioat_cmd_poll(void *channel, dcopy_cmd_t cmd);
 352 void ioat_unregister_complete(void *device_private, int status);
 353
 354
 355 #endif /* _KERNEL */
 356
 357 #ifdef __cplusplus
 358 }
 359 #endif
 360
 361 #endif /* _SYS_IOAT_H */