arch/x86/kernel/platform/i86pc/include/sys/immu.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Portions Copyright (c) 2010, Oracle and/or its affiliates.
  23  * All rights reserved.
  24  */
  25
  26 /*
  27  * Copyright (c) 2008, Intel Corporation.
  28  * All rights reserved.
  29  */
  30
  31 #ifndef _SYS_INTEL_IOMMU_H
  32 #define _SYS_INTEL_IOMMU_H
  33
  34 /*
  35  * Intel IOMMU implementation specific state
  36  */
  37
  38 #ifdef  __cplusplus
  39 extern "C" {
  40 #endif
  41
  42 #include <sys/types.h>
  43 #include <sys/bitset.h>
  44 #include <sys/kstat.h>
  45 #include <sys/kmem.h>
  46 #include <sys/vmem.h>
  47 #include <sys/rootnex.h>
  48 #include <sys/iommulib.h>
  49 #include <sys/sdt.h>
  50
  51 /*
  52  * Some ON drivers have bugs. Keep this define until all such drivers
  53  * have been fixed
  54  */
  55 #define BUGGY_DRIVERS 1
  56
  57 /* PD(T)E entries */
  58 typedef uint64_t hw_pdte_t;
  59
  60 #define IMMU_MAXNAMELEN (64)
  61 #define IMMU_MAXSEG     (1)
  62 #define IMMU_REGSZ      (1UL << 12)
  63 #define IMMU_PAGESIZE   (4096)
  64 #define IMMU_PAGESHIFT  (12)
  65 #define IMMU_PAGEOFFSET (IMMU_PAGESIZE - 1)
  66 #define IMMU_PAGEMASK   (~IMMU_PAGEOFFSET)
  67 #define IMMU_BTOP(b)    (((uint64_t)b) >> IMMU_PAGESHIFT)
  68 #define IMMU_PTOB(p)    (((uint64_t)p) << IMMU_PAGESHIFT)
  69 #define IMMU_BTOPR(x)   ((((x) + IMMU_PAGEOFFSET) >> IMMU_PAGESHIFT))
  70 #define IMMU_PGTABLE_MAX_LEVELS (6)
  71 #define IMMU_ROUNDUP(size) (((size) + IMMU_PAGEOFFSET) & ~IMMU_PAGEOFFSET)
  72 #define IMMU_ROUNDOWN(addr) ((addr) & ~IMMU_PAGEOFFSET)
  73 #define IMMU_PGTABLE_LEVEL_STRIDE       (9)
  74 #define IMMU_PGTABLE_LEVEL_MASK ((1<<IMMU_PGTABLE_LEVEL_STRIDE) - 1)
  75 #define IMMU_PGTABLE_OFFSHIFT  (IMMU_PAGESHIFT - IMMU_PGTABLE_LEVEL_STRIDE)
  76 #define IMMU_PGTABLE_MAXIDX  ((IMMU_PAGESIZE / sizeof (hw_pdte_t)) - 1)
  77
  78 /*
  79  * DMAR global defines
  80  */
  81 #define DMAR_TABLE      "dmar-table"
  82 #define DMAR_INTRMAP_SUPPORT    (0x01)
  83
  84 /* DMAR unit types */
  85 #define DMAR_DRHD       0
  86 #define DMAR_RMRR       1
  87 #define DMAR_ATSR       2
  88 #define DMAR_RHSA       3
  89
  90 /* DRHD flag values */
  91 #define DMAR_INCLUDE_ALL        (0x01)
  92
  93 /* Device scope types */
  94 #define DMAR_ENDPOINT   1
  95 #define DMAR_SUBTREE    2
  96 #define DMAR_IOAPIC     3
  97 #define DMAR_HPET       4
  98
  99
 100 /* Forward declarations for IOMMU state structure and DVMA domain struct */
 101 struct immu;
 102 struct domain;
 103
 104 /*
 105  * The following structure describes the formate of DMAR ACPI table format.
 106  * They are used to parse DMAR ACPI table. Read the spec for the meaning
 107  * of each member.
 108  */
 109
 110 /* lengths of various strings */
 111 #define DMAR_SIG_LEN    (4)     /* table signature */
 112 #define DMAR_OEMID_LEN  (6)     /* OEM ID */
 113 #define DMAR_TBLID_LEN  (8)     /* OEM table ID */
 114 #define DMAR_ASL_LEN    (4)     /* ASL len */
 115
 116 typedef struct dmar_table {
 117         kmutex_t        tbl_lock;
 118         uint8_t         tbl_haw;
 119         boolean_t       tbl_intrmap;
 120         list_t          tbl_drhd_list[IMMU_MAXSEG];
 121         list_t          tbl_rmrr_list[IMMU_MAXSEG];
 122         char            *tbl_oem_id;
 123         char            *tbl_oem_tblid;
 124         uint32_t        tbl_oem_rev;
 125         caddr_t         tbl_raw;
 126         int             tbl_rawlen;
 127 } dmar_table_t;
 128
 129 typedef struct drhd {
 130         kmutex_t        dr_lock;   /* protects the dmar field */
 131         struct immu     *dr_immu;
 132         dev_info_t      *dr_dip;
 133         uint16_t        dr_seg;
 134         uint64_t        dr_regs;
 135         boolean_t       dr_include_all;
 136         list_t          dr_scope_list;
 137         list_node_t     dr_node;
 138 } drhd_t;
 139
 140 typedef struct rmrr {
 141         kmutex_t        rm_lock;
 142         uint16_t        rm_seg;
 143         uint64_t        rm_base;
 144         uint64_t        rm_limit;
 145         list_t          rm_scope_list;
 146         list_node_t     rm_node;
 147 } rmrr_t;
 148
 149 #define IMMU_UNIT_NAME  "iommu"
 150
 151 /*
 152  * Macros based on PCI spec
 153  */
 154 #define IMMU_PCI_DEV(devfunc)    ((uint64_t)devfunc >> 3) /* from devfunc  */
 155 #define IMMU_PCI_FUNC(devfunc)   (devfunc & 7)  /* get func from devfunc */
 156 #define IMMU_PCI_DEVFUNC(d, f)   (((d) << 3) | (f))  /* create devfunc */
 157
 158 typedef struct scope {
 159         uint8_t scp_type;
 160         uint8_t scp_enumid;
 161         uint8_t scp_bus;
 162         uint8_t scp_dev;
 163         uint8_t scp_func;
 164         list_node_t scp_node;
 165 } scope_t;
 166
 167 /*
 168  * interrupt source id and drhd info for ioapic
 169  */
 170 typedef struct ioapic_drhd {
 171         uchar_t         ioapic_ioapicid;
 172         uint16_t        ioapic_sid;     /* ioapic source id */
 173         drhd_t          *ioapic_drhd;
 174         list_node_t     ioapic_node;
 175 } ioapic_drhd_t;
 176
 177 typedef struct memrng {
 178         uint64_t mrng_start;
 179         uint64_t mrng_npages;
 180 } memrng_t;
 181
 182 typedef enum immu_flags {
 183         IMMU_FLAGS_NONE = 0x1,
 184         IMMU_FLAGS_SLEEP = 0x1,
 185         IMMU_FLAGS_NOSLEEP = 0x2,
 186         IMMU_FLAGS_READ = 0x4,
 187         IMMU_FLAGS_WRITE = 0x8,
 188         IMMU_FLAGS_DONTPASS = 0x10,
 189         IMMU_FLAGS_ALLOC = 0x20,
 190         IMMU_FLAGS_MUST_MATCH = 0x40,
 191         IMMU_FLAGS_PAGE1 = 0x80,
 192         IMMU_FLAGS_UNITY = 0x100,
 193         IMMU_FLAGS_DMAHDL = 0x200,
 194         IMMU_FLAGS_MEMRNG = 0x400
 195 } immu_flags_t;
 196
 197 typedef enum cont_avail {
 198         IMMU_CONT_BAD = 0x0,
 199         IMMU_CONT_UNINITED = 0x1,
 200         IMMU_CONT_INITED = 0x2
 201 } cont_avail_t;
 202
 203 /* Size of root and context tables and their entries */
 204 #define IMMU_ROOT_TBLSZ         (4096)
 205 #define IMMU_CONT_TBLSZ         (4096)
 206 #define IMMU_ROOT_NUM           (256)
 207 #define IMMU_CONT_NUM           (256)
 208
 209 /* register offset */
 210 #define IMMU_REG_VERSION        (0x00)  /* Version Rigister, 32 bit */
 211 #define IMMU_REG_CAP            (0x08)  /* Capability Register, 64 bit */
 212 #define IMMU_REG_EXCAP          (0x10)  /* Extended Capability Reg, 64 bit */
 213 #define IMMU_REG_GLOBAL_CMD     (0x18)  /* Global Command Register, 32 bit */
 214 #define IMMU_REG_GLOBAL_STS     (0x1C)  /* Global Status Register, 32 bit */
 215 #define IMMU_REG_ROOTENTRY      (0x20)  /* Root-Entry Table Addr Reg, 64 bit */
 216 #define IMMU_REG_CONTEXT_CMD    (0x28)  /* Context Comand Register, 64 bit */
 217 #define IMMU_REG_FAULT_STS      (0x34)  /* Fault Status Register, 32 bit */
 218 #define IMMU_REG_FEVNT_CON      (0x38)  /* Fault Event Control Reg, 32 bit */
 219 #define IMMU_REG_FEVNT_DATA     (0x3C)  /* Fault Event Data Register, 32 bit */
 220 #define IMMU_REG_FEVNT_ADDR     (0x40)  /* Fault Event Address Reg, 32 bit */
 221 #define IMMU_REG_FEVNT_UADDR    (0x44)  /* Fault Event Upper Addr Reg, 32 bit */
 222 #define IMMU_REG_AFAULT_LOG     (0x58)  /* Advanced Fault Log Reg, 64 bit */
 223 #define IMMU_REG_PMER           (0x64)  /* Protected Memory Enble Reg, 32 bit */
 224 #define IMMU_REG_PLMBR          (0x68)  /* Protected Low Mem Base Reg, 32 bit */
 225 #define IMMU_REG_PLMLR          (0x6C)  /* Protected Low Mem Lim Reg, 32 bit */
 226 #define IMMU_REG_PHMBR          (0X70)  /* Protectd High Mem Base Reg, 64 bit */
 227 #define IMMU_REG_PHMLR          (0x78)  /* Protected High Mem Lim Reg, 64 bit */
 228 #define IMMU_REG_INVAL_QH       (0x80)  /* Invalidation Queue Head, 64 bit */
 229 #define IMMU_REG_INVAL_QT       (0x88)  /* Invalidation Queue Tail, 64 bit */
 230 #define IMMU_REG_INVAL_QAR      (0x90)  /* Invalidtion Queue Addr Reg, 64 bit */
 231 #define IMMU_REG_INVAL_CSR      (0x9C)  /* Inval Compl Status Reg, 32 bit */
 232 #define IMMU_REG_INVAL_CECR     (0xA0)  /* Inval Compl Evnt Ctrl Reg, 32 bit */
 233 #define IMMU_REG_INVAL_CEDR     (0xA4)  /* Inval Compl Evnt Data Reg, 32 bit */
 234 #define IMMU_REG_INVAL_CEAR     (0xA8)  /* Inval Compl Event Addr Reg, 32 bit */
 235 #define IMMU_REG_INVAL_CEUAR    (0xAC)  /* Inval Comp Evnt Up Addr reg, 32bit */
 236 #define IMMU_REG_IRTAR          (0xB8)  /* INTR Remap Tbl Addr Reg, 64 bit */
 237
 238 /* ioapic memory region */
 239 #define IOAPIC_REGION_START     (0xfee00000)
 240 #define IOAPIC_REGION_END       (0xfeefffff)
 241
 242 /* fault register */
 243 #define IMMU_FAULT_STS_PPF              (2)
 244 #define IMMU_FAULT_STS_PFO              (1)
 245 #define IMMU_FAULT_STS_ITE              (1 << 6)
 246 #define IMMU_FAULT_STS_ICE              (1 << 5)
 247 #define IMMU_FAULT_STS_IQE              (1 << 4)
 248 #define IMMU_FAULT_GET_INDEX(x)         ((((uint64_t)x) >> 8) & 0xff)
 249 #define IMMU_FRR_GET_F(x)               (((uint64_t)x) >> 63)
 250 #define IMMU_FRR_GET_FR(x)              ((((uint64_t)x) >> 32) & 0xff)
 251 #define IMMU_FRR_GET_FT(x)              ((((uint64_t)x) >> 62) & 0x1)
 252 #define IMMU_FRR_GET_SID(x)             ((x) & 0xffff)
 253
 254 /* (ex)capability register */
 255 #define IMMU_CAP_GET_NFR(x)             (((((uint64_t)x) >> 40) & 0xff) + 1)
 256 #define IMMU_CAP_GET_DWD(x)             ((((uint64_t)x) >> 54) & 1)
 257 #define IMMU_CAP_GET_DRD(x)             ((((uint64_t)x) >> 55) & 1)
 258 #define IMMU_CAP_GET_PSI(x)             ((((uint64_t)x) >> 39) & 1)
 259 #define IMMU_CAP_GET_SPS(x)             ((((uint64_t)x) >> 34) & 0xf)
 260 #define IMMU_CAP_GET_ISOCH(x)           ((((uint64_t)x) >> 23) & 1)
 261 #define IMMU_CAP_GET_ZLR(x)             ((((uint64_t)x) >> 22) & 1)
 262 #define IMMU_CAP_GET_MAMV(x)            ((((uint64_t)x) >> 48) & 0x3f)
 263 #define IMMU_CAP_GET_CM(x)              ((((uint64_t)x) >> 7) & 1)
 264 #define IMMU_CAP_GET_PHMR(x)            ((((uint64_t)x) >> 6) & 1)
 265 #define IMMU_CAP_GET_PLMR(x)            ((((uint64_t)x) >> 5) & 1)
 266 #define IMMU_CAP_GET_RWBF(x)            ((((uint64_t)x) >> 4) & 1)
 267 #define IMMU_CAP_GET_AFL(x)             ((((uint64_t)x) >> 3) & 1)
 268 #define IMMU_CAP_GET_FRO(x)             (((((uint64_t)x) >> 24) & 0x3ff) * 16)
 269 #define IMMU_CAP_MGAW(x)                (((((uint64_t)x) >> 16) & 0x3f) + 1)
 270 #define IMMU_CAP_SAGAW(x)               ((((uint64_t)x) >> 8) & 0x1f)
 271 #define IMMU_CAP_ND(x)                  (1 << (((x) & 0x7) *2 + 4)) -1
 272 #define IMMU_ECAP_GET_IRO(x)            (((((uint64_t)x) >> 8) & 0x3ff) << 4)
 273 #define IMMU_ECAP_GET_MHMV(x)           (((uint64_t)x >> 20) & 0xf)
 274 #define IMMU_ECAP_GET_SC(x)             ((x) & 0x80)
 275 #define IMMU_ECAP_GET_PT(x)             ((x) & 0x40)
 276 #define IMMU_ECAP_GET_CH(x)             ((x) & 0x20)
 277 #define IMMU_ECAP_GET_EIM(x)            ((x) & 0x10)
 278 #define IMMU_ECAP_GET_IR(x)             ((x) & 0x8)
 279 #define IMMU_ECAP_GET_DI(x)             ((x) & 0x4)
 280 #define IMMU_ECAP_GET_QI(x)             ((x) & 0x2)
 281 #define IMMU_ECAP_GET_C(x)              ((x) & 0x1)
 282
 283 #define IMMU_CAP_SET_RWBF(x)            ((x) |= (1 << 4))
 284
 285
 286 /* iotlb invalidation */
 287 #define TLB_INV_GLOBAL          (((uint64_t)1) << 60)
 288 #define TLB_INV_DOMAIN          (((uint64_t)2) << 60)
 289 #define TLB_INV_PAGE            (((uint64_t)3) << 60)
 290 #define TLB_INV_GET_IAIG(x)     ((((uint64_t)x) >> 57) & 7)
 291 #define TLB_INV_DRAIN_READ      (((uint64_t)1) << 49)
 292 #define TLB_INV_DRAIN_WRITE     (((uint64_t)1) << 48)
 293 #define TLB_INV_DID(x)          (((uint64_t)((x) & 0xffff)) << 32)
 294 #define TLB_INV_IVT             (((uint64_t)1) << 63)
 295 #define TLB_IVA_HINT(x)         (((x) & 0x1) << 6)
 296 #define TLB_IVA_LEAF            1
 297 #define TLB_IVA_WHOLE           0
 298
 299 /* dont use value 0 for  enums - to catch unit 8 */
 300 typedef enum iotlb_inv {
 301         IOTLB_PSI = 1,
 302         IOTLB_DSI,
 303         IOTLB_GLOBAL
 304 } immu_iotlb_inv_t;
 305
 306 typedef enum context_inv {
 307         CONTEXT_FSI = 1,
 308         CONTEXT_DSI,
 309         CONTEXT_GLOBAL
 310 } immu_context_inv_t;
 311
 312 /* context invalidation */
 313 #define CCMD_INV_ICC            (((uint64_t)1) << 63)
 314 #define CCMD_INV_GLOBAL         (((uint64_t)1) << 61)
 315 #define CCMD_INV_DOMAIN         (((uint64_t)2) << 61)
 316 #define CCMD_INV_DEVICE         (((uint64_t)3) << 61)
 317 #define CCMD_INV_DID(x)         ((uint64_t)((x) & 0xffff))
 318 #define CCMD_INV_SID(x)         (((uint64_t)((x) & 0xffff)) << 16)
 319 #define CCMD_INV_FM(x)          (((uint64_t)((x) & 0x3)) << 32)
 320
 321 /* global command register */
 322 #define IMMU_GCMD_TE            (((uint32_t)1) << 31)
 323 #define IMMU_GCMD_SRTP          (((uint32_t)1) << 30)
 324 #define IMMU_GCMD_SFL           (((uint32_t)1) << 29)
 325 #define IMMU_GCMD_EAFL          (((uint32_t)1) << 28)
 326 #define IMMU_GCMD_WBF           (((uint32_t)1) << 27)
 327 #define IMMU_GCMD_QIE           (((uint32_t)1) << 26)
 328 #define IMMU_GCMD_IRE           (((uint32_t)1) << 25)
 329 #define IMMU_GCMD_SIRTP (((uint32_t)1) << 24)
 330 #define IMMU_GCMD_CFI           (((uint32_t)1) << 23)
 331
 332 /* global status register */
 333 #define IMMU_GSTS_TES           (((uint32_t)1) << 31)
 334 #define IMMU_GSTS_RTPS          (((uint32_t)1) << 30)
 335 #define IMMU_GSTS_FLS           (((uint32_t)1) << 29)
 336 #define IMMU_GSTS_AFLS          (((uint32_t)1) << 28)
 337 #define IMMU_GSTS_WBFS          (((uint32_t)1) << 27)
 338 #define IMMU_GSTS_QIES          (((uint32_t)1) << 26)
 339 #define IMMU_GSTS_IRES          (((uint32_t)1) << 25)
 340 #define IMMU_GSTS_IRTPS (((uint32_t)1) << 24)
 341 #define IMMU_GSTS_CFIS          (((uint32_t)1) << 23)
 342
 343 /* psi address mask */
 344 #define ADDR_AM_MAX(m)          (((uint_t)1) << (m))
 345 #define ADDR_AM_OFFSET(n, m)    ((n) & (ADDR_AM_MAX(m) - 1))
 346
 347 /* dmar fault event */
 348 #define IMMU_INTR_IPL                   (4)
 349 #define IMMU_REG_FEVNT_CON_IM_SHIFT     (31)
 350
 351 #define IMMU_ALLOC_RESOURCE_DELAY    (drv_usectohz(5000))
 352
 353 /* max value of Size field of Interrupt Remapping Table Address Register */
 354 #define INTRMAP_MAX_IRTA_SIZE   0xf
 355
 356 /* interrupt remapping table entry size */
 357 #define INTRMAP_RTE_SIZE                0x10
 358
 359 /* ioapic redirection table entry related shift of remappable interrupt */
 360 #define INTRMAP_IOAPIC_IDX_SHIFT                17
 361 #define INTRMAP_IOAPIC_FORMAT_SHIFT     16
 362 #define INTRMAP_IOAPIC_TM_SHIFT         15
 363 #define INTRMAP_IOAPIC_POL_SHIFT                13
 364 #define INTRMAP_IOAPIC_IDX15_SHIFT      11
 365
 366 /* msi intr entry related shift of remappable interrupt */
 367 #define INTRMAP_MSI_IDX_SHIFT   5
 368 #define INTRMAP_MSI_FORMAT_SHIFT        4
 369 #define INTRMAP_MSI_SHV_SHIFT   3
 370 #define INTRMAP_MSI_IDX15_SHIFT 2
 371
 372 #define INTRMAP_IDX_FULL                (uint_t)-1
 373
 374 #define RDT_DLM(rdt)    BITX((rdt), 10, 8)
 375 #define RDT_DM(rdt)     BT_TEST(&(rdt), 11)
 376 #define RDT_POL(rdt)    BT_TEST(&(rdt), 13)
 377 #define RDT_TM(rdt)     BT_TEST(&(rdt), 15)
 378
 379 #define INTRMAP_DISABLE (void *)-1
 380
 381 /*
 382  * invalidation granularity
 383  */
 384 typedef enum {
 385         TLB_INV_G_GLOBAL = 1,
 386         TLB_INV_G_DOMAIN,
 387         TLB_INV_G_PAGE
 388 } tlb_inv_g_t;
 389
 390 typedef enum {
 391         CTT_INV_G_GLOBAL = 1,
 392         CTT_INV_G_DOMAIN,
 393         CTT_INV_G_DEVICE
 394 } ctt_inv_g_t;
 395
 396 typedef enum {
 397         IEC_INV_GLOBAL = 0,
 398         IEC_INV_INDEX
 399 } iec_inv_g_t;
 400
 401
 402 struct inv_queue_state;
 403 struct intrmap_tbl_state;
 404
 405 /* A software page table structure */
 406 typedef struct pgtable {
 407         krwlock_t swpg_rwlock;
 408         caddr_t hwpg_vaddr;   /* HW pgtable VA */
 409         paddr_t hwpg_paddr;   /* HW pgtable PA */
 410         ddi_dma_handle_t hwpg_dmahdl;
 411         ddi_acc_handle_t hwpg_memhdl;
 412         struct pgtable **swpg_next_array;
 413         list_node_t swpg_domain_node;  /* domain list of pgtables */
 414 } pgtable_t;
 415
 416 /* interrupt remapping table state info */
 417 typedef struct intrmap {
 418         kmutex_t                intrmap_lock;
 419         ddi_dma_handle_t        intrmap_dma_hdl;
 420         ddi_acc_handle_t        intrmap_acc_hdl;
 421         caddr_t                 intrmap_vaddr;
 422         paddr_t                 intrmap_paddr;
 423         uint_t                  intrmap_size;
 424         bitset_t                intrmap_map;
 425         uint_t                  intrmap_free;
 426 } intrmap_t;
 427
 428 typedef struct hw_rce {
 429         uint64_t lo;
 430         uint64_t hi;
 431 } hw_rce_t;
 432
 433
 434 #define ROOT_GET_P(hrent) ((hrent)->lo & 0x1)
 435 #define ROOT_SET_P(hrent) ((hrent)->lo |= 0x1)
 436
 437 #define ROOT_GET_CONT(hrent) ((hrent)->lo & ~(0xFFF))
 438 #define ROOT_SET_CONT(hrent, paddr) ((hrent)->lo |= (paddr & (~0xFFF)))
 439
 440 #define TTYPE_XLATE_ONLY  (0x0)
 441 #define TTYPE_XLATE_IOTLB (0x1)
 442 #define TTYPE_PASSTHRU    (0x2)
 443 #define TTYPE_RESERVED    (0x3)
 444
 445 #define CONT_GET_DID(hcent) ((((uint64_t)(hcent)->hi) >> 8) & 0xFFFF)
 446 #define CONT_SET_DID(hcent, did) ((hcent)->hi |= ((0xFFFF & (did)) << 8))
 447
 448 #define CONT_GET_AVAIL(hcent) ((((uint64_t)((hcent)->hi)) >> 0x3) & 0xF)
 449 #define CONT_SET_AVAIL(hcent, av) ((hcent)->hi |= ((0xF & (av)) << 0x3))
 450
 451 #define CONT_GET_LO_AW(hcent) (30 + 9 *((hcent)->hi & 0x7))
 452 #define CONT_GET_AW(hcent) \
 453         ((CONT_GET_LO_AW(hcent) == 66) ? 64 : CONT_GET_LO_AW(hcent))
 454 #define CONT_SET_AW(hcent, aw) \
 455         ((hcent)->hi |= (((((aw) + 2) - 30) / 9) & 0x7))
 456
 457 #define CONT_GET_ASR(hcent) ((hcent)->lo & ~(0xFFF))
 458 #define CONT_SET_ASR(hcent, paddr) ((hcent)->lo |= (paddr & (~0xFFF)))
 459
 460 #define CONT_GET_TTYPE(hcent) ((((uint64_t)(hcent)->lo) >> 0x2) & 0x3)
 461 #define CONT_SET_TTYPE(hcent, ttype) ((hcent)->lo |= (((ttype) & 0x3) << 0x2))
 462
 463 #define CONT_GET_P(hcent) ((hcent)->lo & 0x1)
 464 #define CONT_SET_P(hcent) ((hcent)->lo |= 0x1)
 465
 466 #define CONT_GET_ALH(hcent) ((hcent)->lo & 0x20)
 467 #define CONT_SET_ALH(hcent) ((hcent)->lo |= 0x20)
 468
 469 #define CONT_GET_EH(hcent) ((hcent)->lo & 0x10)
 470 #define CONT_SET_EH(hcent) ((hcent)->lo |= 0x10)
 471
 472
 473 /* we use the bit 63 (available for system SW) as a present bit */
 474 #define PDTE_SW4(hw_pdte) ((hw_pdte) & ((uint64_t)1<<63))
 475 #define PDTE_CLEAR_SW4(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<63))
 476
 477 #define PDTE_P(hw_pdte) ((hw_pdte) & ((uint64_t)1<<63))
 478 #define PDTE_CLEAR_P(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<63))
 479 #define PDTE_SET_P(hw_pdte) ((hw_pdte) |= ((uint64_t)1<<63))
 480
 481 #define PDTE_TM(hw_pdte) ((hw_pdte) & ((uint64_t)1<<62))
 482 #define PDTE_CLEAR_TM(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<62))
 483
 484 #define PDTE_SW3(hw_pdte) \
 485         (((hw_pdte) & ~(((uint64_t)0x3<<62)|(((uint64_t)1<<52)-1))) >> 52)
 486 #define PDTE_SW3_OVERFLOW(hw_pdte) \
 487         (PDTE_SW3(hw_pdte) == 0x3FF)
 488 #define PDTE_CLEAR_SW3(hw_pdte) \
 489         ((hw_pdte) &= (((uint64_t)0x3<<62)|(((uint64_t)1<<52)-1)))
 490 #define PDTE_SET_SW3(hw_pdte, ref) \
 491         ((hw_pdte) |= ((((uint64_t)(ref)) & 0x3FF) << 52))
 492
 493 #define PDTE_PADDR(hw_pdte) ((hw_pdte) & ~(((uint64_t)0xFFF<<52)|((1<<12)-1)))
 494 #define PDTE_CLEAR_PADDR(hw_pdte) \
 495                 ((hw_pdte) &= (((uint64_t)0xFFF<<52)|((1<<12)-1)))
 496 #define PDTE_SET_PADDR(hw_pdte, paddr) ((hw_pdte) |= PDTE_PADDR(paddr))
 497
 498 #define PDTE_SNP(hw_pdte) ((hw_pdte) & (1<<11))
 499 #define PDTE_CLEAR_SNP(hw_pdte) ((hw_pdte) &= ~(1<<11))
 500 #define PDTE_SET_SNP(hw_pdte) ((hw_pdte) |= (1<<11))
 501
 502 #define PDTE_SW2(hw_pdte) ((hw_pdte) & (0x700))
 503 #define PDTE_CLEAR_SW2(hw_pdte) ((hw_pdte) &= ~(0x700))
 504
 505 #define PDTE_SP(hw_pdte) ((hw_pdte) & (0x80))
 506 #define PDTE_CLEAR_SP(hw_pdte) ((hw_pdte) &= ~(0x80))
 507
 508 #define PDTE_SW1(hw_pdte) ((hw_pdte) & (0x7C))
 509 #define PDTE_CLEAR_SW1(hw_pdte) ((hw_pdte) &= ~(0x7C))
 510
 511 #define PDTE_WRITE(hw_pdte) ((hw_pdte) & (0x2))
 512 #define PDTE_CLEAR_WRITE(hw_pdte) ((hw_pdte) &= ~(0x2))
 513 #define PDTE_SET_WRITE(hw_pdte) ((hw_pdte) |= (0x2))
 514
 515 #define PDTE_READ(hw_pdte) ((hw_pdte) & (0x1))
 516 #define PDTE_CLEAR_READ(hw_pdte) ((hw_pdte) &= ~(0x1))
 517 #define PDTE_SET_READ(hw_pdte) ((hw_pdte) |= (0x1))
 518
 519 #define PDTE_MASK_R     ((uint64_t)1 << 0)
 520 #define PDTE_MASK_W     ((uint64_t)1 << 1)
 521 #define PDTE_MASK_SNP   ((uint64_t)1 << 11)
 522 #define PDTE_MASK_TM    ((uint64_t)1 << 62)
 523 #define PDTE_MASK_P     ((uint64_t)1 << 63)
 524
 525 struct immu_flushops;
 526
 527 /*
 528  * Used to wait for invalidation completion.
 529  *     vstatus is the virtual address of the status word that will be written
 530  *     pstatus is the physical addres
 531  * If sync is true, then the the operation will be waited on for
 532  * completion immediately. Else, the wait interface can be called
 533  * to wait for completion later.
 534  */
 535
 536 #define IMMU_INV_DATA_PENDING   1
 537 #define IMMU_INV_DATA_DONE      2
 538
 539 typedef struct immu_inv_wait {
 540         volatile uint32_t iwp_vstatus;
 541         uint64_t iwp_pstatus;
 542         boolean_t iwp_sync;
 543         const char *iwp_name;           /* ID for debugging/statistics */
 544 } immu_inv_wait_t;
 545
 546 /*
 547  * Used to batch IOMMU pagetable writes.
 548  */
 549 typedef struct immu_dcookie {
 550         paddr_t dck_paddr;
 551         uint64_t dck_npages;
 552 } immu_dcookie_t;
 553
 554 typedef struct immu {
 555         kmutex_t                immu_lock;
 556         char                    *immu_name;
 557
 558         /* lock grabbed by interrupt handler */
 559         kmutex_t                immu_intr_lock;
 560
 561         /* ACPI/DMAR table related */
 562         void                    *immu_dmar_unit;
 563         dev_info_t              *immu_dip;
 564         struct domain           *immu_unity_domain;
 565
 566         /* IOMMU register related */
 567         kmutex_t                immu_regs_lock;
 568         kcondvar_t              immu_regs_cv;
 569         boolean_t               immu_regs_busy;
 570         boolean_t               immu_regs_setup;
 571         boolean_t               immu_regs_running;
 572         boolean_t               immu_regs_quiesced;
 573         ddi_acc_handle_t        immu_regs_handle;
 574         caddr_t                 immu_regs_addr;
 575         uint64_t                immu_regs_cap;
 576         uint64_t                immu_regs_excap;
 577         uint32_t                immu_regs_cmdval;
 578         uint32_t                immu_regs_intr_msi_addr;
 579         uint32_t                immu_regs_intr_msi_data;
 580         uint32_t                immu_regs_intr_uaddr;
 581
 582         /* DVMA related */
 583         kmutex_t                immu_dvma_lock;
 584         boolean_t               immu_dvma_setup;
 585         boolean_t               immu_dvma_running;
 586         int                     immu_dvma_gaw;
 587         int                     immu_dvma_agaw;
 588         int                     immu_dvma_nlevels;
 589         boolean_t               immu_dvma_coherent;
 590         boolean_t               immu_TM_reserved;
 591         boolean_t               immu_SNP_reserved;
 592         uint64_t                immu_ptemask;
 593
 594         /* DVMA context related */
 595         krwlock_t               immu_ctx_rwlock;
 596         pgtable_t               *immu_ctx_root;
 597         immu_inv_wait_t         immu_ctx_inv_wait;
 598
 599         /* DVMA domain related */
 600         int                     immu_max_domains;
 601         vmem_t                  *immu_did_arena;
 602         char                    immu_did_arena_name[IMMU_MAXNAMELEN];
 603         list_t                  immu_domain_list;
 604
 605         /* DVMA special devices */
 606         boolean_t               immu_dvma_gfx_only;
 607         list_t                  immu_dvma_lpc_list;
 608         list_t                  immu_dvma_gfx_list;
 609
 610         /* interrupt remapping related */
 611         kmutex_t                immu_intrmap_lock;
 612         boolean_t               immu_intrmap_setup;
 613         boolean_t               immu_intrmap_running;
 614         intrmap_t               *immu_intrmap;
 615         uint64_t                immu_intrmap_irta_reg;
 616         immu_inv_wait_t         immu_intrmap_inv_wait;
 617
 618         /* queued invalidation related */
 619         kmutex_t                immu_qinv_lock;
 620         boolean_t               immu_qinv_setup;
 621         boolean_t               immu_qinv_running;
 622         boolean_t               immu_qinv_enabled;
 623         void                    *immu_qinv;
 624         uint64_t                immu_qinv_reg_value;
 625
 626         /* list_node for system-wide list of DMAR units */
 627         list_node_t             immu_node;
 628
 629         struct immu_flushops    *immu_flushops;
 630
 631         kmem_cache_t            *immu_hdl_cache;
 632         kmem_cache_t            *immu_pgtable_cache;
 633
 634         iommulib_handle_t       immu_iommulib_handle;
 635 } immu_t;
 636
 637 /*
 638  * Enough space to hold the decimal number of any device instance.
 639  * Used for device/cache names.
 640  */
 641 #define IMMU_ISTRLEN    11      /* log10(2^31)  + 1 */
 642
 643 /* properties that control DVMA */
 644 #define DDI_DVMA_MAPTYPE_ROOTNEX_PROP   "immu-dvma-mapping"
 645
 646 #define DDI_DVMA_MAPTYPE_UNITY          "unity"
 647 #define DDI_DVMA_MAPTYPE_XLATE          "xlate"
 648
 649 typedef enum immu_maptype {
 650         IMMU_MAPTYPE_BAD = 0,    /* 0 is always bad */
 651         IMMU_MAPTYPE_UNITY = 1,
 652         IMMU_MAPTYPE_XLATE
 653 } immu_maptype_t;
 654
 655 #define IMMU_COOKIE_HASHSZ      (512)
 656
 657 /*
 658  * domain_t
 659  *
 660  */
 661 typedef struct domain {
 662         /* the basics */
 663         uint_t                  dom_did;
 664         immu_t                  *dom_immu;
 665
 666         /* mapping related */
 667         immu_maptype_t          dom_maptype;
 668         vmem_t                  *dom_dvma_arena;
 669         char                    dom_dvma_arena_name[IMMU_MAXNAMELEN];
 670
 671         /* pgtables */
 672         pgtable_t               *dom_pgtable_root;
 673         krwlock_t               dom_pgtable_rwlock;
 674
 675         /* list node for list of domains (unity or xlate) */
 676         list_node_t             dom_maptype_node;
 677         /* list node for list of domains off immu */
 678         list_node_t             dom_immu_node;
 679
 680         mod_hash_t              *dom_cookie_hash;
 681
 682         /* topmost device in domain; usually the device itself (non-shared) */
 683         dev_info_t              *dom_dip;
 684 } domain_t;
 685
 686 typedef enum immu_pcib {
 687         IMMU_PCIB_BAD = 0,
 688         IMMU_PCIB_NOBDF,
 689         IMMU_PCIB_PCIE_PCIE,
 690         IMMU_PCIB_PCIE_PCI,
 691         IMMU_PCIB_PCI_PCI,
 692         IMMU_PCIB_ENDPOINT
 693 } immu_pcib_t;
 694
 695 /*
 696  *  immu_devi_t
 697  *      Intel IOMMU in devinfo node
 698  */
 699 typedef struct immu_devi {
 700         /* pci seg, bus, dev, func */
 701         int             imd_seg;
 702         int             imd_bus;
 703         int             imd_devfunc;
 704
 705         /* ppb information */
 706         immu_pcib_t     imd_pcib_type;
 707         int             imd_sec;
 708         int             imd_sub;
 709
 710         /* identifier for special devices */
 711         boolean_t       imd_display;
 712         boolean_t       imd_lpc;
 713
 714         /* set if premapped DVMA space is used */
 715         boolean_t       imd_use_premap;
 716
 717         /* dmar unit to which this dip belongs */
 718         immu_t          *imd_immu;
 719
 720         immu_flags_t    imd_dvma_flags;
 721
 722         /* domain ptr */
 723         domain_t        *imd_domain;
 724         dev_info_t      *imd_ddip;
 725
 726         /* my devinfo */
 727         dev_info_t      *imd_dip;
 728
 729         /*
 730          * if we are a "special" devinfo
 731          * the node for the special linked list
 732          * off the DMAR unit structure
 733          */
 734         list_node_t     imd_spc_node;
 735 } immu_devi_t;
 736
 737 #define IMMU_DEVI(dip)          ((immu_devi_t *)(DEVI(dip)->devi_iommu))
 738 #define IMMU_DEVI_SET(dip, imd) (DEVI(dip)->devi_iommu = (void *)imd)
 739
 740 /*
 741  * struct dmar_arg
 742  */
 743 typedef struct immu_arg {
 744         int             ima_seg;
 745         int             ima_bus;
 746         int             ima_devfunc;
 747         dev_info_t      *ima_rdip;
 748         dev_info_t      *ima_ddip;
 749 } immu_arg_t;
 750
 751 #define IMMU_NDVSEG     8
 752 #define IMMU_NDCK       64
 753 #define IMMU_NPREPTES   8
 754
 755 typedef struct immu_hdl_private {
 756         immu_inv_wait_t ihp_inv_wait;
 757         size_t ihp_ndvseg;
 758         struct dvmaseg ihp_dvseg[IMMU_NDVSEG];
 759         immu_dcookie_t ihp_dcookies[IMMU_NDCK];
 760
 761         hw_pdte_t *ihp_preptes[IMMU_NPREPTES];
 762         uint64_t ihp_predvma;
 763         int ihp_npremapped;
 764 } immu_hdl_priv_t;
 765
 766 /*
 767  * Invalidation operation function pointers for context and IOTLB.
 768  * These will be set to either the register or the queue invalidation
 769  * interface functions, since the hardware does not allow using them
 770  * both at the same time.
 771  */
 772 struct immu_flushops {
 773         void (*imf_context_fsi)(immu_t *, uint8_t, uint16_t, uint_t,
 774             immu_inv_wait_t *);
 775         void (*imf_context_dsi)(immu_t *, uint_t, immu_inv_wait_t *);
 776         void (*imf_context_gbl)(immu_t *, immu_inv_wait_t *);
 777
 778         void (*imf_iotlb_psi)(immu_t *, uint_t, uint64_t, uint_t, uint_t,
 779             immu_inv_wait_t *);
 780         void (*imf_iotlb_dsi)(immu_t *, uint_t, immu_inv_wait_t *);
 781         void (*imf_iotlb_gbl)(immu_t *, immu_inv_wait_t *);
 782
 783         void (*imf_wait)(immu_inv_wait_t *);
 784 };
 785
 786 #define immu_flush_context_fsi(i, f, s, d, w) \
 787         (i)->immu_flushops->imf_context_fsi(i, f, s, d, w)
 788 #define immu_flush_context_dsi(i, d, w) \
 789         (i)->immu_flushops->imf_context_dsi(i, d, w)
 790 #define immu_flush_context_gbl(i, w) \
 791         (i)->immu_flushops->imf_context_gbl(i, w)
 792
 793 #define immu_flush_iotlb_psi(i, d, v, c, h, w) \
 794         (i)->immu_flushops->imf_iotlb_psi(i, d, v, c, h, w)
 795 #define immu_flush_iotlb_dsi(i, d, w) \
 796         (i)->immu_flushops->imf_iotlb_dsi(i, d, w)
 797 #define immu_flush_iotlb_gbl(i, w) \
 798         (i)->immu_flushops->imf_iotlb_gbl(i, w)
 799
 800 #define immu_flush_wait(i, w) \
 801         (i)->immu_flushops->imf_wait(w)
 802
 803 /*
 804  * Globals used by IOMMU code
 805  */
 806 /* shared between IOMMU files */
 807 extern dev_info_t *root_devinfo;
 808 extern kmutex_t immu_lock;
 809 extern list_t immu_list;
 810 extern boolean_t immu_setup;
 811 extern boolean_t immu_running;
 812 extern kmutex_t ioapic_drhd_lock;
 813 extern list_t ioapic_drhd_list;
 814 extern struct iommulib_ops immulib_ops;
 815
 816 /* switches */
 817
 818 /* Various features */
 819 extern boolean_t immu_enable;
 820 extern boolean_t immu_gfxdvma_enable;
 821 extern boolean_t immu_intrmap_enable;
 822 extern boolean_t immu_qinv_enable;
 823
 824 /* various quirks that need working around */
 825 extern boolean_t immu_quirk_usbpage0;
 826 extern boolean_t immu_quirk_usbfullpa;
 827 extern boolean_t immu_quirk_usbrmrr;
 828 extern boolean_t immu_quirk_mobile4;
 829
 830 /* debug messages */
 831 extern boolean_t immu_dmar_print;
 832
 833 /* tunables */
 834 extern int64_t immu_flush_gran;
 835
 836 extern immu_flags_t immu_global_dvma_flags;
 837
 838 extern int immu_use_tm;
 839 extern int immu_use_alh;
 840
 841 /* ################### Interfaces exported outside IOMMU code ############## */
 842 void immu_init(void);
 843 void immu_startup(void);
 844 void immu_shutdown(void);
 845 void immu_destroy(void);
 846 int immu_map_sgl(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
 847     int prealloc_count, dev_info_t *rdip);
 848 int immu_unmap_sgl(ddi_dma_impl_t *hp, dev_info_t *rdip);
 849 void immu_device_tree_changed(void);
 850 void immu_physmem_update(uint64_t addr, uint64_t size);
 851 int immu_quiesce(void);
 852 int immu_unquiesce(void);
 853 /* ######################################################################### */
 854
 855 /* ################# Interfaces used within IOMMU code #################### */
 856 /* immu_dmar.c interfaces */
 857 int immu_dmar_setup(void);
 858 int immu_dmar_parse(void);
 859 void immu_dmar_startup(void);
 860 void immu_dmar_shutdown(void);
 861 void immu_dmar_destroy(void);
 862 boolean_t immu_dmar_blacklisted(char **strings_array, uint_t nstrings);
 863 immu_t *immu_dmar_get_immu(dev_info_t *rdip);
 864 dev_info_t *immu_dmar_unit_dip(void *dmar_unit);
 865 void immu_dmar_set_immu(void *dmar_unit, immu_t *immu);
 866 void *immu_dmar_walk_units(int seg, void *dmar_unit);
 867 boolean_t immu_dmar_intrmap_supported(void);
 868 uint16_t immu_dmar_ioapic_sid(int ioapicid);
 869 immu_t *immu_dmar_ioapic_immu(int ioapicid);
 870 void immu_dmar_rmrr_map(void);
 871
 872 /* immu.c interfaces */
 873 int immu_walk_ancestor(dev_info_t *rdip, dev_info_t *ddip,
 874     int (*func)(dev_info_t *, void *arg), void *arg,
 875     int *level, immu_flags_t immu_flags);
 876 void immu_init_inv_wait(immu_inv_wait_t *iwp, const char *s, boolean_t sync);
 877
 878 /* immu_regs.c interfaces */
 879 void immu_regs_setup(list_t *immu_list);
 880 void immu_regs_startup(immu_t *immu);
 881 int immu_regs_resume(immu_t *immu);
 882 void immu_regs_suspend(immu_t *immu);
 883 void immu_regs_shutdown(immu_t *immu);
 884 void immu_regs_destroy(list_t *immu_list);
 885
 886 void immu_regs_intr(immu_t *immu, uint32_t msi_addr, uint32_t msi_data,
 887     uint32_t uaddr);
 888
 889 boolean_t immu_regs_passthru_supported(immu_t *immu);
 890 boolean_t immu_regs_is_TM_reserved(immu_t *immu);
 891 boolean_t immu_regs_is_SNP_reserved(immu_t *immu);
 892
 893 void immu_regs_wbf_flush(immu_t *immu);
 894 void immu_regs_cpu_flush(immu_t *immu, caddr_t addr, uint_t size);
 895
 896 void immu_regs_context_fsi(immu_t *immu, uint8_t function_mask,
 897     uint16_t source_id, uint_t domain_id, immu_inv_wait_t *iwp);
 898 void immu_regs_context_dsi(immu_t *immu, uint_t domain_id,
 899     immu_inv_wait_t *iwp);
 900 void immu_regs_context_gbl(immu_t *immu, immu_inv_wait_t *iwp);
 901 void immu_regs_iotlb_psi(immu_t *immu, uint_t domain_id,
 902     uint64_t dvma, uint_t count, uint_t hint, immu_inv_wait_t *iwp);
 903 void immu_regs_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp);
 904 void immu_regs_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp);
 905
 906 void immu_regs_set_root_table(immu_t *immu);
 907 void immu_regs_qinv_enable(immu_t *immu, uint64_t qinv_reg_value);
 908 void immu_regs_intr_enable(immu_t *immu, uint32_t msi_addr, uint32_t msi_data,
 909     uint32_t uaddr);
 910 void immu_regs_intrmap_enable(immu_t *immu, uint64_t irta_reg);
 911 uint64_t immu_regs_get64(immu_t *immu, uint_t reg);
 912 void immu_regs_put64(immu_t *immu, uint_t reg, uint64_t val);
 913 uint32_t immu_regs_get32(immu_t *immu, uint_t reg);
 914 void immu_regs_put32(immu_t *immu, uint_t reg, uint32_t val);
 915
 916 /* immu_dvma.c interfaces */
 917 void immu_dvma_setup(list_t *immu_list);
 918 void immu_dvma_startup(immu_t *immu);
 919 void immu_dvma_shutdown(immu_t *immu);
 920 void immu_dvma_destroy(list_t *immu_list);
 921
 922 void immu_dvma_physmem_update(uint64_t addr, uint64_t size);
 923 int immu_map_memrange(dev_info_t *, memrng_t *);
 924 int immu_dvma_map(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
 925     uint_t prealloc_count, dev_info_t *rdip);
 926 int immu_dvma_unmap(ddi_dma_impl_t *hp, dev_info_t *rdip);
 927 int immu_devi_set(dev_info_t *dip, immu_flags_t immu_flags);
 928 immu_devi_t *immu_devi_get(dev_info_t *dip);
 929 immu_t *immu_dvma_get_immu(dev_info_t *dip, immu_flags_t immu_flags);
 930 int pgtable_ctor(void *buf, void *arg, int kmflag);
 931 void pgtable_dtor(void *buf, void *arg);
 932
 933 int immu_hdl_priv_ctor(void *buf, void *arg, int kmf);
 934
 935 int immu_dvma_device_setup(dev_info_t *rdip, immu_flags_t immu_flags);
 936
 937 void immu_print_fault_info(uint_t sid, uint64_t dvma);
 938
 939 /* immu_intrmap.c interfaces */
 940 void immu_intrmap_setup(list_t *immu_list);
 941 void immu_intrmap_startup(immu_t *immu);
 942 void immu_intrmap_shutdown(immu_t *immu);
 943 void immu_intrmap_destroy(list_t *immu_list);
 944
 945 /* registers interrupt handler for IOMMU unit */
 946 void immu_intr_register(immu_t *immu);
 947 int immu_intr_handler(immu_t *immu);
 948
 949
 950 /* immu_qinv.c interfaces */
 951 int immu_qinv_setup(list_t *immu_list);
 952 void immu_qinv_startup(immu_t *immu);
 953 void immu_qinv_shutdown(immu_t *immu);
 954 void immu_qinv_destroy(list_t *immu_list);
 955
 956 void immu_qinv_context_fsi(immu_t *immu, uint8_t function_mask,
 957     uint16_t source_id, uint_t domain_id, immu_inv_wait_t *iwp);
 958 void immu_qinv_context_dsi(immu_t *immu, uint_t domain_id,
 959     immu_inv_wait_t *iwp);
 960 void immu_qinv_context_gbl(immu_t *immu, immu_inv_wait_t *iwp);
 961 void immu_qinv_iotlb_psi(immu_t *immu, uint_t domain_id,
 962     uint64_t dvma, uint_t count, uint_t hint, immu_inv_wait_t *iwp);
 963 void immu_qinv_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp);
 964 void immu_qinv_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp);
 965
 966 void immu_qinv_intr_global(immu_t *immu, immu_inv_wait_t *iwp);
 967 void immu_qinv_intr_one_cache(immu_t *immu, uint_t idx, immu_inv_wait_t *iwp);
 968 void immu_qinv_intr_caches(immu_t *immu, uint_t idx, uint_t cnt,
 969     immu_inv_wait_t *);
 970 void immu_qinv_report_fault(immu_t *immu);
 971
 972 #ifdef DEBUG
 973 #define IMMU_DPROBE1(name, type1, arg1) \
 974         DTRACE_PROBE1(name, type1, arg1)
 975 #define IMMU_DPROBE2(name, type1, arg1, type2, arg2) \
 976         DTRACE_PROBE2(name, type1, arg1, type2, arg2)
 977 #define IMMU_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) \
 978         DTRACE_PROBE3(name, type1, arg1, type2, arg2, type3, arg3)
 979 #define IMMU_DPROBE4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4) \
 980         DTRACE_PROBE4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4)
 981 #else
 982 #define IMMU_DPROBE1(name, type1, arg1)
 983 #define IMMU_DPROBE2(name, type1, arg1, type2, arg2)
 984 #define IMMU_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3)
 985 #define IMMU_DPROBE4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4)
 986 #endif
 987
 988
 989 #ifdef  __cplusplus
 990 }
 991 #endif
 992
 993 #endif  /* _SYS_INTEL_IOMMU_H */