?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
dmake: do not set MAKEFLAGS=k
[unleashed/tickless.git] / usr / src / uts / i86pc / io / immu_intrmap.c
blob9ba26b5d745c0a3547a125e11ce153b88976126e
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2009, Intel Corporation.
28 * All rights reserved.
29 */
30
31
32 #include <sys/apic.h>
33 #include <vm/hat_i86.h>
34 #include <sys/sysmacros.h>
35 #include <sys/smp_impldefs.h>
36 #include <sys/immu.h>
37
38
39 typedef struct intrmap_private {
40 immu_t *ir_immu;
41 immu_inv_wait_t ir_inv_wait;
42 uint16_t ir_idx;
43 uint32_t ir_sid_svt_sq;
44 } intrmap_private_t;
45
46 #define INTRMAP_PRIVATE(intrmap) ((intrmap_private_t *)intrmap)
47
48 /* interrupt remapping table entry */
49 typedef struct intrmap_rte {
50 uint64_t lo;
51 uint64_t hi;
52 } intrmap_rte_t;
53
54 #define IRTE_HIGH(sid_svt_sq) (sid_svt_sq)
55 #define IRTE_LOW(dst, vector, dlm, tm, rh, dm, fpd, p) \
56 (((uint64_t)(dst) << 32) | \
57 ((uint64_t)(vector) << 16) | \
58 ((uint64_t)(dlm) << 5) | \
59 ((uint64_t)(tm) << 4) | \
60 ((uint64_t)(rh) << 3) | \
61 ((uint64_t)(dm) << 2) | \
62 ((uint64_t)(fpd) << 1) | \
63 (p))
64
65 typedef enum {
66 SVT_NO_VERIFY = 0, /* no verification */
67 SVT_ALL_VERIFY, /* using sid and sq to verify */
68 SVT_BUS_VERIFY, /* verify #startbus and #endbus */
69 SVT_RSVD
70 } intrmap_svt_t;
71
72 typedef enum {
73 SQ_VERIFY_ALL = 0, /* verify all 16 bits */
74 SQ_VERIFY_IGR_1, /* ignore bit 3 */
75 SQ_VERIFY_IGR_2, /* ignore bit 2-3 */
76 SQ_VERIFY_IGR_3 /* ignore bit 1-3 */
77 } intrmap_sq_t;
78
79 /*
80 * S field of the Interrupt Remapping Table Address Register
81 * the size of the interrupt remapping table is 1 << (immu_intrmap_irta_s + 1)
82 */
83 static uint_t intrmap_irta_s = INTRMAP_MAX_IRTA_SIZE;
84
85 /*
86 * If true, arrange to suppress broadcast EOI by setting edge-triggered mode
87 * even for level-triggered interrupts in the interrupt-remapping engine.
88 * If false, broadcast EOI can still be suppressed if the CPU supports the
89 * APIC_SVR_SUPPRESS_BROADCAST_EOI bit. In both cases, the IOAPIC is still
90 * programmed with the correct trigger mode, and pcplusmp must send an EOI
91 * to the IOAPIC by writing to the IOAPIC's EOI register to make up for the
92 * missing broadcast EOI.
93 */
94 static int intrmap_suppress_brdcst_eoi = 0;
95
96 /*
97 * whether verify the source id of interrupt request
98 */
99 static int intrmap_enable_sid_verify = 0;
100
101 /* fault types for DVMA remapping */
102 static char *immu_dvma_faults[] = {
103 "Reserved",
104 "The present field in root-entry is Clear",
105 "The present field in context-entry is Clear",
106 "Hardware detected invalid programming of a context-entry",
107 "The DMA request attempted to access an address beyond max support",
108 "The Write field in a page-table entry is Clear when DMA write",
109 "The Read field in a page-table entry is Clear when DMA read",
110 "Access the next level page table resulted in error",
111 "Access the root-entry table resulted in error",
112 "Access the context-entry table resulted in error",
113 "Reserved field not initialized to zero in a present root-entry",
114 "Reserved field not initialized to zero in a present context-entry",
115 "Reserved field not initialized to zero in a present page-table entry",
116 "DMA blocked due to the Translation Type field in context-entry",
117 "Incorrect fault event reason number",
118 };
119 #define DVMA_MAX_FAULTS (sizeof (immu_dvma_faults)/(sizeof (char *))) - 1
120
121 /* fault types for interrupt remapping */
122 static char *immu_intrmap_faults[] = {
123 "reserved field set in IRTE",
124 "interrupt_index exceed the intr-remap table size",
125 "present field in IRTE is clear",
126 "hardware access intr-remap table address resulted in error",
127 "reserved field set in IRTE, include various conditional",
128 "hardware blocked an interrupt request in Compatibility format",
129 "remappable interrupt request blocked due to verification failure"
130 };
131 #define INTRMAP_MAX_FAULTS \
132 (sizeof (immu_intrmap_faults) / (sizeof (char *))) - 1
133
134 /* Function prototypes */
135 static int immu_intrmap_init(int apic_mode);
136 static void immu_intrmap_switchon(int suppress_brdcst_eoi);
137 static void immu_intrmap_alloc(void **intrmap_private_tbl, dev_info_t *dip,
138 uint16_t type, int count, uchar_t ioapic_index);
139 static void immu_intrmap_map(void *intrmap_private, void *intrmap_data,
140 uint16_t type, int count);
141 static void immu_intrmap_free(void **intrmap_privatep);
142 static void immu_intrmap_rdt(void *intrmap_private, ioapic_rdt_t *irdt);
143 static void immu_intrmap_msi(void *intrmap_private, msi_regs_t *mregs);
144
145 static struct apic_intrmap_ops intrmap_ops = {
146 immu_intrmap_init,
147 immu_intrmap_switchon,
148 immu_intrmap_alloc,
149 immu_intrmap_map,
150 immu_intrmap_free,
151 immu_intrmap_rdt,
152 immu_intrmap_msi,
153 };
154
155 /* apic mode, APIC/X2APIC */
156 static int intrmap_apic_mode = LOCAL_APIC;
157
158
159 /*
160 * helper functions
161 */
162 static uint_t
163 bitset_find_free(bitset_t *b, uint_t post)
164 {
165 uint_t i;
166 uint_t cap = bitset_capacity(b);
167
168 if (post == cap)
169 post = 0;
170
171 ASSERT(post < cap);
172
173 for (i = post; i < cap; i++) {
174 if (!bitset_in_set(b, i))
175 return (i);
176 }
177
178 for (i = 0; i < post; i++) {
179 if (!bitset_in_set(b, i))
180 return (i);
181 }
182
183 return (INTRMAP_IDX_FULL); /* no free index */
184 }
185
186 /*
187 * helper function to find 'count' contigous free
188 * interrupt remapping table entries
189 */
190 static uint_t
191 bitset_find_multi_free(bitset_t *b, uint_t post, uint_t count)
192 {
193 uint_t i, j;
194 uint_t cap = bitset_capacity(b);
195
196 if (post == INTRMAP_IDX_FULL) {
197 return (INTRMAP_IDX_FULL);
198 }
199
200 if (count > cap)
201 return (INTRMAP_IDX_FULL);
202
203 ASSERT(post < cap);
204
205 for (i = post; (i + count) <= cap; i++) {
206 for (j = 0; j < count; j++) {
207 if (bitset_in_set(b, (i + j))) {
208 i = i + j;
209 break;
210 }
211 if (j == count - 1)
212 return (i);
213 }
214 }
215
216 for (i = 0; (i < post) && ((i + count) <= cap); i++) {
217 for (j = 0; j < count; j++) {
218 if (bitset_in_set(b, (i + j))) {
219 i = i + j;
220 break;
221 }
222 if (j == count - 1)
223 return (i);
224 }
225 }
226
227 return (INTRMAP_IDX_FULL); /* no free index */
228 }
229
230 /* alloc one interrupt remapping table entry */
231 static int
232 alloc_tbl_entry(intrmap_t *intrmap)
233 {
234 uint32_t idx;
235
236 for (;;) {
237 mutex_enter(&intrmap->intrmap_lock);
238 idx = intrmap->intrmap_free;
239 if (idx != INTRMAP_IDX_FULL) {
240 bitset_add(&intrmap->intrmap_map, idx);
241 intrmap->intrmap_free =
242 bitset_find_free(&intrmap->intrmap_map, idx + 1);
243 mutex_exit(&intrmap->intrmap_lock);
244 break;
245 }
246
247 /* no free intr entry, use compatible format intr */
248 mutex_exit(&intrmap->intrmap_lock);
249
250 if (intrmap_apic_mode != LOCAL_X2APIC) {
251 break;
252 }
253
254 /*
255 * x2apic mode not allowed compatible
256 * interrupt
257 */
258 delay(IMMU_ALLOC_RESOURCE_DELAY);
259 }
260
261 return (idx);
262 }
263
264 /* alloc 'cnt' contigous interrupt remapping table entries */
265 static int
266 alloc_tbl_multi_entries(intrmap_t *intrmap, uint_t cnt)
267 {
268 uint_t idx, pos, i;
269
270 for (; ; ) {
271 mutex_enter(&intrmap->intrmap_lock);
272 pos = intrmap->intrmap_free;
273 idx = bitset_find_multi_free(&intrmap->intrmap_map, pos, cnt);
274
275 if (idx != INTRMAP_IDX_FULL) {
276 if (idx <= pos && pos < (idx + cnt)) {
277 intrmap->intrmap_free = bitset_find_free(
278 &intrmap->intrmap_map, idx + cnt);
279 }
280 for (i = 0; i < cnt; i++) {
281 bitset_add(&intrmap->intrmap_map, idx + i);
282 }
283 mutex_exit(&intrmap->intrmap_lock);
284 break;
285 }
286
287 mutex_exit(&intrmap->intrmap_lock);
288
289 if (intrmap_apic_mode != LOCAL_X2APIC) {
290 break;
291 }
292
293 /* x2apic mode not allowed comapitible interrupt */
294 delay(IMMU_ALLOC_RESOURCE_DELAY);
295 }
296
297 return (idx);
298 }
299
300 /* init interrupt remapping table */
301 static int
302 init_unit(immu_t *immu)
303 {
304 intrmap_t *intrmap;
305 size_t size;
306
307 ddi_dma_attr_t intrmap_dma_attr = {
308 DMA_ATTR_V0,
309 0U,
310 0xffffffffffffffffULL,
311 0xffffffffU,
312 MMU_PAGESIZE, /* page aligned */
313 0x1,
314 0x1,
315 0xffffffffU,
316 0xffffffffffffffffULL,
317 1,
318 4,
319 0
320 };
321
322 ddi_device_acc_attr_t intrmap_acc_attr = {
323 DDI_DEVICE_ATTR_V0,
324 DDI_NEVERSWAP_ACC,
325 DDI_STRICTORDER_ACC
326 };
327
328 /*
329 * Using interrupt remapping implies using the queue
330 * invalidation interface. According to Intel,
331 * hardware that supports interrupt remapping should
332 * also support QI.
333 */
334 ASSERT(IMMU_ECAP_GET_QI(immu->immu_regs_excap));
335
336 if (intrmap_apic_mode == LOCAL_X2APIC) {
337 if (!IMMU_ECAP_GET_EIM(immu->immu_regs_excap)) {
338 return (DDI_FAILURE);
339 }
340 }
341
342 if (intrmap_irta_s > INTRMAP_MAX_IRTA_SIZE) {
343 intrmap_irta_s = INTRMAP_MAX_IRTA_SIZE;
344 }
345
346 intrmap = kmem_zalloc(sizeof (intrmap_t), KM_SLEEP);
347
348 if (ddi_dma_alloc_handle(immu->immu_dip,
349 &intrmap_dma_attr,
350 DDI_DMA_SLEEP,
351 NULL,
352 &(intrmap->intrmap_dma_hdl)) != DDI_SUCCESS) {
353 kmem_free(intrmap, sizeof (intrmap_t));
354 return (DDI_FAILURE);
355 }
356
357 intrmap->intrmap_size = 1 << (intrmap_irta_s + 1);
358 size = intrmap->intrmap_size * INTRMAP_RTE_SIZE;
359 if (ddi_dma_mem_alloc(intrmap->intrmap_dma_hdl,
360 size,
361 &intrmap_acc_attr,
362 DDI_DMA_CONSISTENT | IOMEM_DATA_UNCACHED,
363 DDI_DMA_SLEEP,
364 NULL,
365 &(intrmap->intrmap_vaddr),
366 &size,
367 &(intrmap->intrmap_acc_hdl)) != DDI_SUCCESS) {
368 ddi_dma_free_handle(&(intrmap->intrmap_dma_hdl));
369 kmem_free(intrmap, sizeof (intrmap_t));
370 return (DDI_FAILURE);
371 }
372
373 ASSERT(!((uintptr_t)intrmap->intrmap_vaddr & MMU_PAGEOFFSET));
374 bzero(intrmap->intrmap_vaddr, size);
375 intrmap->intrmap_paddr = pfn_to_pa(
376 hat_getpfnum(kas.a_hat, intrmap->intrmap_vaddr));
377
378 mutex_init(&(intrmap->intrmap_lock), NULL, MUTEX_DRIVER, NULL);
379 bitset_init(&intrmap->intrmap_map);
380 bitset_resize(&intrmap->intrmap_map, intrmap->intrmap_size);
381 intrmap->intrmap_free = 0;
382
383 immu->immu_intrmap = intrmap;
384
385 return (DDI_SUCCESS);
386 }
387
388 static immu_t *
389 get_immu(dev_info_t *dip, uint16_t type, uchar_t ioapic_index)
390 {
391 immu_t *immu = NULL;
392
393 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
394 immu = immu_dmar_ioapic_immu(ioapic_index);
395 } else {
396 if (dip != NULL)
397 immu = immu_dmar_get_immu(dip);
398 }
399
400 return (immu);
401 }
402
403 static int
404 get_top_pcibridge(dev_info_t *dip, void *arg)
405 {
406 dev_info_t **topdipp = arg;
407 immu_devi_t *immu_devi;
408
409 mutex_enter(&(DEVI(dip)->devi_lock));
410 immu_devi = DEVI(dip)->devi_iommu;
411 mutex_exit(&(DEVI(dip)->devi_lock));
412
413 if (immu_devi == NULL || immu_devi->imd_pcib_type == IMMU_PCIB_BAD ||
414 immu_devi->imd_pcib_type == IMMU_PCIB_ENDPOINT) {
415 return (DDI_WALK_CONTINUE);
416 }
417
418 *topdipp = dip;
419
420 return (DDI_WALK_CONTINUE);
421 }
422
423 static dev_info_t *
424 intrmap_top_pcibridge(dev_info_t *rdip)
425 {
426 dev_info_t *top_pcibridge = NULL;
427
428 if (immu_walk_ancestor(rdip, NULL, get_top_pcibridge,
429 &top_pcibridge, NULL, 0) != DDI_SUCCESS) {
430 return (NULL);
431 }
432
433 return (top_pcibridge);
434 }
435
436 /* function to get interrupt request source id */
437 static uint32_t
438 get_sid(dev_info_t *dip, uint16_t type, uchar_t ioapic_index)
439 {
440 dev_info_t *pdip;
441 immu_devi_t *immu_devi;
442 uint16_t sid;
443 uchar_t svt, sq;
444
445 if (!intrmap_enable_sid_verify) {
446 return (0);
447 }
448
449 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
450 /* for interrupt through I/O APIC */
451 sid = immu_dmar_ioapic_sid(ioapic_index);
452 svt = SVT_ALL_VERIFY;
453 sq = SQ_VERIFY_ALL;
454 } else {
455 /* MSI/MSI-X interrupt */
456 ASSERT(dip);
457 pdip = intrmap_top_pcibridge(dip);
458 ASSERT(pdip);
459 immu_devi = DEVI(pdip)->devi_iommu;
460 ASSERT(immu_devi);
461 if (immu_devi->imd_pcib_type == IMMU_PCIB_PCIE_PCI) {
462 /* device behind pcie to pci bridge */
463 sid = (immu_devi->imd_bus << 8) | immu_devi->imd_sec;
464 svt = SVT_BUS_VERIFY;
465 sq = SQ_VERIFY_ALL;
466 } else {
467 /* pcie device or device behind pci to pci bridge */
468 sid = (immu_devi->imd_bus << 8) |
469 immu_devi->imd_devfunc;
470 svt = SVT_ALL_VERIFY;
471 sq = SQ_VERIFY_ALL;
472 }
473 }
474
475 return (sid | (svt << 18) | (sq << 16));
476 }
477
478 static void
479 intrmap_enable(immu_t *immu)
480 {
481 intrmap_t *intrmap;
482 uint64_t irta_reg;
483
484 intrmap = immu->immu_intrmap;
485
486 irta_reg = intrmap->intrmap_paddr | intrmap_irta_s;
487 if (intrmap_apic_mode == LOCAL_X2APIC) {
488 irta_reg |= (0x1 << 11);
489 }
490
491 immu_regs_intrmap_enable(immu, irta_reg);
492 }
493
494 /* ####################################################################### */
495
496 /*
497 * immu_intr_handler()
498 * the fault event handler for a single immu unit
499 */
500 int
501 immu_intr_handler(immu_t *immu)
502 {
503 uint32_t status;
504 int index, fault_reg_offset;
505 int max_fault_index;
506 boolean_t found_fault;
507 dev_info_t *idip;
508
509 mutex_enter(&(immu->immu_intr_lock));
510 mutex_enter(&(immu->immu_regs_lock));
511
512 /* read the fault status */
513 status = immu_regs_get32(immu, IMMU_REG_FAULT_STS);
514
515 idip = immu->immu_dip;
516 ASSERT(idip);
517
518 /* check if we have a pending fault for this immu unit */
519 if ((status & IMMU_FAULT_STS_PPF) == 0) {
520 mutex_exit(&(immu->immu_regs_lock));
521 mutex_exit(&(immu->immu_intr_lock));
522 return (DDI_INTR_UNCLAIMED);
523 }
524
525 /*
526 * handle all primary pending faults
527 */
528 index = IMMU_FAULT_GET_INDEX(status);
529 max_fault_index = IMMU_CAP_GET_NFR(immu->immu_regs_cap) - 1;
530 fault_reg_offset = IMMU_CAP_GET_FRO(immu->immu_regs_cap);
531
532 found_fault = B_FALSE;
533 _NOTE(CONSTCOND)
534 while (1) {
535 uint64_t val;
536 uint8_t fault_reason;
537 uint8_t fault_type;
538 uint16_t sid;
539 uint64_t pg_addr;
540 uint64_t idx;
541
542 /* read the higher 64bits */
543 val = immu_regs_get64(immu, fault_reg_offset + index * 16 + 8);
544
545 /* check if this fault register has pending fault */
546 if (!IMMU_FRR_GET_F(val)) {
547 break;
548 }
549
550 found_fault = B_TRUE;
551
552 /* get the fault reason, fault type and sid */
553 fault_reason = IMMU_FRR_GET_FR(val);
554 fault_type = IMMU_FRR_GET_FT(val);
555 sid = IMMU_FRR_GET_SID(val);
556
557 /* read the first 64bits */
558 val = immu_regs_get64(immu, fault_reg_offset + index * 16);
559 pg_addr = val & IMMU_PAGEMASK;
560 idx = val >> 48;
561
562 /* clear the fault */
563 immu_regs_put32(immu, fault_reg_offset + index * 16 + 12,
564 (((uint32_t)1) << 31));
565
566 /* report the fault info */
567 if (fault_reason < 0x20) {
568 /* immu-remapping fault */
569 ddi_err(DER_WARN, idip,
570 "generated a fault event when translating DMA %s\n"
571 "\t on address 0x%" PRIx64 " for PCI(%d, %d, %d), "
572 "the reason is:\n\t %s",
573 fault_type ? "read" : "write", pg_addr,
574 (sid >> 8) & 0xff, (sid >> 3) & 0x1f, sid & 0x7,
575 immu_dvma_faults[MIN(fault_reason,
576 DVMA_MAX_FAULTS)]);
577 immu_print_fault_info(sid, pg_addr);
578 } else if (fault_reason < 0x27) {
579 /* intr-remapping fault */
580 ddi_err(DER_WARN, idip,
581 "generated a fault event when translating "
582 "interrupt request\n"
583 "\t on index 0x%" PRIx64 " for PCI(%d, %d, %d), "
584 "the reason is:\n\t %s",
585 idx,
586 (sid >> 8) & 0xff, (sid >> 3) & 0x1f, sid & 0x7,
587 immu_intrmap_faults[MIN((fault_reason - 0x20),
588 INTRMAP_MAX_FAULTS)]);
589 } else {
590 ddi_err(DER_WARN, idip, "Unknown fault reason: 0x%x",
591 fault_reason);
592 }
593
594 index++;
595 if (index > max_fault_index)
596 index = 0;
597 }
598
599 /* Clear the fault */
600 if (!found_fault) {
601 ddi_err(DER_MODE, idip,
602 "Fault register set but no fault present");
603 }
604 immu_regs_put32(immu, IMMU_REG_FAULT_STS, 1);
605 mutex_exit(&(immu->immu_regs_lock));
606 mutex_exit(&(immu->immu_intr_lock));
607 return (DDI_INTR_CLAIMED);
608 }
609 /* ######################################################################### */
610
611 /*
612 * Interrupt remap entry points
613 */
614
615 /* initialize interrupt remapping */
616 static int
617 immu_intrmap_init(int apic_mode)
618 {
619 immu_t *immu;
620 int error = DDI_FAILURE;
621
622 if (immu_intrmap_enable == B_FALSE) {
623 return (DDI_SUCCESS);
624 }
625
626 intrmap_apic_mode = apic_mode;
627
628 immu = list_head(&immu_list);
629 for (; immu; immu = list_next(&immu_list, immu)) {
630 if ((immu->immu_intrmap_running == B_TRUE) &&
631 IMMU_ECAP_GET_IR(immu->immu_regs_excap)) {
632 if (init_unit(immu) == DDI_SUCCESS) {
633 error = DDI_SUCCESS;
634 }
635 }
636 }
637
638 /*
639 * if all IOMMU units disable intr remapping,
640 * return FAILURE
641 */
642 return (error);
643 }
644
645
646
647 /* enable interrupt remapping */
648 static void
649 immu_intrmap_switchon(int suppress_brdcst_eoi)
650 {
651 immu_t *immu;
652
653
654 intrmap_suppress_brdcst_eoi = suppress_brdcst_eoi;
655
656 immu = list_head(&immu_list);
657 for (; immu; immu = list_next(&immu_list, immu)) {
658 if (immu->immu_intrmap_setup == B_TRUE) {
659 intrmap_enable(immu);
660 }
661 }
662 }
663
664 /* alloc remapping entry for the interrupt */
665 static void
666 immu_intrmap_alloc(void **intrmap_private_tbl, dev_info_t *dip,
667 uint16_t type, int count, uchar_t ioapic_index)
668 {
669 immu_t *immu;
670 intrmap_t *intrmap;
671 immu_inv_wait_t *iwp;
672 uint32_t idx, i;
673 uint32_t sid_svt_sq;
674 intrmap_private_t *intrmap_private;
675
676 if (intrmap_private_tbl[0] == INTRMAP_DISABLE ||
677 intrmap_private_tbl[0] != NULL) {
678 return;
679 }
680
681 intrmap_private_tbl[0] =
682 kmem_zalloc(sizeof (intrmap_private_t), KM_SLEEP);
683 intrmap_private = INTRMAP_PRIVATE(intrmap_private_tbl[0]);
684
685 immu = get_immu(dip, type, ioapic_index);
686 if ((immu != NULL) && (immu->immu_intrmap_running == B_TRUE)) {
687 intrmap_private->ir_immu = immu;
688 } else {
689 goto intrmap_disable;
690 }
691
692 intrmap = immu->immu_intrmap;
693
694 if (count == 1) {
695 idx = alloc_tbl_entry(intrmap);
696 } else {
697 idx = alloc_tbl_multi_entries(intrmap, count);
698 }
699
700 if (idx == INTRMAP_IDX_FULL) {
701 goto intrmap_disable;
702 }
703
704 intrmap_private->ir_idx = idx;
705
706 sid_svt_sq = intrmap_private->ir_sid_svt_sq =
707 get_sid(dip, type, ioapic_index);
708 iwp = &intrmap_private->ir_inv_wait;
709 immu_init_inv_wait(iwp, "intrmaplocal", B_TRUE);
710
711 if (count == 1) {
712 if (IMMU_CAP_GET_CM(immu->immu_regs_cap)) {
713 immu_qinv_intr_one_cache(immu, idx, iwp);
714 } else {
715 immu_regs_wbf_flush(immu);
716 }
717 return;
718 }
719
720 for (i = 1; i < count; i++) {
721 intrmap_private_tbl[i] =
722 kmem_zalloc(sizeof (intrmap_private_t), KM_SLEEP);
723
724 INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_immu = immu;
725 INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_sid_svt_sq =
726 sid_svt_sq;
727 INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_idx = idx + i;
728 }
729
730 if (IMMU_CAP_GET_CM(immu->immu_regs_cap)) {
731 immu_qinv_intr_caches(immu, idx, count, iwp);
732 } else {
733 immu_regs_wbf_flush(immu);
734 }
735
736 return;
737
738 intrmap_disable:
739 kmem_free(intrmap_private_tbl[0], sizeof (intrmap_private_t));
740 intrmap_private_tbl[0] = INTRMAP_DISABLE;
741 }
742
743
744 /* remapping the interrupt */
745 static void
746 immu_intrmap_map(void *intrmap_private, void *intrmap_data, uint16_t type,
747 int count)
748 {
749 immu_t *immu;
750 immu_inv_wait_t *iwp;
751 intrmap_t *intrmap;
752 ioapic_rdt_t *irdt = (ioapic_rdt_t *)intrmap_data;
753 msi_regs_t *mregs = (msi_regs_t *)intrmap_data;
754 intrmap_rte_t irte;
755 uint_t idx, i;
756 uint32_t dst, sid_svt_sq;
757 uchar_t vector, dlm, tm, rh, dm;
758
759 if (intrmap_private == INTRMAP_DISABLE)
760 return;
761
762 idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
763 immu = INTRMAP_PRIVATE(intrmap_private)->ir_immu;
764 iwp = &INTRMAP_PRIVATE(intrmap_private)->ir_inv_wait;
765 intrmap = immu->immu_intrmap;
766 sid_svt_sq = INTRMAP_PRIVATE(intrmap_private)->ir_sid_svt_sq;
767
768 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
769 dm = RDT_DM(irdt->ir_lo);
770 rh = 0;
771 tm = RDT_TM(irdt->ir_lo);
772 dlm = RDT_DLM(irdt->ir_lo);
773 dst = irdt->ir_hi;
774
775 /*
776 * Mark the IRTE's TM as Edge to suppress broadcast EOI.
777 */
778 if (intrmap_suppress_brdcst_eoi) {
779 tm = TRIGGER_MODE_EDGE;
780 }
781
782 vector = RDT_VECTOR(irdt->ir_lo);
783 } else {
784 dm = MSI_ADDR_DM_PHYSICAL;
785 rh = MSI_ADDR_RH_FIXED;
786 tm = TRIGGER_MODE_EDGE;
787 dlm = 0;
788 dst = mregs->mr_addr;
789
790 vector = mregs->mr_data & 0xff;
791 }
792
793 if (intrmap_apic_mode == LOCAL_APIC)
794 dst = (dst & 0xFF) << 8;
795
796 if (count == 1) {
797 irte.lo = IRTE_LOW(dst, vector, dlm, tm, rh, dm, 0, 1);
798 irte.hi = IRTE_HIGH(sid_svt_sq);
799
800 /* set interrupt remapping table entry */
801 bcopy(&irte, intrmap->intrmap_vaddr +
802 idx * INTRMAP_RTE_SIZE,
803 INTRMAP_RTE_SIZE);
804
805 immu_qinv_intr_one_cache(immu, idx, iwp);
806
807 } else {
808 for (i = 0; i < count; i++) {
809 irte.lo = IRTE_LOW(dst, vector, dlm, tm, rh, dm, 0, 1);
810 irte.hi = IRTE_HIGH(sid_svt_sq);
811
812 /* set interrupt remapping table entry */
813 bcopy(&irte, intrmap->intrmap_vaddr +
814 idx * INTRMAP_RTE_SIZE,
815 INTRMAP_RTE_SIZE);
816 vector++;
817 idx++;
818 }
819
820 immu_qinv_intr_caches(immu, idx, count, iwp);
821 }
822 }
823
824 /* free the remapping entry */
825 static void
826 immu_intrmap_free(void **intrmap_privatep)
827 {
828 immu_t *immu;
829 immu_inv_wait_t *iwp;
830 intrmap_t *intrmap;
831 uint32_t idx;
832
833 if (*intrmap_privatep == INTRMAP_DISABLE || *intrmap_privatep == NULL) {
834 *intrmap_privatep = NULL;
835 return;
836 }
837
838 immu = INTRMAP_PRIVATE(*intrmap_privatep)->ir_immu;
839 iwp = &INTRMAP_PRIVATE(*intrmap_privatep)->ir_inv_wait;
840 intrmap = immu->immu_intrmap;
841 idx = INTRMAP_PRIVATE(*intrmap_privatep)->ir_idx;
842
843 bzero(intrmap->intrmap_vaddr + idx * INTRMAP_RTE_SIZE,
844 INTRMAP_RTE_SIZE);
845
846 immu_qinv_intr_one_cache(immu, idx, iwp);
847
848 mutex_enter(&intrmap->intrmap_lock);
849 bitset_del(&intrmap->intrmap_map, idx);
850 if (intrmap->intrmap_free == INTRMAP_IDX_FULL) {
851 intrmap->intrmap_free = idx;
852 }
853 mutex_exit(&intrmap->intrmap_lock);
854
855 kmem_free(*intrmap_privatep, sizeof (intrmap_private_t));
856 *intrmap_privatep = NULL;
857 }
858
859 /* record the ioapic rdt entry */
860 static void
861 immu_intrmap_rdt(void *intrmap_private, ioapic_rdt_t *irdt)
862 {
863 uint32_t rdt_entry, tm, pol, idx, vector;
864
865 rdt_entry = irdt->ir_lo;
866
867 if (intrmap_private != INTRMAP_DISABLE && intrmap_private != NULL) {
868 idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
869 tm = RDT_TM(rdt_entry);
870 pol = RDT_POL(rdt_entry);
871 vector = RDT_VECTOR(rdt_entry);
872 irdt->ir_lo = (tm << INTRMAP_IOAPIC_TM_SHIFT) |
873 (pol << INTRMAP_IOAPIC_POL_SHIFT) |
874 ((idx >> 15) << INTRMAP_IOAPIC_IDX15_SHIFT) |
875 vector;
876 irdt->ir_hi = (idx << INTRMAP_IOAPIC_IDX_SHIFT) |
877 (1 << INTRMAP_IOAPIC_FORMAT_SHIFT);
878 } else {
879 irdt->ir_hi <<= APIC_ID_BIT_OFFSET;
880 }
881 }
882
883 /* record the msi interrupt structure */
884 /*ARGSUSED*/
885 static void
886 immu_intrmap_msi(void *intrmap_private, msi_regs_t *mregs)
887 {
888 uint_t idx;
889
890 if (intrmap_private != INTRMAP_DISABLE && intrmap_private != NULL) {
891 idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
892
893 mregs->mr_data = 0;
894 mregs->mr_addr = MSI_ADDR_HDR |
895 ((idx & 0x7fff) << INTRMAP_MSI_IDX_SHIFT) |
896 (1 << INTRMAP_MSI_FORMAT_SHIFT) |
897 (1 << INTRMAP_MSI_SHV_SHIFT) |
898 ((idx >> 15) << INTRMAP_MSI_IDX15_SHIFT);
899 } else {
900 mregs->mr_addr = MSI_ADDR_HDR |
901 (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
902 (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT) |
903 (mregs->mr_addr << MSI_ADDR_DEST_SHIFT);
904 mregs->mr_data = (MSI_DATA_TM_EDGE << MSI_DATA_TM_SHIFT) |
905 mregs->mr_data;
906 }
907 }
908
909 /* ######################################################################### */
910 /*
911 * Functions exported by immu_intr.c
912 */
913 void
914 immu_intrmap_setup(list_t *listp)
915 {
916 immu_t *immu;
917
918 /*
919 * Check if ACPI DMAR tables say that
920 * interrupt remapping is supported
921 */
922 if (immu_dmar_intrmap_supported() == B_FALSE) {
923 return;
924 }
925
926 /*
927 * Check if interrupt remapping is disabled.
928 */
929 if (immu_intrmap_enable == B_FALSE) {
930 return;
931 }
932
933 psm_vt_ops = &intrmap_ops;
934
935 immu = list_head(listp);
936 for (; immu; immu = list_next(listp, immu)) {
937 mutex_init(&(immu->immu_intrmap_lock), NULL,
938 MUTEX_DEFAULT, NULL);
939 mutex_enter(&(immu->immu_intrmap_lock));
940 immu_init_inv_wait(&immu->immu_intrmap_inv_wait,
941 "intrmapglobal", B_TRUE);
942 immu->immu_intrmap_setup = B_TRUE;
943 mutex_exit(&(immu->immu_intrmap_lock));
944 }
945 }
946
947 void
948 immu_intrmap_startup(immu_t *immu)
949 {
950 /* do nothing */
951 mutex_enter(&(immu->immu_intrmap_lock));
952 if (immu->immu_intrmap_setup == B_TRUE) {
953 immu->immu_intrmap_running = B_TRUE;
954 }
955 mutex_exit(&(immu->immu_intrmap_lock));
956 }
957
958 /*
959 * Register a Intel IOMMU unit (i.e. DMAR unit's)
960 * interrupt handler
961 */
962 void
963 immu_intr_register(immu_t *immu)
964 {
965 int irq, vect;
966 char intr_handler_name[IMMU_MAXNAMELEN];
967 uint32_t msi_data;
968 uint32_t uaddr;
969 uint32_t msi_addr;
970 uint32_t localapic_id = 0;
971
972 if (psm_get_localapicid)
973 localapic_id = psm_get_localapicid(0);
974
975 msi_addr = (MSI_ADDR_HDR |
976 ((localapic_id & 0xFF) << MSI_ADDR_DEST_SHIFT) |
977 (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
978 (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT));
979
980 if (intrmap_apic_mode == LOCAL_X2APIC) {
981 uaddr = localapic_id & 0xFFFFFF00;
982 } else {
983 uaddr = 0;
984 }
985
986 /* Dont need to hold immu_intr_lock since we are in boot */
987 irq = vect = psm_get_ipivect(IMMU_INTR_IPL, -1);
988 if (psm_xlate_vector_by_irq != NULL)
989 vect = psm_xlate_vector_by_irq(irq);
990
991 msi_data = ((MSI_DATA_DELIVERY_FIXED <<
992 MSI_DATA_DELIVERY_SHIFT) | vect);
993
994 (void) snprintf(intr_handler_name, sizeof (intr_handler_name),
995 "%s-intr-handler", immu->immu_name);
996
997 (void) add_avintr(NULL, IMMU_INTR_IPL,
998 (avfunc)(immu_intr_handler), intr_handler_name, irq,
999 (caddr_t)immu, NULL, NULL, NULL);
1000
1001 immu_regs_intr_enable(immu, msi_addr, msi_data, uaddr);
1002
1003 (void) immu_intr_handler(immu);
1004 }
unleashed repo fork