?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'akpm'
[linux-2.6/next.git] / drivers / iommu / dmar.c
blobf0db877d71e6043133a373f7776a27292f10676c
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 *
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */
28
29 #include <linux/pci.h>
30 #include <linux/dmar.h>
31 #include <linux/iova.h>
32 #include <linux/intel-iommu.h>
33 #include <linux/timer.h>
34 #include <linux/irq.h>
35 #include <linux/interrupt.h>
36 #include <linux/tboot.h>
37 #include <linux/dmi.h>
38 #include <linux/slab.h>
39 #include <asm/iommu_table.h>
40
41 #define PREFIX "DMAR: "
42
43 /* No locks are needed as DMA remapping hardware unit
44 * list is constructed at boot time and hotplug of
45 * these units are not supported by the architecture.
46 */
47 LIST_HEAD(dmar_drhd_units);
48
49 static struct acpi_table_header * __initdata dmar_tbl;
50 static acpi_size dmar_tbl_size;
51
52 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
53 {
54 /*
55 * add INCLUDE_ALL at the tail, so scan the list will find it at
56 * the very end.
57 */
58 if (drhd->include_all)
59 list_add_tail(&drhd->list, &dmar_drhd_units);
60 else
61 list_add(&drhd->list, &dmar_drhd_units);
62 }
63
64 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
65 struct pci_dev **dev, u16 segment)
66 {
67 struct pci_bus *bus;
68 struct pci_dev *pdev = NULL;
69 struct acpi_dmar_pci_path *path;
70 int count;
71
72 bus = pci_find_bus(segment, scope->bus);
73 path = (struct acpi_dmar_pci_path *)(scope + 1);
74 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
75 / sizeof(struct acpi_dmar_pci_path);
76
77 while (count) {
78 if (pdev)
79 pci_dev_put(pdev);
80 /*
81 * Some BIOSes list non-exist devices in DMAR table, just
82 * ignore it
83 */
84 if (!bus) {
85 printk(KERN_WARNING
86 PREFIX "Device scope bus [%d] not found\n",
87 scope->bus);
88 break;
89 }
90 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
91 if (!pdev) {
92 printk(KERN_WARNING PREFIX
93 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
94 segment, bus->number, path->dev, path->fn);
95 break;
96 }
97 path ++;
98 count --;
99 bus = pdev->subordinate;
100 }
101 if (!pdev) {
102 printk(KERN_WARNING PREFIX
103 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
104 segment, scope->bus, path->dev, path->fn);
105 *dev = NULL;
106 return 0;
107 }
108 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
109 pdev->subordinate) || (scope->entry_type == \
110 ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
111 pci_dev_put(pdev);
112 printk(KERN_WARNING PREFIX
113 "Device scope type does not match for %s\n",
114 pci_name(pdev));
115 return -EINVAL;
116 }
117 *dev = pdev;
118 return 0;
119 }
120
121 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
122 struct pci_dev ***devices, u16 segment)
123 {
124 struct acpi_dmar_device_scope *scope;
125 void * tmp = start;
126 int index;
127 int ret;
128
129 *cnt = 0;
130 while (start < end) {
131 scope = start;
132 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
133 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
134 (*cnt)++;
135 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
136 printk(KERN_WARNING PREFIX
137 "Unsupported device scope\n");
138 }
139 start += scope->length;
140 }
141 if (*cnt == 0)
142 return 0;
143
144 *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
145 if (!*devices)
146 return -ENOMEM;
147
148 start = tmp;
149 index = 0;
150 while (start < end) {
151 scope = start;
152 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
153 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
154 ret = dmar_parse_one_dev_scope(scope,
155 &(*devices)[index], segment);
156 if (ret) {
157 kfree(*devices);
158 return ret;
159 }
160 index ++;
161 }
162 start += scope->length;
163 }
164
165 return 0;
166 }
167
168 /**
169 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
170 * structure which uniquely represent one DMA remapping hardware unit
171 * present in the platform
172 */
173 static int __init
174 dmar_parse_one_drhd(struct acpi_dmar_header *header)
175 {
176 struct acpi_dmar_hardware_unit *drhd;
177 struct dmar_drhd_unit *dmaru;
178 int ret = 0;
179
180 drhd = (struct acpi_dmar_hardware_unit *)header;
181 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
182 if (!dmaru)
183 return -ENOMEM;
184
185 dmaru->hdr = header;
186 dmaru->reg_base_addr = drhd->address;
187 dmaru->segment = drhd->segment;
188 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
189
190 ret = alloc_iommu(dmaru);
191 if (ret) {
192 kfree(dmaru);
193 return ret;
194 }
195 dmar_register_drhd_unit(dmaru);
196 return 0;
197 }
198
199 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
200 {
201 struct acpi_dmar_hardware_unit *drhd;
202 int ret = 0;
203
204 drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
205
206 if (dmaru->include_all)
207 return 0;
208
209 ret = dmar_parse_dev_scope((void *)(drhd + 1),
210 ((void *)drhd) + drhd->header.length,
211 &dmaru->devices_cnt, &dmaru->devices,
212 drhd->segment);
213 if (ret) {
214 list_del(&dmaru->list);
215 kfree(dmaru);
216 }
217 return ret;
218 }
219
220 #ifdef CONFIG_DMAR
221 LIST_HEAD(dmar_rmrr_units);
222
223 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
224 {
225 list_add(&rmrr->list, &dmar_rmrr_units);
226 }
227
228
229 static int __init
230 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
231 {
232 struct acpi_dmar_reserved_memory *rmrr;
233 struct dmar_rmrr_unit *rmrru;
234
235 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
236 if (!rmrru)
237 return -ENOMEM;
238
239 rmrru->hdr = header;
240 rmrr = (struct acpi_dmar_reserved_memory *)header;
241 rmrru->base_address = rmrr->base_address;
242 rmrru->end_address = rmrr->end_address;
243
244 dmar_register_rmrr_unit(rmrru);
245 return 0;
246 }
247
248 static int __init
249 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
250 {
251 struct acpi_dmar_reserved_memory *rmrr;
252 int ret;
253
254 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
255 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
256 ((void *)rmrr) + rmrr->header.length,
257 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
258
259 if (ret || (rmrru->devices_cnt == 0)) {
260 list_del(&rmrru->list);
261 kfree(rmrru);
262 }
263 return ret;
264 }
265
266 LIST_HEAD(dmar_atsr_units);
267
268 static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
269 {
270 struct acpi_dmar_atsr *atsr;
271 struct dmar_atsr_unit *atsru;
272
273 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
274 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
275 if (!atsru)
276 return -ENOMEM;
277
278 atsru->hdr = hdr;
279 atsru->include_all = atsr->flags & 0x1;
280 atsru->segment = atsr->segment;
281
282 list_add(&atsru->list, &dmar_atsr_units);
283
284 return 0;
285 }
286
287 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
288 {
289 int rc;
290 struct acpi_dmar_atsr *atsr;
291
292 if (atsru->include_all)
293 return 0;
294
295 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
296 rc = dmar_parse_dev_scope((void *)(atsr + 1),
297 (void *)atsr + atsr->header.length,
298 &atsru->devices_cnt, &atsru->devices,
299 atsr->segment);
300 if (rc || !atsru->devices_cnt) {
301 list_del(&atsru->list);
302 kfree(atsru);
303 }
304
305 return rc;
306 }
307
308 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
309 {
310 int i;
311 struct pci_bus *bus;
312 struct dmar_atsr_unit *atsru;
313
314 dev = pci_physfn(dev);
315
316 list_for_each_entry(atsru, &dmar_atsr_units, list) {
317 if (atsru->segment == pci_domain_nr(dev->bus))
318 goto found;
319 }
320
321 return 0;
322
323 found:
324 for (bus = dev->bus; bus; bus = bus->parent) {
325 struct pci_dev *bridge = bus->self;
326
327 if (!bridge || !pci_is_pcie(bridge) ||
328 bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
329 return 0;
330
331 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
332 for (i = 0; i < atsru->devices_cnt; i++)
333 if (atsru->devices[i] == bridge)
334 return 1;
335 break;
336 }
337 }
338
339 if (atsru->include_all)
340 return 1;
341
342 return 0;
343 }
344 #endif
345
346 #ifdef CONFIG_ACPI_NUMA
347 static int __init
348 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
349 {
350 struct acpi_dmar_rhsa *rhsa;
351 struct dmar_drhd_unit *drhd;
352
353 rhsa = (struct acpi_dmar_rhsa *)header;
354 for_each_drhd_unit(drhd) {
355 if (drhd->reg_base_addr == rhsa->base_address) {
356 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
357
358 if (!node_online(node))
359 node = -1;
360 drhd->iommu->node = node;
361 return 0;
362 }
363 }
364 WARN_TAINT(
365 1, TAINT_FIRMWARE_WORKAROUND,
366 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
367 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
368 drhd->reg_base_addr,
369 dmi_get_system_info(DMI_BIOS_VENDOR),
370 dmi_get_system_info(DMI_BIOS_VERSION),
371 dmi_get_system_info(DMI_PRODUCT_VERSION));
372
373 return 0;
374 }
375 #endif
376
377 static void __init
378 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
379 {
380 struct acpi_dmar_hardware_unit *drhd;
381 struct acpi_dmar_reserved_memory *rmrr;
382 struct acpi_dmar_atsr *atsr;
383 struct acpi_dmar_rhsa *rhsa;
384
385 switch (header->type) {
386 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
387 drhd = container_of(header, struct acpi_dmar_hardware_unit,
388 header);
389 printk (KERN_INFO PREFIX
390 "DRHD base: %#016Lx flags: %#x\n",
391 (unsigned long long)drhd->address, drhd->flags);
392 break;
393 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
394 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
395 header);
396 printk (KERN_INFO PREFIX
397 "RMRR base: %#016Lx end: %#016Lx\n",
398 (unsigned long long)rmrr->base_address,
399 (unsigned long long)rmrr->end_address);
400 break;
401 case ACPI_DMAR_TYPE_ATSR:
402 atsr = container_of(header, struct acpi_dmar_atsr, header);
403 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
404 break;
405 case ACPI_DMAR_HARDWARE_AFFINITY:
406 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
407 printk(KERN_INFO PREFIX "RHSA base: %#016Lx proximity domain: %#x\n",
408 (unsigned long long)rhsa->base_address,
409 rhsa->proximity_domain);
410 break;
411 }
412 }
413
414 /**
415 * dmar_table_detect - checks to see if the platform supports DMAR devices
416 */
417 static int __init dmar_table_detect(void)
418 {
419 acpi_status status = AE_OK;
420
421 /* if we could find DMAR table, then there are DMAR devices */
422 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
423 (struct acpi_table_header **)&dmar_tbl,
424 &dmar_tbl_size);
425
426 if (ACPI_SUCCESS(status) && !dmar_tbl) {
427 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
428 status = AE_NOT_FOUND;
429 }
430
431 return (ACPI_SUCCESS(status) ? 1 : 0);
432 }
433
434 /**
435 * parse_dmar_table - parses the DMA reporting table
436 */
437 static int __init
438 parse_dmar_table(void)
439 {
440 struct acpi_table_dmar *dmar;
441 struct acpi_dmar_header *entry_header;
442 int ret = 0;
443
444 /*
445 * Do it again, earlier dmar_tbl mapping could be mapped with
446 * fixed map.
447 */
448 dmar_table_detect();
449
450 /*
451 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
452 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
453 */
454 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
455
456 dmar = (struct acpi_table_dmar *)dmar_tbl;
457 if (!dmar)
458 return -ENODEV;
459
460 if (dmar->width < PAGE_SHIFT - 1) {
461 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
462 return -EINVAL;
463 }
464
465 printk (KERN_INFO PREFIX "Host address width %d\n",
466 dmar->width + 1);
467
468 entry_header = (struct acpi_dmar_header *)(dmar + 1);
469 while (((unsigned long)entry_header) <
470 (((unsigned long)dmar) + dmar_tbl->length)) {
471 /* Avoid looping forever on bad ACPI tables */
472 if (entry_header->length == 0) {
473 printk(KERN_WARNING PREFIX
474 "Invalid 0-length structure\n");
475 ret = -EINVAL;
476 break;
477 }
478
479 dmar_table_print_dmar_entry(entry_header);
480
481 switch (entry_header->type) {
482 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
483 ret = dmar_parse_one_drhd(entry_header);
484 break;
485 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
486 #ifdef CONFIG_DMAR
487 ret = dmar_parse_one_rmrr(entry_header);
488 #endif
489 break;
490 case ACPI_DMAR_TYPE_ATSR:
491 #ifdef CONFIG_DMAR
492 ret = dmar_parse_one_atsr(entry_header);
493 #endif
494 break;
495 case ACPI_DMAR_HARDWARE_AFFINITY:
496 #ifdef CONFIG_ACPI_NUMA
497 ret = dmar_parse_one_rhsa(entry_header);
498 #endif
499 break;
500 default:
501 printk(KERN_WARNING PREFIX
502 "Unknown DMAR structure type %d\n",
503 entry_header->type);
504 ret = 0; /* for forward compatibility */
505 break;
506 }
507 if (ret)
508 break;
509
510 entry_header = ((void *)entry_header + entry_header->length);
511 }
512 return ret;
513 }
514
515 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
516 struct pci_dev *dev)
517 {
518 int index;
519
520 while (dev) {
521 for (index = 0; index < cnt; index++)
522 if (dev == devices[index])
523 return 1;
524
525 /* Check our parent */
526 dev = dev->bus->self;
527 }
528
529 return 0;
530 }
531
532 struct dmar_drhd_unit *
533 dmar_find_matched_drhd_unit(struct pci_dev *dev)
534 {
535 struct dmar_drhd_unit *dmaru = NULL;
536 struct acpi_dmar_hardware_unit *drhd;
537
538 dev = pci_physfn(dev);
539
540 list_for_each_entry(dmaru, &dmar_drhd_units, list) {
541 drhd = container_of(dmaru->hdr,
542 struct acpi_dmar_hardware_unit,
543 header);
544
545 if (dmaru->include_all &&
546 drhd->segment == pci_domain_nr(dev->bus))
547 return dmaru;
548
549 if (dmar_pci_device_match(dmaru->devices,
550 dmaru->devices_cnt, dev))
551 return dmaru;
552 }
553
554 return NULL;
555 }
556
557 int __init dmar_dev_scope_init(void)
558 {
559 struct dmar_drhd_unit *drhd, *drhd_n;
560 int ret = -ENODEV;
561
562 list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
563 ret = dmar_parse_dev(drhd);
564 if (ret)
565 return ret;
566 }
567
568 #ifdef CONFIG_DMAR
569 {
570 struct dmar_rmrr_unit *rmrr, *rmrr_n;
571 struct dmar_atsr_unit *atsr, *atsr_n;
572
573 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
574 ret = rmrr_parse_dev(rmrr);
575 if (ret)
576 return ret;
577 }
578
579 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
580 ret = atsr_parse_dev(atsr);
581 if (ret)
582 return ret;
583 }
584 }
585 #endif
586
587 return ret;
588 }
589
590
591 int __init dmar_table_init(void)
592 {
593 static int dmar_table_initialized;
594 int ret;
595
596 if (dmar_table_initialized)
597 return 0;
598
599 dmar_table_initialized = 1;
600
601 ret = parse_dmar_table();
602 if (ret) {
603 if (ret != -ENODEV)
604 printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
605 return ret;
606 }
607
608 if (list_empty(&dmar_drhd_units)) {
609 printk(KERN_INFO PREFIX "No DMAR devices found\n");
610 return -ENODEV;
611 }
612
613 #ifdef CONFIG_DMAR
614 if (list_empty(&dmar_rmrr_units))
615 printk(KERN_INFO PREFIX "No RMRR found\n");
616
617 if (list_empty(&dmar_atsr_units))
618 printk(KERN_INFO PREFIX "No ATSR found\n");
619 #endif
620
621 return 0;
622 }
623
624 static void warn_invalid_dmar(u64 addr, const char *message)
625 {
626 WARN_TAINT_ONCE(
627 1, TAINT_FIRMWARE_WORKAROUND,
628 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
629 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
630 addr, message,
631 dmi_get_system_info(DMI_BIOS_VENDOR),
632 dmi_get_system_info(DMI_BIOS_VERSION),
633 dmi_get_system_info(DMI_PRODUCT_VERSION));
634 }
635
636 int __init check_zero_address(void)
637 {
638 struct acpi_table_dmar *dmar;
639 struct acpi_dmar_header *entry_header;
640 struct acpi_dmar_hardware_unit *drhd;
641
642 dmar = (struct acpi_table_dmar *)dmar_tbl;
643 entry_header = (struct acpi_dmar_header *)(dmar + 1);
644
645 while (((unsigned long)entry_header) <
646 (((unsigned long)dmar) + dmar_tbl->length)) {
647 /* Avoid looping forever on bad ACPI tables */
648 if (entry_header->length == 0) {
649 printk(KERN_WARNING PREFIX
650 "Invalid 0-length structure\n");
651 return 0;
652 }
653
654 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
655 void __iomem *addr;
656 u64 cap, ecap;
657
658 drhd = (void *)entry_header;
659 if (!drhd->address) {
660 warn_invalid_dmar(0, "");
661 goto failed;
662 }
663
664 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
665 if (!addr ) {
666 printk("IOMMU: can't validate: %llx\n", drhd->address);
667 goto failed;
668 }
669 cap = dmar_readq(addr + DMAR_CAP_REG);
670 ecap = dmar_readq(addr + DMAR_ECAP_REG);
671 early_iounmap(addr, VTD_PAGE_SIZE);
672 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
673 warn_invalid_dmar(drhd->address,
674 " returns all ones");
675 goto failed;
676 }
677 }
678
679 entry_header = ((void *)entry_header + entry_header->length);
680 }
681 return 1;
682
683 failed:
684 #ifdef CONFIG_DMAR
685 dmar_disabled = 1;
686 #endif
687 return 0;
688 }
689
690 int __init detect_intel_iommu(void)
691 {
692 int ret;
693
694 ret = dmar_table_detect();
695 if (ret)
696 ret = check_zero_address();
697 {
698 #ifdef CONFIG_INTR_REMAP
699 struct acpi_table_dmar *dmar;
700
701 dmar = (struct acpi_table_dmar *) dmar_tbl;
702 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
703 printk(KERN_INFO
704 "Queued invalidation will be enabled to support "
705 "x2apic and Intr-remapping.\n");
706 #endif
707 #ifdef CONFIG_DMAR
708 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
709 iommu_detected = 1;
710 /* Make sure ACS will be enabled */
711 pci_request_acs();
712 }
713 #endif
714 #ifdef CONFIG_X86
715 if (ret)
716 x86_init.iommu.iommu_init = intel_iommu_init;
717 #endif
718 }
719 early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
720 dmar_tbl = NULL;
721
722 return ret ? 1 : -ENODEV;
723 }
724
725
726 int alloc_iommu(struct dmar_drhd_unit *drhd)
727 {
728 struct intel_iommu *iommu;
729 int map_size;
730 u32 ver;
731 static int iommu_allocated = 0;
732 int agaw = 0;
733 int msagaw = 0;
734
735 if (!drhd->reg_base_addr) {
736 warn_invalid_dmar(0, "");
737 return -EINVAL;
738 }
739
740 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
741 if (!iommu)
742 return -ENOMEM;
743
744 iommu->seq_id = iommu_allocated++;
745 sprintf (iommu->name, "dmar%d", iommu->seq_id);
746
747 iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
748 if (!iommu->reg) {
749 printk(KERN_ERR "IOMMU: can't map the region\n");
750 goto error;
751 }
752 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
753 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
754
755 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
756 warn_invalid_dmar(drhd->reg_base_addr, " returns all ones");
757 goto err_unmap;
758 }
759
760 #ifdef CONFIG_DMAR
761 agaw = iommu_calculate_agaw(iommu);
762 if (agaw < 0) {
763 printk(KERN_ERR
764 "Cannot get a valid agaw for iommu (seq_id = %d)\n",
765 iommu->seq_id);
766 goto err_unmap;
767 }
768 msagaw = iommu_calculate_max_sagaw(iommu);
769 if (msagaw < 0) {
770 printk(KERN_ERR
771 "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
772 iommu->seq_id);
773 goto err_unmap;
774 }
775 #endif
776 iommu->agaw = agaw;
777 iommu->msagaw = msagaw;
778
779 iommu->node = -1;
780
781 /* the registers might be more than one page */
782 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
783 cap_max_fault_reg_offset(iommu->cap));
784 map_size = VTD_PAGE_ALIGN(map_size);
785 if (map_size > VTD_PAGE_SIZE) {
786 iounmap(iommu->reg);
787 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
788 if (!iommu->reg) {
789 printk(KERN_ERR "IOMMU: can't map the region\n");
790 goto error;
791 }
792 }
793
794 ver = readl(iommu->reg + DMAR_VER_REG);
795 pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
796 iommu->seq_id,
797 (unsigned long long)drhd->reg_base_addr,
798 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
799 (unsigned long long)iommu->cap,
800 (unsigned long long)iommu->ecap);
801
802 spin_lock_init(&iommu->register_lock);
803
804 drhd->iommu = iommu;
805 return 0;
806
807 err_unmap:
808 iounmap(iommu->reg);
809 error:
810 kfree(iommu);
811 return -1;
812 }
813
814 void free_iommu(struct intel_iommu *iommu)
815 {
816 if (!iommu)
817 return;
818
819 #ifdef CONFIG_DMAR
820 free_dmar_iommu(iommu);
821 #endif
822
823 if (iommu->reg)
824 iounmap(iommu->reg);
825 kfree(iommu);
826 }
827
828 /*
829 * Reclaim all the submitted descriptors which have completed its work.
830 */
831 static inline void reclaim_free_desc(struct q_inval *qi)
832 {
833 while (qi->desc_status[qi->free_tail] == QI_DONE ||
834 qi->desc_status[qi->free_tail] == QI_ABORT) {
835 qi->desc_status[qi->free_tail] = QI_FREE;
836 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
837 qi->free_cnt++;
838 }
839 }
840
841 static int qi_check_fault(struct intel_iommu *iommu, int index)
842 {
843 u32 fault;
844 int head, tail;
845 struct q_inval *qi = iommu->qi;
846 int wait_index = (index + 1) % QI_LENGTH;
847
848 if (qi->desc_status[wait_index] == QI_ABORT)
849 return -EAGAIN;
850
851 fault = readl(iommu->reg + DMAR_FSTS_REG);
852
853 /*
854 * If IQE happens, the head points to the descriptor associated
855 * with the error. No new descriptors are fetched until the IQE
856 * is cleared.
857 */
858 if (fault & DMA_FSTS_IQE) {
859 head = readl(iommu->reg + DMAR_IQH_REG);
860 if ((head >> DMAR_IQ_SHIFT) == index) {
861 printk(KERN_ERR "VT-d detected invalid descriptor: "
862 "low=%llx, high=%llx\n",
863 (unsigned long long)qi->desc[index].low,
864 (unsigned long long)qi->desc[index].high);
865 memcpy(&qi->desc[index], &qi->desc[wait_index],
866 sizeof(struct qi_desc));
867 __iommu_flush_cache(iommu, &qi->desc[index],
868 sizeof(struct qi_desc));
869 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
870 return -EINVAL;
871 }
872 }
873
874 /*
875 * If ITE happens, all pending wait_desc commands are aborted.
876 * No new descriptors are fetched until the ITE is cleared.
877 */
878 if (fault & DMA_FSTS_ITE) {
879 head = readl(iommu->reg + DMAR_IQH_REG);
880 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
881 head |= 1;
882 tail = readl(iommu->reg + DMAR_IQT_REG);
883 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
884
885 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
886
887 do {
888 if (qi->desc_status[head] == QI_IN_USE)
889 qi->desc_status[head] = QI_ABORT;
890 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
891 } while (head != tail);
892
893 if (qi->desc_status[wait_index] == QI_ABORT)
894 return -EAGAIN;
895 }
896
897 if (fault & DMA_FSTS_ICE)
898 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
899
900 return 0;
901 }
902
903 /*
904 * Submit the queued invalidation descriptor to the remapping
905 * hardware unit and wait for its completion.
906 */
907 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
908 {
909 int rc;
910 struct q_inval *qi = iommu->qi;
911 struct qi_desc *hw, wait_desc;
912 int wait_index, index;
913 unsigned long flags;
914
915 if (!qi)
916 return 0;
917
918 hw = qi->desc;
919
920 restart:
921 rc = 0;
922
923 spin_lock_irqsave(&qi->q_lock, flags);
924 while (qi->free_cnt < 3) {
925 spin_unlock_irqrestore(&qi->q_lock, flags);
926 cpu_relax();
927 spin_lock_irqsave(&qi->q_lock, flags);
928 }
929
930 index = qi->free_head;
931 wait_index = (index + 1) % QI_LENGTH;
932
933 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
934
935 hw[index] = *desc;
936
937 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
938 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
939 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
940
941 hw[wait_index] = wait_desc;
942
943 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
944 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
945
946 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
947 qi->free_cnt -= 2;
948
949 /*
950 * update the HW tail register indicating the presence of
951 * new descriptors.
952 */
953 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
954
955 while (qi->desc_status[wait_index] != QI_DONE) {
956 /*
957 * We will leave the interrupts disabled, to prevent interrupt
958 * context to queue another cmd while a cmd is already submitted
959 * and waiting for completion on this cpu. This is to avoid
960 * a deadlock where the interrupt context can wait indefinitely
961 * for free slots in the queue.
962 */
963 rc = qi_check_fault(iommu, index);
964 if (rc)
965 break;
966
967 spin_unlock(&qi->q_lock);
968 cpu_relax();
969 spin_lock(&qi->q_lock);
970 }
971
972 qi->desc_status[index] = QI_DONE;
973
974 reclaim_free_desc(qi);
975 spin_unlock_irqrestore(&qi->q_lock, flags);
976
977 if (rc == -EAGAIN)
978 goto restart;
979
980 return rc;
981 }
982
983 /*
984 * Flush the global interrupt entry cache.
985 */
986 void qi_global_iec(struct intel_iommu *iommu)
987 {
988 struct qi_desc desc;
989
990 desc.low = QI_IEC_TYPE;
991 desc.high = 0;
992
993 /* should never fail */
994 qi_submit_sync(&desc, iommu);
995 }
996
997 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
998 u64 type)
999 {
1000 struct qi_desc desc;
1001
1002 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1003 | QI_CC_GRAN(type) | QI_CC_TYPE;
1004 desc.high = 0;
1005
1006 qi_submit_sync(&desc, iommu);
1007 }
1008
1009 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1010 unsigned int size_order, u64 type)
1011 {
1012 u8 dw = 0, dr = 0;
1013
1014 struct qi_desc desc;
1015 int ih = 0;
1016
1017 if (cap_write_drain(iommu->cap))
1018 dw = 1;
1019
1020 if (cap_read_drain(iommu->cap))
1021 dr = 1;
1022
1023 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1024 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1025 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1026 | QI_IOTLB_AM(size_order);
1027
1028 qi_submit_sync(&desc, iommu);
1029 }
1030
1031 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1032 u64 addr, unsigned mask)
1033 {
1034 struct qi_desc desc;
1035
1036 if (mask) {
1037 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1038 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1039 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1040 } else
1041 desc.high = QI_DEV_IOTLB_ADDR(addr);
1042
1043 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1044 qdep = 0;
1045
1046 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1047 QI_DIOTLB_TYPE;
1048
1049 qi_submit_sync(&desc, iommu);
1050 }
1051
1052 /*
1053 * Disable Queued Invalidation interface.
1054 */
1055 void dmar_disable_qi(struct intel_iommu *iommu)
1056 {
1057 unsigned long flags;
1058 u32 sts;
1059 cycles_t start_time = get_cycles();
1060
1061 if (!ecap_qis(iommu->ecap))
1062 return;
1063
1064 spin_lock_irqsave(&iommu->register_lock, flags);
1065
1066 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
1067 if (!(sts & DMA_GSTS_QIES))
1068 goto end;
1069
1070 /*
1071 * Give a chance to HW to complete the pending invalidation requests.
1072 */
1073 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1074 readl(iommu->reg + DMAR_IQH_REG)) &&
1075 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1076 cpu_relax();
1077
1078 iommu->gcmd &= ~DMA_GCMD_QIE;
1079 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1080
1081 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1082 !(sts & DMA_GSTS_QIES), sts);
1083 end:
1084 spin_unlock_irqrestore(&iommu->register_lock, flags);
1085 }
1086
1087 /*
1088 * Enable queued invalidation.
1089 */
1090 static void __dmar_enable_qi(struct intel_iommu *iommu)
1091 {
1092 u32 sts;
1093 unsigned long flags;
1094 struct q_inval *qi = iommu->qi;
1095
1096 qi->free_head = qi->free_tail = 0;
1097 qi->free_cnt = QI_LENGTH;
1098
1099 spin_lock_irqsave(&iommu->register_lock, flags);
1100
1101 /* write zero to the tail reg */
1102 writel(0, iommu->reg + DMAR_IQT_REG);
1103
1104 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1105
1106 iommu->gcmd |= DMA_GCMD_QIE;
1107 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1108
1109 /* Make sure hardware complete it */
1110 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1111
1112 spin_unlock_irqrestore(&iommu->register_lock, flags);
1113 }
1114
1115 /*
1116 * Enable Queued Invalidation interface. This is a must to support
1117 * interrupt-remapping. Also used by DMA-remapping, which replaces
1118 * register based IOTLB invalidation.
1119 */
1120 int dmar_enable_qi(struct intel_iommu *iommu)
1121 {
1122 struct q_inval *qi;
1123 struct page *desc_page;
1124
1125 if (!ecap_qis(iommu->ecap))
1126 return -ENOENT;
1127
1128 /*
1129 * queued invalidation is already setup and enabled.
1130 */
1131 if (iommu->qi)
1132 return 0;
1133
1134 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1135 if (!iommu->qi)
1136 return -ENOMEM;
1137
1138 qi = iommu->qi;
1139
1140
1141 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1142 if (!desc_page) {
1143 kfree(qi);
1144 iommu->qi = 0;
1145 return -ENOMEM;
1146 }
1147
1148 qi->desc = page_address(desc_page);
1149
1150 qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1151 if (!qi->desc_status) {
1152 free_page((unsigned long) qi->desc);
1153 kfree(qi);
1154 iommu->qi = 0;
1155 return -ENOMEM;
1156 }
1157
1158 qi->free_head = qi->free_tail = 0;
1159 qi->free_cnt = QI_LENGTH;
1160
1161 spin_lock_init(&qi->q_lock);
1162
1163 __dmar_enable_qi(iommu);
1164
1165 return 0;
1166 }
1167
1168 /* iommu interrupt handling. Most stuff are MSI-like. */
1169
1170 enum faulttype {
1171 DMA_REMAP,
1172 INTR_REMAP,
1173 UNKNOWN,
1174 };
1175
1176 static const char *dma_remap_fault_reasons[] =
1177 {
1178 "Software",
1179 "Present bit in root entry is clear",
1180 "Present bit in context entry is clear",
1181 "Invalid context entry",
1182 "Access beyond MGAW",
1183 "PTE Write access is not set",
1184 "PTE Read access is not set",
1185 "Next page table ptr is invalid",
1186 "Root table address invalid",
1187 "Context table ptr is invalid",
1188 "non-zero reserved fields in RTP",
1189 "non-zero reserved fields in CTP",
1190 "non-zero reserved fields in PTE",
1191 };
1192
1193 static const char *intr_remap_fault_reasons[] =
1194 {
1195 "Detected reserved fields in the decoded interrupt-remapped request",
1196 "Interrupt index exceeded the interrupt-remapping table size",
1197 "Present field in the IRTE entry is clear",
1198 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1199 "Detected reserved fields in the IRTE entry",
1200 "Blocked a compatibility format interrupt request",
1201 "Blocked an interrupt request due to source-id verification failure",
1202 };
1203
1204 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
1205
1206 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1207 {
1208 if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1209 ARRAY_SIZE(intr_remap_fault_reasons))) {
1210 *fault_type = INTR_REMAP;
1211 return intr_remap_fault_reasons[fault_reason - 0x20];
1212 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1213 *fault_type = DMA_REMAP;
1214 return dma_remap_fault_reasons[fault_reason];
1215 } else {
1216 *fault_type = UNKNOWN;
1217 return "Unknown";
1218 }
1219 }
1220
1221 void dmar_msi_unmask(struct irq_data *data)
1222 {
1223 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1224 unsigned long flag;
1225
1226 /* unmask it */
1227 spin_lock_irqsave(&iommu->register_lock, flag);
1228 writel(0, iommu->reg + DMAR_FECTL_REG);
1229 /* Read a reg to force flush the post write */
1230 readl(iommu->reg + DMAR_FECTL_REG);
1231 spin_unlock_irqrestore(&iommu->register_lock, flag);
1232 }
1233
1234 void dmar_msi_mask(struct irq_data *data)
1235 {
1236 unsigned long flag;
1237 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1238
1239 /* mask it */
1240 spin_lock_irqsave(&iommu->register_lock, flag);
1241 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1242 /* Read a reg to force flush the post write */
1243 readl(iommu->reg + DMAR_FECTL_REG);
1244 spin_unlock_irqrestore(&iommu->register_lock, flag);
1245 }
1246
1247 void dmar_msi_write(int irq, struct msi_msg *msg)
1248 {
1249 struct intel_iommu *iommu = irq_get_handler_data(irq);
1250 unsigned long flag;
1251
1252 spin_lock_irqsave(&iommu->register_lock, flag);
1253 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1254 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1255 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1256 spin_unlock_irqrestore(&iommu->register_lock, flag);
1257 }
1258
1259 void dmar_msi_read(int irq, struct msi_msg *msg)
1260 {
1261 struct intel_iommu *iommu = irq_get_handler_data(irq);
1262 unsigned long flag;
1263
1264 spin_lock_irqsave(&iommu->register_lock, flag);
1265 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1266 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1267 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1268 spin_unlock_irqrestore(&iommu->register_lock, flag);
1269 }
1270
1271 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1272 u8 fault_reason, u16 source_id, unsigned long long addr)
1273 {
1274 const char *reason;
1275 int fault_type;
1276
1277 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1278
1279 if (fault_type == INTR_REMAP)
1280 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1281 "fault index %llx\n"
1282 "INTR-REMAP:[fault reason %02d] %s\n",
1283 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1284 PCI_FUNC(source_id & 0xFF), addr >> 48,
1285 fault_reason, reason);
1286 else
1287 printk(KERN_ERR
1288 "DMAR:[%s] Request device [%02x:%02x.%d] "
1289 "fault addr %llx \n"
1290 "DMAR:[fault reason %02d] %s\n",
1291 (type ? "DMA Read" : "DMA Write"),
1292 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1293 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1294 return 0;
1295 }
1296
1297 #define PRIMARY_FAULT_REG_LEN (16)
1298 irqreturn_t dmar_fault(int irq, void *dev_id)
1299 {
1300 struct intel_iommu *iommu = dev_id;
1301 int reg, fault_index;
1302 u32 fault_status;
1303 unsigned long flag;
1304
1305 spin_lock_irqsave(&iommu->register_lock, flag);
1306 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1307 if (fault_status)
1308 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1309 fault_status);
1310
1311 /* TBD: ignore advanced fault log currently */
1312 if (!(fault_status & DMA_FSTS_PPF))
1313 goto clear_rest;
1314
1315 fault_index = dma_fsts_fault_record_index(fault_status);
1316 reg = cap_fault_reg_offset(iommu->cap);
1317 while (1) {
1318 u8 fault_reason;
1319 u16 source_id;
1320 u64 guest_addr;
1321 int type;
1322 u32 data;
1323
1324 /* highest 32 bits */
1325 data = readl(iommu->reg + reg +
1326 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1327 if (!(data & DMA_FRCD_F))
1328 break;
1329
1330 fault_reason = dma_frcd_fault_reason(data);
1331 type = dma_frcd_type(data);
1332
1333 data = readl(iommu->reg + reg +
1334 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1335 source_id = dma_frcd_source_id(data);
1336
1337 guest_addr = dmar_readq(iommu->reg + reg +
1338 fault_index * PRIMARY_FAULT_REG_LEN);
1339 guest_addr = dma_frcd_page_addr(guest_addr);
1340 /* clear the fault */
1341 writel(DMA_FRCD_F, iommu->reg + reg +
1342 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1343
1344 spin_unlock_irqrestore(&iommu->register_lock, flag);
1345
1346 dmar_fault_do_one(iommu, type, fault_reason,
1347 source_id, guest_addr);
1348
1349 fault_index++;
1350 if (fault_index >= cap_num_fault_regs(iommu->cap))
1351 fault_index = 0;
1352 spin_lock_irqsave(&iommu->register_lock, flag);
1353 }
1354 clear_rest:
1355 /* clear all the other faults */
1356 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1357 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1358
1359 spin_unlock_irqrestore(&iommu->register_lock, flag);
1360 return IRQ_HANDLED;
1361 }
1362
1363 int dmar_set_interrupt(struct intel_iommu *iommu)
1364 {
1365 int irq, ret;
1366
1367 /*
1368 * Check if the fault interrupt is already initialized.
1369 */
1370 if (iommu->irq)
1371 return 0;
1372
1373 irq = create_irq();
1374 if (!irq) {
1375 printk(KERN_ERR "IOMMU: no free vectors\n");
1376 return -EINVAL;
1377 }
1378
1379 irq_set_handler_data(irq, iommu);
1380 iommu->irq = irq;
1381
1382 ret = arch_setup_dmar_msi(irq);
1383 if (ret) {
1384 irq_set_handler_data(irq, NULL);
1385 iommu->irq = 0;
1386 destroy_irq(irq);
1387 return ret;
1388 }
1389
1390 ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1391 if (ret)
1392 printk(KERN_ERR "IOMMU: can't request irq\n");
1393 return ret;
1394 }
1395
1396 int __init enable_drhd_fault_handling(void)
1397 {
1398 struct dmar_drhd_unit *drhd;
1399
1400 /*
1401 * Enable fault control interrupt.
1402 */
1403 for_each_drhd_unit(drhd) {
1404 int ret;
1405 struct intel_iommu *iommu = drhd->iommu;
1406 ret = dmar_set_interrupt(iommu);
1407
1408 if (ret) {
1409 printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1410 " interrupt, ret %d\n",
1411 (unsigned long long)drhd->reg_base_addr, ret);
1412 return -1;
1413 }
1414
1415 /*
1416 * Clear any previous faults.
1417 */
1418 dmar_fault(iommu->irq, iommu);
1419 }
1420
1421 return 0;
1422 }
1423
1424 /*
1425 * Re-enable Queued Invalidation interface.
1426 */
1427 int dmar_reenable_qi(struct intel_iommu *iommu)
1428 {
1429 if (!ecap_qis(iommu->ecap))
1430 return -ENOENT;
1431
1432 if (!iommu->qi)
1433 return -ENOENT;
1434
1435 /*
1436 * First disable queued invalidation.
1437 */
1438 dmar_disable_qi(iommu);
1439 /*
1440 * Then enable queued invalidation again. Since there is no pending
1441 * invalidation requests now, it's safe to re-enable queued
1442 * invalidation.
1443 */
1444 __dmar_enable_qi(iommu);
1445
1446 return 0;
1447 }
1448
1449 /*
1450 * Check interrupt remapping support in DMAR table description.
1451 */
1452 int __init dmar_ir_support(void)
1453 {
1454 struct acpi_table_dmar *dmar;
1455 dmar = (struct acpi_table_dmar *)dmar_tbl;
1456 if (!dmar)
1457 return 0;
1458 return dmar->flags & 0x1;
1459 }
1460 IOMMU_INIT_POST(detect_intel_iommu);
linux-next