?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[media] stb0899: use sign_extend32() for sign extension
[linux/fpc-iii.git] / drivers / iommu / dmar.c
blob9847613085e157976707e0d1aa0cc87c3e8b3c68
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 *
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */
28
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */
30
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/iova.h>
34 #include <linux/intel-iommu.h>
35 #include <linux/timer.h>
36 #include <linux/irq.h>
37 #include <linux/interrupt.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/slab.h>
41 #include <linux/iommu.h>
42 #include <asm/irq_remapping.h>
43 #include <asm/iommu_table.h>
44
45 #include "irq_remapping.h"
46
47 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
48 struct dmar_res_callback {
49 dmar_res_handler_t cb[ACPI_DMAR_TYPE_RESERVED];
50 void *arg[ACPI_DMAR_TYPE_RESERVED];
51 bool ignore_unhandled;
52 bool print_entry;
53 };
54
55 /*
56 * Assumptions:
57 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
58 * before IO devices managed by that unit.
59 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
60 * after IO devices managed by that unit.
61 * 3) Hotplug events are rare.
62 *
63 * Locking rules for DMA and interrupt remapping related global data structures:
64 * 1) Use dmar_global_lock in process context
65 * 2) Use RCU in interrupt context
66 */
67 DECLARE_RWSEM(dmar_global_lock);
68 LIST_HEAD(dmar_drhd_units);
69
70 struct acpi_table_header * __initdata dmar_tbl;
71 static acpi_size dmar_tbl_size;
72 static int dmar_dev_scope_status = 1;
73 static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
74
75 static int alloc_iommu(struct dmar_drhd_unit *drhd);
76 static void free_iommu(struct intel_iommu *iommu);
77
78 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
79 {
80 /*
81 * add INCLUDE_ALL at the tail, so scan the list will find it at
82 * the very end.
83 */
84 if (drhd->include_all)
85 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
86 else
87 list_add_rcu(&drhd->list, &dmar_drhd_units);
88 }
89
90 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
91 {
92 struct acpi_dmar_device_scope *scope;
93
94 *cnt = 0;
95 while (start < end) {
96 scope = start;
97 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
98 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
99 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
100 (*cnt)++;
101 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
102 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
103 pr_warn("Unsupported device scope\n");
104 }
105 start += scope->length;
106 }
107 if (*cnt == 0)
108 return NULL;
109
110 return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
111 }
112
113 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
114 {
115 int i;
116 struct device *tmp_dev;
117
118 if (*devices && *cnt) {
119 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
120 put_device(tmp_dev);
121 kfree(*devices);
122 }
123
124 *devices = NULL;
125 *cnt = 0;
126 }
127
128 /* Optimize out kzalloc()/kfree() for normal cases */
129 static char dmar_pci_notify_info_buf[64];
130
131 static struct dmar_pci_notify_info *
132 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
133 {
134 int level = 0;
135 size_t size;
136 struct pci_dev *tmp;
137 struct dmar_pci_notify_info *info;
138
139 BUG_ON(dev->is_virtfn);
140
141 /* Only generate path[] for device addition event */
142 if (event == BUS_NOTIFY_ADD_DEVICE)
143 for (tmp = dev; tmp; tmp = tmp->bus->self)
144 level++;
145
146 size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
147 if (size <= sizeof(dmar_pci_notify_info_buf)) {
148 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
149 } else {
150 info = kzalloc(size, GFP_KERNEL);
151 if (!info) {
152 pr_warn("Out of memory when allocating notify_info "
153 "for %s.\n", pci_name(dev));
154 if (dmar_dev_scope_status == 0)
155 dmar_dev_scope_status = -ENOMEM;
156 return NULL;
157 }
158 }
159
160 info->event = event;
161 info->dev = dev;
162 info->seg = pci_domain_nr(dev->bus);
163 info->level = level;
164 if (event == BUS_NOTIFY_ADD_DEVICE) {
165 for (tmp = dev; tmp; tmp = tmp->bus->self) {
166 level--;
167 info->path[level].bus = tmp->bus->number;
168 info->path[level].device = PCI_SLOT(tmp->devfn);
169 info->path[level].function = PCI_FUNC(tmp->devfn);
170 if (pci_is_root_bus(tmp->bus))
171 info->bus = tmp->bus->number;
172 }
173 }
174
175 return info;
176 }
177
178 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
179 {
180 if ((void *)info != dmar_pci_notify_info_buf)
181 kfree(info);
182 }
183
184 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
185 struct acpi_dmar_pci_path *path, int count)
186 {
187 int i;
188
189 if (info->bus != bus)
190 goto fallback;
191 if (info->level != count)
192 goto fallback;
193
194 for (i = 0; i < count; i++) {
195 if (path[i].device != info->path[i].device ||
196 path[i].function != info->path[i].function)
197 goto fallback;
198 }
199
200 return true;
201
202 fallback:
203
204 if (count != 1)
205 return false;
206
207 i = info->level - 1;
208 if (bus == info->path[i].bus &&
209 path[0].device == info->path[i].device &&
210 path[0].function == info->path[i].function) {
211 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
212 bus, path[0].device, path[0].function);
213 return true;
214 }
215
216 return false;
217 }
218
219 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
220 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
221 void *start, void*end, u16 segment,
222 struct dmar_dev_scope *devices,
223 int devices_cnt)
224 {
225 int i, level;
226 struct device *tmp, *dev = &info->dev->dev;
227 struct acpi_dmar_device_scope *scope;
228 struct acpi_dmar_pci_path *path;
229
230 if (segment != info->seg)
231 return 0;
232
233 for (; start < end; start += scope->length) {
234 scope = start;
235 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
236 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
237 continue;
238
239 path = (struct acpi_dmar_pci_path *)(scope + 1);
240 level = (scope->length - sizeof(*scope)) / sizeof(*path);
241 if (!dmar_match_pci_path(info, scope->bus, path, level))
242 continue;
243
244 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^
245 (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) {
246 pr_warn("Device scope type does not match for %s\n",
247 pci_name(info->dev));
248 return -EINVAL;
249 }
250
251 for_each_dev_scope(devices, devices_cnt, i, tmp)
252 if (tmp == NULL) {
253 devices[i].bus = info->dev->bus->number;
254 devices[i].devfn = info->dev->devfn;
255 rcu_assign_pointer(devices[i].dev,
256 get_device(dev));
257 return 1;
258 }
259 BUG_ON(i >= devices_cnt);
260 }
261
262 return 0;
263 }
264
265 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
266 struct dmar_dev_scope *devices, int count)
267 {
268 int index;
269 struct device *tmp;
270
271 if (info->seg != segment)
272 return 0;
273
274 for_each_active_dev_scope(devices, count, index, tmp)
275 if (tmp == &info->dev->dev) {
276 RCU_INIT_POINTER(devices[index].dev, NULL);
277 synchronize_rcu();
278 put_device(tmp);
279 return 1;
280 }
281
282 return 0;
283 }
284
285 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
286 {
287 int ret = 0;
288 struct dmar_drhd_unit *dmaru;
289 struct acpi_dmar_hardware_unit *drhd;
290
291 for_each_drhd_unit(dmaru) {
292 if (dmaru->include_all)
293 continue;
294
295 drhd = container_of(dmaru->hdr,
296 struct acpi_dmar_hardware_unit, header);
297 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
298 ((void *)drhd) + drhd->header.length,
299 dmaru->segment,
300 dmaru->devices, dmaru->devices_cnt);
301 if (ret != 0)
302 break;
303 }
304 if (ret >= 0)
305 ret = dmar_iommu_notify_scope_dev(info);
306 if (ret < 0 && dmar_dev_scope_status == 0)
307 dmar_dev_scope_status = ret;
308
309 return ret;
310 }
311
312 static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
313 {
314 struct dmar_drhd_unit *dmaru;
315
316 for_each_drhd_unit(dmaru)
317 if (dmar_remove_dev_scope(info, dmaru->segment,
318 dmaru->devices, dmaru->devices_cnt))
319 break;
320 dmar_iommu_notify_scope_dev(info);
321 }
322
323 static int dmar_pci_bus_notifier(struct notifier_block *nb,
324 unsigned long action, void *data)
325 {
326 struct pci_dev *pdev = to_pci_dev(data);
327 struct dmar_pci_notify_info *info;
328
329 /* Only care about add/remove events for physical functions */
330 if (pdev->is_virtfn)
331 return NOTIFY_DONE;
332 if (action != BUS_NOTIFY_ADD_DEVICE && action != BUS_NOTIFY_DEL_DEVICE)
333 return NOTIFY_DONE;
334
335 info = dmar_alloc_pci_notify_info(pdev, action);
336 if (!info)
337 return NOTIFY_DONE;
338
339 down_write(&dmar_global_lock);
340 if (action == BUS_NOTIFY_ADD_DEVICE)
341 dmar_pci_bus_add_dev(info);
342 else if (action == BUS_NOTIFY_DEL_DEVICE)
343 dmar_pci_bus_del_dev(info);
344 up_write(&dmar_global_lock);
345
346 dmar_free_pci_notify_info(info);
347
348 return NOTIFY_OK;
349 }
350
351 static struct notifier_block dmar_pci_bus_nb = {
352 .notifier_call = dmar_pci_bus_notifier,
353 .priority = INT_MIN,
354 };
355
356 static struct dmar_drhd_unit *
357 dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
358 {
359 struct dmar_drhd_unit *dmaru;
360
361 list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list)
362 if (dmaru->segment == drhd->segment &&
363 dmaru->reg_base_addr == drhd->address)
364 return dmaru;
365
366 return NULL;
367 }
368
369 /**
370 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
371 * structure which uniquely represent one DMA remapping hardware unit
372 * present in the platform
373 */
374 static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
375 {
376 struct acpi_dmar_hardware_unit *drhd;
377 struct dmar_drhd_unit *dmaru;
378 int ret = 0;
379
380 drhd = (struct acpi_dmar_hardware_unit *)header;
381 dmaru = dmar_find_dmaru(drhd);
382 if (dmaru)
383 goto out;
384
385 dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
386 if (!dmaru)
387 return -ENOMEM;
388
389 /*
390 * If header is allocated from slab by ACPI _DSM method, we need to
391 * copy the content because the memory buffer will be freed on return.
392 */
393 dmaru->hdr = (void *)(dmaru + 1);
394 memcpy(dmaru->hdr, header, header->length);
395 dmaru->reg_base_addr = drhd->address;
396 dmaru->segment = drhd->segment;
397 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
398 dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
399 ((void *)drhd) + drhd->header.length,
400 &dmaru->devices_cnt);
401 if (dmaru->devices_cnt && dmaru->devices == NULL) {
402 kfree(dmaru);
403 return -ENOMEM;
404 }
405
406 ret = alloc_iommu(dmaru);
407 if (ret) {
408 dmar_free_dev_scope(&dmaru->devices,
409 &dmaru->devices_cnt);
410 kfree(dmaru);
411 return ret;
412 }
413 dmar_register_drhd_unit(dmaru);
414
415 out:
416 if (arg)
417 (*(int *)arg)++;
418
419 return 0;
420 }
421
422 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
423 {
424 if (dmaru->devices && dmaru->devices_cnt)
425 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
426 if (dmaru->iommu)
427 free_iommu(dmaru->iommu);
428 kfree(dmaru);
429 }
430
431 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
432 void *arg)
433 {
434 struct acpi_dmar_andd *andd = (void *)header;
435
436 /* Check for NUL termination within the designated length */
437 if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
438 WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
439 "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
440 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
441 dmi_get_system_info(DMI_BIOS_VENDOR),
442 dmi_get_system_info(DMI_BIOS_VERSION),
443 dmi_get_system_info(DMI_PRODUCT_VERSION));
444 return -EINVAL;
445 }
446 pr_info("ANDD device: %x name: %s\n", andd->device_number,
447 andd->device_name);
448
449 return 0;
450 }
451
452 #ifdef CONFIG_ACPI_NUMA
453 static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
454 {
455 struct acpi_dmar_rhsa *rhsa;
456 struct dmar_drhd_unit *drhd;
457
458 rhsa = (struct acpi_dmar_rhsa *)header;
459 for_each_drhd_unit(drhd) {
460 if (drhd->reg_base_addr == rhsa->base_address) {
461 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
462
463 if (!node_online(node))
464 node = -1;
465 drhd->iommu->node = node;
466 return 0;
467 }
468 }
469 WARN_TAINT(
470 1, TAINT_FIRMWARE_WORKAROUND,
471 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
472 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
473 drhd->reg_base_addr,
474 dmi_get_system_info(DMI_BIOS_VENDOR),
475 dmi_get_system_info(DMI_BIOS_VERSION),
476 dmi_get_system_info(DMI_PRODUCT_VERSION));
477
478 return 0;
479 }
480 #else
481 #define dmar_parse_one_rhsa dmar_res_noop
482 #endif
483
484 static void __init
485 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
486 {
487 struct acpi_dmar_hardware_unit *drhd;
488 struct acpi_dmar_reserved_memory *rmrr;
489 struct acpi_dmar_atsr *atsr;
490 struct acpi_dmar_rhsa *rhsa;
491
492 switch (header->type) {
493 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
494 drhd = container_of(header, struct acpi_dmar_hardware_unit,
495 header);
496 pr_info("DRHD base: %#016Lx flags: %#x\n",
497 (unsigned long long)drhd->address, drhd->flags);
498 break;
499 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
500 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
501 header);
502 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
503 (unsigned long long)rmrr->base_address,
504 (unsigned long long)rmrr->end_address);
505 break;
506 case ACPI_DMAR_TYPE_ROOT_ATS:
507 atsr = container_of(header, struct acpi_dmar_atsr, header);
508 pr_info("ATSR flags: %#x\n", atsr->flags);
509 break;
510 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
511 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
512 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
513 (unsigned long long)rhsa->base_address,
514 rhsa->proximity_domain);
515 break;
516 case ACPI_DMAR_TYPE_NAMESPACE:
517 /* We don't print this here because we need to sanity-check
518 it first. So print it in dmar_parse_one_andd() instead. */
519 break;
520 }
521 }
522
523 /**
524 * dmar_table_detect - checks to see if the platform supports DMAR devices
525 */
526 static int __init dmar_table_detect(void)
527 {
528 acpi_status status = AE_OK;
529
530 /* if we could find DMAR table, then there are DMAR devices */
531 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
532 (struct acpi_table_header **)&dmar_tbl,
533 &dmar_tbl_size);
534
535 if (ACPI_SUCCESS(status) && !dmar_tbl) {
536 pr_warn("Unable to map DMAR\n");
537 status = AE_NOT_FOUND;
538 }
539
540 return (ACPI_SUCCESS(status) ? 1 : 0);
541 }
542
543 static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
544 size_t len, struct dmar_res_callback *cb)
545 {
546 int ret = 0;
547 struct acpi_dmar_header *iter, *next;
548 struct acpi_dmar_header *end = ((void *)start) + len;
549
550 for (iter = start; iter < end && ret == 0; iter = next) {
551 next = (void *)iter + iter->length;
552 if (iter->length == 0) {
553 /* Avoid looping forever on bad ACPI tables */
554 pr_debug(FW_BUG "Invalid 0-length structure\n");
555 break;
556 } else if (next > end) {
557 /* Avoid passing table end */
558 pr_warn(FW_BUG "record passes table end\n");
559 ret = -EINVAL;
560 break;
561 }
562
563 if (cb->print_entry)
564 dmar_table_print_dmar_entry(iter);
565
566 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
567 /* continue for forward compatibility */
568 pr_debug("Unknown DMAR structure type %d\n",
569 iter->type);
570 } else if (cb->cb[iter->type]) {
571 ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
572 } else if (!cb->ignore_unhandled) {
573 pr_warn("No handler for DMAR structure type %d\n",
574 iter->type);
575 ret = -EINVAL;
576 }
577 }
578
579 return ret;
580 }
581
582 static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
583 struct dmar_res_callback *cb)
584 {
585 return dmar_walk_remapping_entries((void *)(dmar + 1),
586 dmar->header.length - sizeof(*dmar), cb);
587 }
588
589 /**
590 * parse_dmar_table - parses the DMA reporting table
591 */
592 static int __init
593 parse_dmar_table(void)
594 {
595 struct acpi_table_dmar *dmar;
596 int ret = 0;
597 int drhd_count = 0;
598 struct dmar_res_callback cb = {
599 .print_entry = true,
600 .ignore_unhandled = true,
601 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
602 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
603 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
604 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
605 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
606 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
607 };
608
609 /*
610 * Do it again, earlier dmar_tbl mapping could be mapped with
611 * fixed map.
612 */
613 dmar_table_detect();
614
615 /*
616 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
617 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
618 */
619 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
620
621 dmar = (struct acpi_table_dmar *)dmar_tbl;
622 if (!dmar)
623 return -ENODEV;
624
625 if (dmar->width < PAGE_SHIFT - 1) {
626 pr_warn("Invalid DMAR haw\n");
627 return -EINVAL;
628 }
629
630 pr_info("Host address width %d\n", dmar->width + 1);
631 ret = dmar_walk_dmar_table(dmar, &cb);
632 if (ret == 0 && drhd_count == 0)
633 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
634
635 return ret;
636 }
637
638 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
639 int cnt, struct pci_dev *dev)
640 {
641 int index;
642 struct device *tmp;
643
644 while (dev) {
645 for_each_active_dev_scope(devices, cnt, index, tmp)
646 if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
647 return 1;
648
649 /* Check our parent */
650 dev = dev->bus->self;
651 }
652
653 return 0;
654 }
655
656 struct dmar_drhd_unit *
657 dmar_find_matched_drhd_unit(struct pci_dev *dev)
658 {
659 struct dmar_drhd_unit *dmaru;
660 struct acpi_dmar_hardware_unit *drhd;
661
662 dev = pci_physfn(dev);
663
664 rcu_read_lock();
665 for_each_drhd_unit(dmaru) {
666 drhd = container_of(dmaru->hdr,
667 struct acpi_dmar_hardware_unit,
668 header);
669
670 if (dmaru->include_all &&
671 drhd->segment == pci_domain_nr(dev->bus))
672 goto out;
673
674 if (dmar_pci_device_match(dmaru->devices,
675 dmaru->devices_cnt, dev))
676 goto out;
677 }
678 dmaru = NULL;
679 out:
680 rcu_read_unlock();
681
682 return dmaru;
683 }
684
685 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
686 struct acpi_device *adev)
687 {
688 struct dmar_drhd_unit *dmaru;
689 struct acpi_dmar_hardware_unit *drhd;
690 struct acpi_dmar_device_scope *scope;
691 struct device *tmp;
692 int i;
693 struct acpi_dmar_pci_path *path;
694
695 for_each_drhd_unit(dmaru) {
696 drhd = container_of(dmaru->hdr,
697 struct acpi_dmar_hardware_unit,
698 header);
699
700 for (scope = (void *)(drhd + 1);
701 (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
702 scope = ((void *)scope) + scope->length) {
703 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
704 continue;
705 if (scope->enumeration_id != device_number)
706 continue;
707
708 path = (void *)(scope + 1);
709 pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
710 dev_name(&adev->dev), dmaru->reg_base_addr,
711 scope->bus, path->device, path->function);
712 for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
713 if (tmp == NULL) {
714 dmaru->devices[i].bus = scope->bus;
715 dmaru->devices[i].devfn = PCI_DEVFN(path->device,
716 path->function);
717 rcu_assign_pointer(dmaru->devices[i].dev,
718 get_device(&adev->dev));
719 return;
720 }
721 BUG_ON(i >= dmaru->devices_cnt);
722 }
723 }
724 pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
725 device_number, dev_name(&adev->dev));
726 }
727
728 static int __init dmar_acpi_dev_scope_init(void)
729 {
730 struct acpi_dmar_andd *andd;
731
732 if (dmar_tbl == NULL)
733 return -ENODEV;
734
735 for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
736 ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
737 andd = ((void *)andd) + andd->header.length) {
738 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
739 acpi_handle h;
740 struct acpi_device *adev;
741
742 if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
743 andd->device_name,
744 &h))) {
745 pr_err("Failed to find handle for ACPI object %s\n",
746 andd->device_name);
747 continue;
748 }
749 if (acpi_bus_get_device(h, &adev)) {
750 pr_err("Failed to get device for ACPI object %s\n",
751 andd->device_name);
752 continue;
753 }
754 dmar_acpi_insert_dev_scope(andd->device_number, adev);
755 }
756 }
757 return 0;
758 }
759
760 int __init dmar_dev_scope_init(void)
761 {
762 struct pci_dev *dev = NULL;
763 struct dmar_pci_notify_info *info;
764
765 if (dmar_dev_scope_status != 1)
766 return dmar_dev_scope_status;
767
768 if (list_empty(&dmar_drhd_units)) {
769 dmar_dev_scope_status = -ENODEV;
770 } else {
771 dmar_dev_scope_status = 0;
772
773 dmar_acpi_dev_scope_init();
774
775 for_each_pci_dev(dev) {
776 if (dev->is_virtfn)
777 continue;
778
779 info = dmar_alloc_pci_notify_info(dev,
780 BUS_NOTIFY_ADD_DEVICE);
781 if (!info) {
782 return dmar_dev_scope_status;
783 } else {
784 dmar_pci_bus_add_dev(info);
785 dmar_free_pci_notify_info(info);
786 }
787 }
788
789 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
790 }
791
792 return dmar_dev_scope_status;
793 }
794
795
796 int __init dmar_table_init(void)
797 {
798 static int dmar_table_initialized;
799 int ret;
800
801 if (dmar_table_initialized == 0) {
802 ret = parse_dmar_table();
803 if (ret < 0) {
804 if (ret != -ENODEV)
805 pr_info("parse DMAR table failure.\n");
806 } else if (list_empty(&dmar_drhd_units)) {
807 pr_info("No DMAR devices found\n");
808 ret = -ENODEV;
809 }
810
811 if (ret < 0)
812 dmar_table_initialized = ret;
813 else
814 dmar_table_initialized = 1;
815 }
816
817 return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
818 }
819
820 static void warn_invalid_dmar(u64 addr, const char *message)
821 {
822 WARN_TAINT_ONCE(
823 1, TAINT_FIRMWARE_WORKAROUND,
824 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
825 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
826 addr, message,
827 dmi_get_system_info(DMI_BIOS_VENDOR),
828 dmi_get_system_info(DMI_BIOS_VERSION),
829 dmi_get_system_info(DMI_PRODUCT_VERSION));
830 }
831
832 static int __ref
833 dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
834 {
835 struct acpi_dmar_hardware_unit *drhd;
836 void __iomem *addr;
837 u64 cap, ecap;
838
839 drhd = (void *)entry;
840 if (!drhd->address) {
841 warn_invalid_dmar(0, "");
842 return -EINVAL;
843 }
844
845 if (arg)
846 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
847 else
848 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
849 if (!addr) {
850 pr_warn("IOMMU: can't validate: %llx\n", drhd->address);
851 return -EINVAL;
852 }
853
854 cap = dmar_readq(addr + DMAR_CAP_REG);
855 ecap = dmar_readq(addr + DMAR_ECAP_REG);
856
857 if (arg)
858 iounmap(addr);
859 else
860 early_iounmap(addr, VTD_PAGE_SIZE);
861
862 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
863 warn_invalid_dmar(drhd->address, " returns all ones");
864 return -EINVAL;
865 }
866
867 return 0;
868 }
869
870 int __init detect_intel_iommu(void)
871 {
872 int ret;
873 struct dmar_res_callback validate_drhd_cb = {
874 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
875 .ignore_unhandled = true,
876 };
877
878 down_write(&dmar_global_lock);
879 ret = dmar_table_detect();
880 if (ret)
881 ret = !dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
882 &validate_drhd_cb);
883 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
884 iommu_detected = 1;
885 /* Make sure ACS will be enabled */
886 pci_request_acs();
887 }
888
889 #ifdef CONFIG_X86
890 if (ret)
891 x86_init.iommu.iommu_init = intel_iommu_init;
892 #endif
893
894 early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
895 dmar_tbl = NULL;
896 up_write(&dmar_global_lock);
897
898 return ret ? 1 : -ENODEV;
899 }
900
901
902 static void unmap_iommu(struct intel_iommu *iommu)
903 {
904 iounmap(iommu->reg);
905 release_mem_region(iommu->reg_phys, iommu->reg_size);
906 }
907
908 /**
909 * map_iommu: map the iommu's registers
910 * @iommu: the iommu to map
911 * @phys_addr: the physical address of the base resgister
912 *
913 * Memory map the iommu's registers. Start w/ a single page, and
914 * possibly expand if that turns out to be insufficent.
915 */
916 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
917 {
918 int map_size, err=0;
919
920 iommu->reg_phys = phys_addr;
921 iommu->reg_size = VTD_PAGE_SIZE;
922
923 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
924 pr_err("IOMMU: can't reserve memory\n");
925 err = -EBUSY;
926 goto out;
927 }
928
929 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
930 if (!iommu->reg) {
931 pr_err("IOMMU: can't map the region\n");
932 err = -ENOMEM;
933 goto release;
934 }
935
936 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
937 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
938
939 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
940 err = -EINVAL;
941 warn_invalid_dmar(phys_addr, " returns all ones");
942 goto unmap;
943 }
944
945 /* the registers might be more than one page */
946 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
947 cap_max_fault_reg_offset(iommu->cap));
948 map_size = VTD_PAGE_ALIGN(map_size);
949 if (map_size > iommu->reg_size) {
950 iounmap(iommu->reg);
951 release_mem_region(iommu->reg_phys, iommu->reg_size);
952 iommu->reg_size = map_size;
953 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
954 iommu->name)) {
955 pr_err("IOMMU: can't reserve memory\n");
956 err = -EBUSY;
957 goto out;
958 }
959 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
960 if (!iommu->reg) {
961 pr_err("IOMMU: can't map the region\n");
962 err = -ENOMEM;
963 goto release;
964 }
965 }
966 err = 0;
967 goto out;
968
969 unmap:
970 iounmap(iommu->reg);
971 release:
972 release_mem_region(iommu->reg_phys, iommu->reg_size);
973 out:
974 return err;
975 }
976
977 static int dmar_alloc_seq_id(struct intel_iommu *iommu)
978 {
979 iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
980 DMAR_UNITS_SUPPORTED);
981 if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
982 iommu->seq_id = -1;
983 } else {
984 set_bit(iommu->seq_id, dmar_seq_ids);
985 sprintf(iommu->name, "dmar%d", iommu->seq_id);
986 }
987
988 return iommu->seq_id;
989 }
990
991 static void dmar_free_seq_id(struct intel_iommu *iommu)
992 {
993 if (iommu->seq_id >= 0) {
994 clear_bit(iommu->seq_id, dmar_seq_ids);
995 iommu->seq_id = -1;
996 }
997 }
998
999 static int alloc_iommu(struct dmar_drhd_unit *drhd)
1000 {
1001 struct intel_iommu *iommu;
1002 u32 ver, sts;
1003 int agaw = 0;
1004 int msagaw = 0;
1005 int err;
1006
1007 if (!drhd->reg_base_addr) {
1008 warn_invalid_dmar(0, "");
1009 return -EINVAL;
1010 }
1011
1012 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1013 if (!iommu)
1014 return -ENOMEM;
1015
1016 if (dmar_alloc_seq_id(iommu) < 0) {
1017 pr_err("IOMMU: failed to allocate seq_id\n");
1018 err = -ENOSPC;
1019 goto error;
1020 }
1021
1022 err = map_iommu(iommu, drhd->reg_base_addr);
1023 if (err) {
1024 pr_err("IOMMU: failed to map %s\n", iommu->name);
1025 goto error_free_seq_id;
1026 }
1027
1028 err = -EINVAL;
1029 agaw = iommu_calculate_agaw(iommu);
1030 if (agaw < 0) {
1031 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1032 iommu->seq_id);
1033 goto err_unmap;
1034 }
1035 msagaw = iommu_calculate_max_sagaw(iommu);
1036 if (msagaw < 0) {
1037 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1038 iommu->seq_id);
1039 goto err_unmap;
1040 }
1041 iommu->agaw = agaw;
1042 iommu->msagaw = msagaw;
1043 iommu->segment = drhd->segment;
1044
1045 iommu->node = -1;
1046
1047 ver = readl(iommu->reg + DMAR_VER_REG);
1048 pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1049 iommu->seq_id,
1050 (unsigned long long)drhd->reg_base_addr,
1051 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1052 (unsigned long long)iommu->cap,
1053 (unsigned long long)iommu->ecap);
1054
1055 /* Reflect status in gcmd */
1056 sts = readl(iommu->reg + DMAR_GSTS_REG);
1057 if (sts & DMA_GSTS_IRES)
1058 iommu->gcmd |= DMA_GCMD_IRE;
1059 if (sts & DMA_GSTS_TES)
1060 iommu->gcmd |= DMA_GCMD_TE;
1061 if (sts & DMA_GSTS_QIES)
1062 iommu->gcmd |= DMA_GCMD_QIE;
1063
1064 raw_spin_lock_init(&iommu->register_lock);
1065
1066 drhd->iommu = iommu;
1067
1068 if (intel_iommu_enabled)
1069 iommu->iommu_dev = iommu_device_create(NULL, iommu,
1070 intel_iommu_groups,
1071 iommu->name);
1072
1073 return 0;
1074
1075 err_unmap:
1076 unmap_iommu(iommu);
1077 error_free_seq_id:
1078 dmar_free_seq_id(iommu);
1079 error:
1080 kfree(iommu);
1081 return err;
1082 }
1083
1084 static void free_iommu(struct intel_iommu *iommu)
1085 {
1086 iommu_device_destroy(iommu->iommu_dev);
1087
1088 if (iommu->irq) {
1089 free_irq(iommu->irq, iommu);
1090 irq_set_handler_data(iommu->irq, NULL);
1091 dmar_free_hwirq(iommu->irq);
1092 }
1093
1094 if (iommu->qi) {
1095 free_page((unsigned long)iommu->qi->desc);
1096 kfree(iommu->qi->desc_status);
1097 kfree(iommu->qi);
1098 }
1099
1100 if (iommu->reg)
1101 unmap_iommu(iommu);
1102
1103 dmar_free_seq_id(iommu);
1104 kfree(iommu);
1105 }
1106
1107 /*
1108 * Reclaim all the submitted descriptors which have completed its work.
1109 */
1110 static inline void reclaim_free_desc(struct q_inval *qi)
1111 {
1112 while (qi->desc_status[qi->free_tail] == QI_DONE ||
1113 qi->desc_status[qi->free_tail] == QI_ABORT) {
1114 qi->desc_status[qi->free_tail] = QI_FREE;
1115 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1116 qi->free_cnt++;
1117 }
1118 }
1119
1120 static int qi_check_fault(struct intel_iommu *iommu, int index)
1121 {
1122 u32 fault;
1123 int head, tail;
1124 struct q_inval *qi = iommu->qi;
1125 int wait_index = (index + 1) % QI_LENGTH;
1126
1127 if (qi->desc_status[wait_index] == QI_ABORT)
1128 return -EAGAIN;
1129
1130 fault = readl(iommu->reg + DMAR_FSTS_REG);
1131
1132 /*
1133 * If IQE happens, the head points to the descriptor associated
1134 * with the error. No new descriptors are fetched until the IQE
1135 * is cleared.
1136 */
1137 if (fault & DMA_FSTS_IQE) {
1138 head = readl(iommu->reg + DMAR_IQH_REG);
1139 if ((head >> DMAR_IQ_SHIFT) == index) {
1140 pr_err("VT-d detected invalid descriptor: "
1141 "low=%llx, high=%llx\n",
1142 (unsigned long long)qi->desc[index].low,
1143 (unsigned long long)qi->desc[index].high);
1144 memcpy(&qi->desc[index], &qi->desc[wait_index],
1145 sizeof(struct qi_desc));
1146 __iommu_flush_cache(iommu, &qi->desc[index],
1147 sizeof(struct qi_desc));
1148 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1149 return -EINVAL;
1150 }
1151 }
1152
1153 /*
1154 * If ITE happens, all pending wait_desc commands are aborted.
1155 * No new descriptors are fetched until the ITE is cleared.
1156 */
1157 if (fault & DMA_FSTS_ITE) {
1158 head = readl(iommu->reg + DMAR_IQH_REG);
1159 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1160 head |= 1;
1161 tail = readl(iommu->reg + DMAR_IQT_REG);
1162 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1163
1164 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1165
1166 do {
1167 if (qi->desc_status[head] == QI_IN_USE)
1168 qi->desc_status[head] = QI_ABORT;
1169 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1170 } while (head != tail);
1171
1172 if (qi->desc_status[wait_index] == QI_ABORT)
1173 return -EAGAIN;
1174 }
1175
1176 if (fault & DMA_FSTS_ICE)
1177 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1178
1179 return 0;
1180 }
1181
1182 /*
1183 * Submit the queued invalidation descriptor to the remapping
1184 * hardware unit and wait for its completion.
1185 */
1186 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1187 {
1188 int rc;
1189 struct q_inval *qi = iommu->qi;
1190 struct qi_desc *hw, wait_desc;
1191 int wait_index, index;
1192 unsigned long flags;
1193
1194 if (!qi)
1195 return 0;
1196
1197 hw = qi->desc;
1198
1199 restart:
1200 rc = 0;
1201
1202 raw_spin_lock_irqsave(&qi->q_lock, flags);
1203 while (qi->free_cnt < 3) {
1204 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1205 cpu_relax();
1206 raw_spin_lock_irqsave(&qi->q_lock, flags);
1207 }
1208
1209 index = qi->free_head;
1210 wait_index = (index + 1) % QI_LENGTH;
1211
1212 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1213
1214 hw[index] = *desc;
1215
1216 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1217 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1218 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1219
1220 hw[wait_index] = wait_desc;
1221
1222 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
1223 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
1224
1225 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1226 qi->free_cnt -= 2;
1227
1228 /*
1229 * update the HW tail register indicating the presence of
1230 * new descriptors.
1231 */
1232 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
1233
1234 while (qi->desc_status[wait_index] != QI_DONE) {
1235 /*
1236 * We will leave the interrupts disabled, to prevent interrupt
1237 * context to queue another cmd while a cmd is already submitted
1238 * and waiting for completion on this cpu. This is to avoid
1239 * a deadlock where the interrupt context can wait indefinitely
1240 * for free slots in the queue.
1241 */
1242 rc = qi_check_fault(iommu, index);
1243 if (rc)
1244 break;
1245
1246 raw_spin_unlock(&qi->q_lock);
1247 cpu_relax();
1248 raw_spin_lock(&qi->q_lock);
1249 }
1250
1251 qi->desc_status[index] = QI_DONE;
1252
1253 reclaim_free_desc(qi);
1254 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1255
1256 if (rc == -EAGAIN)
1257 goto restart;
1258
1259 return rc;
1260 }
1261
1262 /*
1263 * Flush the global interrupt entry cache.
1264 */
1265 void qi_global_iec(struct intel_iommu *iommu)
1266 {
1267 struct qi_desc desc;
1268
1269 desc.low = QI_IEC_TYPE;
1270 desc.high = 0;
1271
1272 /* should never fail */
1273 qi_submit_sync(&desc, iommu);
1274 }
1275
1276 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1277 u64 type)
1278 {
1279 struct qi_desc desc;
1280
1281 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1282 | QI_CC_GRAN(type) | QI_CC_TYPE;
1283 desc.high = 0;
1284
1285 qi_submit_sync(&desc, iommu);
1286 }
1287
1288 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1289 unsigned int size_order, u64 type)
1290 {
1291 u8 dw = 0, dr = 0;
1292
1293 struct qi_desc desc;
1294 int ih = 0;
1295
1296 if (cap_write_drain(iommu->cap))
1297 dw = 1;
1298
1299 if (cap_read_drain(iommu->cap))
1300 dr = 1;
1301
1302 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1303 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1304 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1305 | QI_IOTLB_AM(size_order);
1306
1307 qi_submit_sync(&desc, iommu);
1308 }
1309
1310 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1311 u64 addr, unsigned mask)
1312 {
1313 struct qi_desc desc;
1314
1315 if (mask) {
1316 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1317 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1318 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1319 } else
1320 desc.high = QI_DEV_IOTLB_ADDR(addr);
1321
1322 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1323 qdep = 0;
1324
1325 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1326 QI_DIOTLB_TYPE;
1327
1328 qi_submit_sync(&desc, iommu);
1329 }
1330
1331 /*
1332 * Disable Queued Invalidation interface.
1333 */
1334 void dmar_disable_qi(struct intel_iommu *iommu)
1335 {
1336 unsigned long flags;
1337 u32 sts;
1338 cycles_t start_time = get_cycles();
1339
1340 if (!ecap_qis(iommu->ecap))
1341 return;
1342
1343 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1344
1345 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
1346 if (!(sts & DMA_GSTS_QIES))
1347 goto end;
1348
1349 /*
1350 * Give a chance to HW to complete the pending invalidation requests.
1351 */
1352 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1353 readl(iommu->reg + DMAR_IQH_REG)) &&
1354 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1355 cpu_relax();
1356
1357 iommu->gcmd &= ~DMA_GCMD_QIE;
1358 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1359
1360 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1361 !(sts & DMA_GSTS_QIES), sts);
1362 end:
1363 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1364 }
1365
1366 /*
1367 * Enable queued invalidation.
1368 */
1369 static void __dmar_enable_qi(struct intel_iommu *iommu)
1370 {
1371 u32 sts;
1372 unsigned long flags;
1373 struct q_inval *qi = iommu->qi;
1374
1375 qi->free_head = qi->free_tail = 0;
1376 qi->free_cnt = QI_LENGTH;
1377
1378 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1379
1380 /* write zero to the tail reg */
1381 writel(0, iommu->reg + DMAR_IQT_REG);
1382
1383 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1384
1385 iommu->gcmd |= DMA_GCMD_QIE;
1386 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1387
1388 /* Make sure hardware complete it */
1389 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1390
1391 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1392 }
1393
1394 /*
1395 * Enable Queued Invalidation interface. This is a must to support
1396 * interrupt-remapping. Also used by DMA-remapping, which replaces
1397 * register based IOTLB invalidation.
1398 */
1399 int dmar_enable_qi(struct intel_iommu *iommu)
1400 {
1401 struct q_inval *qi;
1402 struct page *desc_page;
1403
1404 if (!ecap_qis(iommu->ecap))
1405 return -ENOENT;
1406
1407 /*
1408 * queued invalidation is already setup and enabled.
1409 */
1410 if (iommu->qi)
1411 return 0;
1412
1413 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1414 if (!iommu->qi)
1415 return -ENOMEM;
1416
1417 qi = iommu->qi;
1418
1419
1420 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1421 if (!desc_page) {
1422 kfree(qi);
1423 iommu->qi = NULL;
1424 return -ENOMEM;
1425 }
1426
1427 qi->desc = page_address(desc_page);
1428
1429 qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1430 if (!qi->desc_status) {
1431 free_page((unsigned long) qi->desc);
1432 kfree(qi);
1433 iommu->qi = NULL;
1434 return -ENOMEM;
1435 }
1436
1437 raw_spin_lock_init(&qi->q_lock);
1438
1439 __dmar_enable_qi(iommu);
1440
1441 return 0;
1442 }
1443
1444 /* iommu interrupt handling. Most stuff are MSI-like. */
1445
1446 enum faulttype {
1447 DMA_REMAP,
1448 INTR_REMAP,
1449 UNKNOWN,
1450 };
1451
1452 static const char *dma_remap_fault_reasons[] =
1453 {
1454 "Software",
1455 "Present bit in root entry is clear",
1456 "Present bit in context entry is clear",
1457 "Invalid context entry",
1458 "Access beyond MGAW",
1459 "PTE Write access is not set",
1460 "PTE Read access is not set",
1461 "Next page table ptr is invalid",
1462 "Root table address invalid",
1463 "Context table ptr is invalid",
1464 "non-zero reserved fields in RTP",
1465 "non-zero reserved fields in CTP",
1466 "non-zero reserved fields in PTE",
1467 "PCE for translation request specifies blocking",
1468 };
1469
1470 static const char *irq_remap_fault_reasons[] =
1471 {
1472 "Detected reserved fields in the decoded interrupt-remapped request",
1473 "Interrupt index exceeded the interrupt-remapping table size",
1474 "Present field in the IRTE entry is clear",
1475 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1476 "Detected reserved fields in the IRTE entry",
1477 "Blocked a compatibility format interrupt request",
1478 "Blocked an interrupt request due to source-id verification failure",
1479 };
1480
1481 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1482 {
1483 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1484 ARRAY_SIZE(irq_remap_fault_reasons))) {
1485 *fault_type = INTR_REMAP;
1486 return irq_remap_fault_reasons[fault_reason - 0x20];
1487 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1488 *fault_type = DMA_REMAP;
1489 return dma_remap_fault_reasons[fault_reason];
1490 } else {
1491 *fault_type = UNKNOWN;
1492 return "Unknown";
1493 }
1494 }
1495
1496 void dmar_msi_unmask(struct irq_data *data)
1497 {
1498 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1499 unsigned long flag;
1500
1501 /* unmask it */
1502 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1503 writel(0, iommu->reg + DMAR_FECTL_REG);
1504 /* Read a reg to force flush the post write */
1505 readl(iommu->reg + DMAR_FECTL_REG);
1506 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1507 }
1508
1509 void dmar_msi_mask(struct irq_data *data)
1510 {
1511 unsigned long flag;
1512 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1513
1514 /* mask it */
1515 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1516 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1517 /* Read a reg to force flush the post write */
1518 readl(iommu->reg + DMAR_FECTL_REG);
1519 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1520 }
1521
1522 void dmar_msi_write(int irq, struct msi_msg *msg)
1523 {
1524 struct intel_iommu *iommu = irq_get_handler_data(irq);
1525 unsigned long flag;
1526
1527 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1528 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1529 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1530 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1531 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1532 }
1533
1534 void dmar_msi_read(int irq, struct msi_msg *msg)
1535 {
1536 struct intel_iommu *iommu = irq_get_handler_data(irq);
1537 unsigned long flag;
1538
1539 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1540 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1541 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1542 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1543 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1544 }
1545
1546 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1547 u8 fault_reason, u16 source_id, unsigned long long addr)
1548 {
1549 const char *reason;
1550 int fault_type;
1551
1552 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1553
1554 if (fault_type == INTR_REMAP)
1555 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1556 "fault index %llx\n"
1557 "INTR-REMAP:[fault reason %02d] %s\n",
1558 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1559 PCI_FUNC(source_id & 0xFF), addr >> 48,
1560 fault_reason, reason);
1561 else
1562 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1563 "fault addr %llx \n"
1564 "DMAR:[fault reason %02d] %s\n",
1565 (type ? "DMA Read" : "DMA Write"),
1566 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1567 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1568 return 0;
1569 }
1570
1571 #define PRIMARY_FAULT_REG_LEN (16)
1572 irqreturn_t dmar_fault(int irq, void *dev_id)
1573 {
1574 struct intel_iommu *iommu = dev_id;
1575 int reg, fault_index;
1576 u32 fault_status;
1577 unsigned long flag;
1578
1579 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1580 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1581 if (fault_status)
1582 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1583
1584 /* TBD: ignore advanced fault log currently */
1585 if (!(fault_status & DMA_FSTS_PPF))
1586 goto unlock_exit;
1587
1588 fault_index = dma_fsts_fault_record_index(fault_status);
1589 reg = cap_fault_reg_offset(iommu->cap);
1590 while (1) {
1591 u8 fault_reason;
1592 u16 source_id;
1593 u64 guest_addr;
1594 int type;
1595 u32 data;
1596
1597 /* highest 32 bits */
1598 data = readl(iommu->reg + reg +
1599 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1600 if (!(data & DMA_FRCD_F))
1601 break;
1602
1603 fault_reason = dma_frcd_fault_reason(data);
1604 type = dma_frcd_type(data);
1605
1606 data = readl(iommu->reg + reg +
1607 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1608 source_id = dma_frcd_source_id(data);
1609
1610 guest_addr = dmar_readq(iommu->reg + reg +
1611 fault_index * PRIMARY_FAULT_REG_LEN);
1612 guest_addr = dma_frcd_page_addr(guest_addr);
1613 /* clear the fault */
1614 writel(DMA_FRCD_F, iommu->reg + reg +
1615 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1616
1617 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1618
1619 dmar_fault_do_one(iommu, type, fault_reason,
1620 source_id, guest_addr);
1621
1622 fault_index++;
1623 if (fault_index >= cap_num_fault_regs(iommu->cap))
1624 fault_index = 0;
1625 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1626 }
1627
1628 writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1629
1630 unlock_exit:
1631 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1632 return IRQ_HANDLED;
1633 }
1634
1635 int dmar_set_interrupt(struct intel_iommu *iommu)
1636 {
1637 int irq, ret;
1638
1639 /*
1640 * Check if the fault interrupt is already initialized.
1641 */
1642 if (iommu->irq)
1643 return 0;
1644
1645 irq = dmar_alloc_hwirq();
1646 if (irq <= 0) {
1647 pr_err("IOMMU: no free vectors\n");
1648 return -EINVAL;
1649 }
1650
1651 irq_set_handler_data(irq, iommu);
1652 iommu->irq = irq;
1653
1654 ret = arch_setup_dmar_msi(irq);
1655 if (ret) {
1656 irq_set_handler_data(irq, NULL);
1657 iommu->irq = 0;
1658 dmar_free_hwirq(irq);
1659 return ret;
1660 }
1661
1662 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1663 if (ret)
1664 pr_err("IOMMU: can't request irq\n");
1665 return ret;
1666 }
1667
1668 int __init enable_drhd_fault_handling(void)
1669 {
1670 struct dmar_drhd_unit *drhd;
1671 struct intel_iommu *iommu;
1672
1673 /*
1674 * Enable fault control interrupt.
1675 */
1676 for_each_iommu(iommu, drhd) {
1677 u32 fault_status;
1678 int ret = dmar_set_interrupt(iommu);
1679
1680 if (ret) {
1681 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1682 (unsigned long long)drhd->reg_base_addr, ret);
1683 return -1;
1684 }
1685
1686 /*
1687 * Clear any previous faults.
1688 */
1689 dmar_fault(iommu->irq, iommu);
1690 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1691 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1692 }
1693
1694 return 0;
1695 }
1696
1697 /*
1698 * Re-enable Queued Invalidation interface.
1699 */
1700 int dmar_reenable_qi(struct intel_iommu *iommu)
1701 {
1702 if (!ecap_qis(iommu->ecap))
1703 return -ENOENT;
1704
1705 if (!iommu->qi)
1706 return -ENOENT;
1707
1708 /*
1709 * First disable queued invalidation.
1710 */
1711 dmar_disable_qi(iommu);
1712 /*
1713 * Then enable queued invalidation again. Since there is no pending
1714 * invalidation requests now, it's safe to re-enable queued
1715 * invalidation.
1716 */
1717 __dmar_enable_qi(iommu);
1718
1719 return 0;
1720 }
1721
1722 /*
1723 * Check interrupt remapping support in DMAR table description.
1724 */
1725 int __init dmar_ir_support(void)
1726 {
1727 struct acpi_table_dmar *dmar;
1728 dmar = (struct acpi_table_dmar *)dmar_tbl;
1729 if (!dmar)
1730 return 0;
1731 return dmar->flags & 0x1;
1732 }
1733
1734 /* Check whether DMAR units are in use */
1735 static inline bool dmar_in_use(void)
1736 {
1737 return irq_remapping_enabled || intel_iommu_enabled;
1738 }
1739
1740 static int __init dmar_free_unused_resources(void)
1741 {
1742 struct dmar_drhd_unit *dmaru, *dmaru_n;
1743
1744 if (dmar_in_use())
1745 return 0;
1746
1747 if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1748 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1749
1750 down_write(&dmar_global_lock);
1751 list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1752 list_del(&dmaru->list);
1753 dmar_free_drhd(dmaru);
1754 }
1755 up_write(&dmar_global_lock);
1756
1757 return 0;
1758 }
1759
1760 late_initcall(dmar_free_unused_resources);
1761 IOMMU_INIT_POST(detect_intel_iommu);
1762
1763 /*
1764 * DMAR Hotplug Support
1765 * For more details, please refer to Intel(R) Virtualization Technology
1766 * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
1767 * "Remapping Hardware Unit Hot Plug".
1768 */
1769 static u8 dmar_hp_uuid[] = {
1770 /* 0000 */ 0xA6, 0xA3, 0xC1, 0xD8, 0x9B, 0xBE, 0x9B, 0x4C,
1771 /* 0008 */ 0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF
1772 };
1773
1774 /*
1775 * Currently there's only one revision and BIOS will not check the revision id,
1776 * so use 0 for safety.
1777 */
1778 #define DMAR_DSM_REV_ID 0
1779 #define DMAR_DSM_FUNC_DRHD 1
1780 #define DMAR_DSM_FUNC_ATSR 2
1781 #define DMAR_DSM_FUNC_RHSA 3
1782
1783 static inline bool dmar_detect_dsm(acpi_handle handle, int func)
1784 {
1785 return acpi_check_dsm(handle, dmar_hp_uuid, DMAR_DSM_REV_ID, 1 << func);
1786 }
1787
1788 static int dmar_walk_dsm_resource(acpi_handle handle, int func,
1789 dmar_res_handler_t handler, void *arg)
1790 {
1791 int ret = -ENODEV;
1792 union acpi_object *obj;
1793 struct acpi_dmar_header *start;
1794 struct dmar_res_callback callback;
1795 static int res_type[] = {
1796 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
1797 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
1798 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
1799 };
1800
1801 if (!dmar_detect_dsm(handle, func))
1802 return 0;
1803
1804 obj = acpi_evaluate_dsm_typed(handle, dmar_hp_uuid, DMAR_DSM_REV_ID,
1805 func, NULL, ACPI_TYPE_BUFFER);
1806 if (!obj)
1807 return -ENODEV;
1808
1809 memset(&callback, 0, sizeof(callback));
1810 callback.cb[res_type[func]] = handler;
1811 callback.arg[res_type[func]] = arg;
1812 start = (struct acpi_dmar_header *)obj->buffer.pointer;
1813 ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
1814
1815 ACPI_FREE(obj);
1816
1817 return ret;
1818 }
1819
1820 static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
1821 {
1822 int ret;
1823 struct dmar_drhd_unit *dmaru;
1824
1825 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1826 if (!dmaru)
1827 return -ENODEV;
1828
1829 ret = dmar_ir_hotplug(dmaru, true);
1830 if (ret == 0)
1831 ret = dmar_iommu_hotplug(dmaru, true);
1832
1833 return ret;
1834 }
1835
1836 static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
1837 {
1838 int i, ret;
1839 struct device *dev;
1840 struct dmar_drhd_unit *dmaru;
1841
1842 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1843 if (!dmaru)
1844 return 0;
1845
1846 /*
1847 * All PCI devices managed by this unit should have been destroyed.
1848 */
1849 if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt)
1850 for_each_active_dev_scope(dmaru->devices,
1851 dmaru->devices_cnt, i, dev)
1852 return -EBUSY;
1853
1854 ret = dmar_ir_hotplug(dmaru, false);
1855 if (ret == 0)
1856 ret = dmar_iommu_hotplug(dmaru, false);
1857
1858 return ret;
1859 }
1860
1861 static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
1862 {
1863 struct dmar_drhd_unit *dmaru;
1864
1865 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1866 if (dmaru) {
1867 list_del_rcu(&dmaru->list);
1868 synchronize_rcu();
1869 dmar_free_drhd(dmaru);
1870 }
1871
1872 return 0;
1873 }
1874
1875 static int dmar_hotplug_insert(acpi_handle handle)
1876 {
1877 int ret;
1878 int drhd_count = 0;
1879
1880 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1881 &dmar_validate_one_drhd, (void *)1);
1882 if (ret)
1883 goto out;
1884
1885 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1886 &dmar_parse_one_drhd, (void *)&drhd_count);
1887 if (ret == 0 && drhd_count == 0) {
1888 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
1889 goto out;
1890 } else if (ret) {
1891 goto release_drhd;
1892 }
1893
1894 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
1895 &dmar_parse_one_rhsa, NULL);
1896 if (ret)
1897 goto release_drhd;
1898
1899 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1900 &dmar_parse_one_atsr, NULL);
1901 if (ret)
1902 goto release_atsr;
1903
1904 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1905 &dmar_hp_add_drhd, NULL);
1906 if (!ret)
1907 return 0;
1908
1909 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1910 &dmar_hp_remove_drhd, NULL);
1911 release_atsr:
1912 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1913 &dmar_release_one_atsr, NULL);
1914 release_drhd:
1915 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1916 &dmar_hp_release_drhd, NULL);
1917 out:
1918 return ret;
1919 }
1920
1921 static int dmar_hotplug_remove(acpi_handle handle)
1922 {
1923 int ret;
1924
1925 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1926 &dmar_check_one_atsr, NULL);
1927 if (ret)
1928 return ret;
1929
1930 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1931 &dmar_hp_remove_drhd, NULL);
1932 if (ret == 0) {
1933 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1934 &dmar_release_one_atsr, NULL));
1935 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1936 &dmar_hp_release_drhd, NULL));
1937 } else {
1938 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1939 &dmar_hp_add_drhd, NULL);
1940 }
1941
1942 return ret;
1943 }
1944
1945 static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
1946 void *context, void **retval)
1947 {
1948 acpi_handle *phdl = retval;
1949
1950 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1951 *phdl = handle;
1952 return AE_CTRL_TERMINATE;
1953 }
1954
1955 return AE_OK;
1956 }
1957
1958 static int dmar_device_hotplug(acpi_handle handle, bool insert)
1959 {
1960 int ret;
1961 acpi_handle tmp = NULL;
1962 acpi_status status;
1963
1964 if (!dmar_in_use())
1965 return 0;
1966
1967 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1968 tmp = handle;
1969 } else {
1970 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
1971 ACPI_UINT32_MAX,
1972 dmar_get_dsm_handle,
1973 NULL, NULL, &tmp);
1974 if (ACPI_FAILURE(status)) {
1975 pr_warn("Failed to locate _DSM method.\n");
1976 return -ENXIO;
1977 }
1978 }
1979 if (tmp == NULL)
1980 return 0;
1981
1982 down_write(&dmar_global_lock);
1983 if (insert)
1984 ret = dmar_hotplug_insert(tmp);
1985 else
1986 ret = dmar_hotplug_remove(tmp);
1987 up_write(&dmar_global_lock);
1988
1989 return ret;
1990 }
1991
1992 int dmar_device_add(acpi_handle handle)
1993 {
1994 return dmar_device_hotplug(handle, true);
1995 }
1996
1997 int dmar_device_remove(acpi_handle handle)
1998 {
1999 return dmar_device_hotplug(handle, false);
2000 }
Linux with added FPC-III support