?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
USB: EHCI: fix compiler warning introduced by commit 2a40f324541e
[linux/fpc-iii.git] / drivers / iommu / dmar.c
blob62a400c5ba0614fde00d18ad2207b430d99433ee
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 *
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */
28
29 #define pr_fmt(fmt) "DMAR: " fmt
30
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/iova.h>
34 #include <linux/intel-iommu.h>
35 #include <linux/timer.h>
36 #include <linux/irq.h>
37 #include <linux/interrupt.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/slab.h>
41 #include <linux/iommu.h>
42 #include <asm/irq_remapping.h>
43 #include <asm/iommu_table.h>
44
45 #include "irq_remapping.h"
46
47 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
48 struct dmar_res_callback {
49 dmar_res_handler_t cb[ACPI_DMAR_TYPE_RESERVED];
50 void *arg[ACPI_DMAR_TYPE_RESERVED];
51 bool ignore_unhandled;
52 bool print_entry;
53 };
54
55 /*
56 * Assumptions:
57 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
58 * before IO devices managed by that unit.
59 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
60 * after IO devices managed by that unit.
61 * 3) Hotplug events are rare.
62 *
63 * Locking rules for DMA and interrupt remapping related global data structures:
64 * 1) Use dmar_global_lock in process context
65 * 2) Use RCU in interrupt context
66 */
67 DECLARE_RWSEM(dmar_global_lock);
68 LIST_HEAD(dmar_drhd_units);
69
70 struct acpi_table_header * __initdata dmar_tbl;
71 static acpi_size dmar_tbl_size;
72 static int dmar_dev_scope_status = 1;
73 static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
74
75 static int alloc_iommu(struct dmar_drhd_unit *drhd);
76 static void free_iommu(struct intel_iommu *iommu);
77
78 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
79 {
80 /*
81 * add INCLUDE_ALL at the tail, so scan the list will find it at
82 * the very end.
83 */
84 if (drhd->include_all)
85 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
86 else
87 list_add_rcu(&drhd->list, &dmar_drhd_units);
88 }
89
90 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
91 {
92 struct acpi_dmar_device_scope *scope;
93
94 *cnt = 0;
95 while (start < end) {
96 scope = start;
97 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
98 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
99 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
100 (*cnt)++;
101 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
102 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
103 pr_warn("Unsupported device scope\n");
104 }
105 start += scope->length;
106 }
107 if (*cnt == 0)
108 return NULL;
109
110 return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
111 }
112
113 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
114 {
115 int i;
116 struct device *tmp_dev;
117
118 if (*devices && *cnt) {
119 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
120 put_device(tmp_dev);
121 kfree(*devices);
122 }
123
124 *devices = NULL;
125 *cnt = 0;
126 }
127
128 /* Optimize out kzalloc()/kfree() for normal cases */
129 static char dmar_pci_notify_info_buf[64];
130
131 static struct dmar_pci_notify_info *
132 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
133 {
134 int level = 0;
135 size_t size;
136 struct pci_dev *tmp;
137 struct dmar_pci_notify_info *info;
138
139 BUG_ON(dev->is_virtfn);
140
141 /* Only generate path[] for device addition event */
142 if (event == BUS_NOTIFY_ADD_DEVICE)
143 for (tmp = dev; tmp; tmp = tmp->bus->self)
144 level++;
145
146 size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
147 if (size <= sizeof(dmar_pci_notify_info_buf)) {
148 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
149 } else {
150 info = kzalloc(size, GFP_KERNEL);
151 if (!info) {
152 pr_warn("Out of memory when allocating notify_info "
153 "for %s.\n", pci_name(dev));
154 if (dmar_dev_scope_status == 0)
155 dmar_dev_scope_status = -ENOMEM;
156 return NULL;
157 }
158 }
159
160 info->event = event;
161 info->dev = dev;
162 info->seg = pci_domain_nr(dev->bus);
163 info->level = level;
164 if (event == BUS_NOTIFY_ADD_DEVICE) {
165 for (tmp = dev; tmp; tmp = tmp->bus->self) {
166 level--;
167 info->path[level].bus = tmp->bus->number;
168 info->path[level].device = PCI_SLOT(tmp->devfn);
169 info->path[level].function = PCI_FUNC(tmp->devfn);
170 if (pci_is_root_bus(tmp->bus))
171 info->bus = tmp->bus->number;
172 }
173 }
174
175 return info;
176 }
177
178 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
179 {
180 if ((void *)info != dmar_pci_notify_info_buf)
181 kfree(info);
182 }
183
184 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
185 struct acpi_dmar_pci_path *path, int count)
186 {
187 int i;
188
189 if (info->bus != bus)
190 goto fallback;
191 if (info->level != count)
192 goto fallback;
193
194 for (i = 0; i < count; i++) {
195 if (path[i].device != info->path[i].device ||
196 path[i].function != info->path[i].function)
197 goto fallback;
198 }
199
200 return true;
201
202 fallback:
203
204 if (count != 1)
205 return false;
206
207 i = info->level - 1;
208 if (bus == info->path[i].bus &&
209 path[0].device == info->path[i].device &&
210 path[0].function == info->path[i].function) {
211 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
212 bus, path[0].device, path[0].function);
213 return true;
214 }
215
216 return false;
217 }
218
219 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
220 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
221 void *start, void*end, u16 segment,
222 struct dmar_dev_scope *devices,
223 int devices_cnt)
224 {
225 int i, level;
226 struct device *tmp, *dev = &info->dev->dev;
227 struct acpi_dmar_device_scope *scope;
228 struct acpi_dmar_pci_path *path;
229
230 if (segment != info->seg)
231 return 0;
232
233 for (; start < end; start += scope->length) {
234 scope = start;
235 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
236 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
237 continue;
238
239 path = (struct acpi_dmar_pci_path *)(scope + 1);
240 level = (scope->length - sizeof(*scope)) / sizeof(*path);
241 if (!dmar_match_pci_path(info, scope->bus, path, level))
242 continue;
243
244 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^
245 (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) {
246 pr_warn("Device scope type does not match for %s\n",
247 pci_name(info->dev));
248 return -EINVAL;
249 }
250
251 for_each_dev_scope(devices, devices_cnt, i, tmp)
252 if (tmp == NULL) {
253 devices[i].bus = info->dev->bus->number;
254 devices[i].devfn = info->dev->devfn;
255 rcu_assign_pointer(devices[i].dev,
256 get_device(dev));
257 return 1;
258 }
259 BUG_ON(i >= devices_cnt);
260 }
261
262 return 0;
263 }
264
265 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
266 struct dmar_dev_scope *devices, int count)
267 {
268 int index;
269 struct device *tmp;
270
271 if (info->seg != segment)
272 return 0;
273
274 for_each_active_dev_scope(devices, count, index, tmp)
275 if (tmp == &info->dev->dev) {
276 RCU_INIT_POINTER(devices[index].dev, NULL);
277 synchronize_rcu();
278 put_device(tmp);
279 return 1;
280 }
281
282 return 0;
283 }
284
285 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
286 {
287 int ret = 0;
288 struct dmar_drhd_unit *dmaru;
289 struct acpi_dmar_hardware_unit *drhd;
290
291 for_each_drhd_unit(dmaru) {
292 if (dmaru->include_all)
293 continue;
294
295 drhd = container_of(dmaru->hdr,
296 struct acpi_dmar_hardware_unit, header);
297 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
298 ((void *)drhd) + drhd->header.length,
299 dmaru->segment,
300 dmaru->devices, dmaru->devices_cnt);
301 if (ret != 0)
302 break;
303 }
304 if (ret >= 0)
305 ret = dmar_iommu_notify_scope_dev(info);
306 if (ret < 0 && dmar_dev_scope_status == 0)
307 dmar_dev_scope_status = ret;
308
309 return ret;
310 }
311
312 static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
313 {
314 struct dmar_drhd_unit *dmaru;
315
316 for_each_drhd_unit(dmaru)
317 if (dmar_remove_dev_scope(info, dmaru->segment,
318 dmaru->devices, dmaru->devices_cnt))
319 break;
320 dmar_iommu_notify_scope_dev(info);
321 }
322
323 static int dmar_pci_bus_notifier(struct notifier_block *nb,
324 unsigned long action, void *data)
325 {
326 struct pci_dev *pdev = to_pci_dev(data);
327 struct dmar_pci_notify_info *info;
328
329 /* Only care about add/remove events for physical functions */
330 if (pdev->is_virtfn)
331 return NOTIFY_DONE;
332 if (action != BUS_NOTIFY_ADD_DEVICE && action != BUS_NOTIFY_DEL_DEVICE)
333 return NOTIFY_DONE;
334
335 info = dmar_alloc_pci_notify_info(pdev, action);
336 if (!info)
337 return NOTIFY_DONE;
338
339 down_write(&dmar_global_lock);
340 if (action == BUS_NOTIFY_ADD_DEVICE)
341 dmar_pci_bus_add_dev(info);
342 else if (action == BUS_NOTIFY_DEL_DEVICE)
343 dmar_pci_bus_del_dev(info);
344 up_write(&dmar_global_lock);
345
346 dmar_free_pci_notify_info(info);
347
348 return NOTIFY_OK;
349 }
350
351 static struct notifier_block dmar_pci_bus_nb = {
352 .notifier_call = dmar_pci_bus_notifier,
353 .priority = INT_MIN,
354 };
355
356 static struct dmar_drhd_unit *
357 dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
358 {
359 struct dmar_drhd_unit *dmaru;
360
361 list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list)
362 if (dmaru->segment == drhd->segment &&
363 dmaru->reg_base_addr == drhd->address)
364 return dmaru;
365
366 return NULL;
367 }
368
369 /**
370 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
371 * structure which uniquely represent one DMA remapping hardware unit
372 * present in the platform
373 */
374 static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
375 {
376 struct acpi_dmar_hardware_unit *drhd;
377 struct dmar_drhd_unit *dmaru;
378 int ret = 0;
379
380 drhd = (struct acpi_dmar_hardware_unit *)header;
381 dmaru = dmar_find_dmaru(drhd);
382 if (dmaru)
383 goto out;
384
385 dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
386 if (!dmaru)
387 return -ENOMEM;
388
389 /*
390 * If header is allocated from slab by ACPI _DSM method, we need to
391 * copy the content because the memory buffer will be freed on return.
392 */
393 dmaru->hdr = (void *)(dmaru + 1);
394 memcpy(dmaru->hdr, header, header->length);
395 dmaru->reg_base_addr = drhd->address;
396 dmaru->segment = drhd->segment;
397 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
398 dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
399 ((void *)drhd) + drhd->header.length,
400 &dmaru->devices_cnt);
401 if (dmaru->devices_cnt && dmaru->devices == NULL) {
402 kfree(dmaru);
403 return -ENOMEM;
404 }
405
406 ret = alloc_iommu(dmaru);
407 if (ret) {
408 dmar_free_dev_scope(&dmaru->devices,
409 &dmaru->devices_cnt);
410 kfree(dmaru);
411 return ret;
412 }
413 dmar_register_drhd_unit(dmaru);
414
415 out:
416 if (arg)
417 (*(int *)arg)++;
418
419 return 0;
420 }
421
422 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
423 {
424 if (dmaru->devices && dmaru->devices_cnt)
425 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
426 if (dmaru->iommu)
427 free_iommu(dmaru->iommu);
428 kfree(dmaru);
429 }
430
431 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
432 void *arg)
433 {
434 struct acpi_dmar_andd *andd = (void *)header;
435
436 /* Check for NUL termination within the designated length */
437 if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
438 WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
439 "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
440 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
441 dmi_get_system_info(DMI_BIOS_VENDOR),
442 dmi_get_system_info(DMI_BIOS_VERSION),
443 dmi_get_system_info(DMI_PRODUCT_VERSION));
444 return -EINVAL;
445 }
446 pr_info("ANDD device: %x name: %s\n", andd->device_number,
447 andd->device_name);
448
449 return 0;
450 }
451
452 #ifdef CONFIG_ACPI_NUMA
453 static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
454 {
455 struct acpi_dmar_rhsa *rhsa;
456 struct dmar_drhd_unit *drhd;
457
458 rhsa = (struct acpi_dmar_rhsa *)header;
459 for_each_drhd_unit(drhd) {
460 if (drhd->reg_base_addr == rhsa->base_address) {
461 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
462
463 if (!node_online(node))
464 node = -1;
465 drhd->iommu->node = node;
466 return 0;
467 }
468 }
469 WARN_TAINT(
470 1, TAINT_FIRMWARE_WORKAROUND,
471 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
472 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
473 drhd->reg_base_addr,
474 dmi_get_system_info(DMI_BIOS_VENDOR),
475 dmi_get_system_info(DMI_BIOS_VERSION),
476 dmi_get_system_info(DMI_PRODUCT_VERSION));
477
478 return 0;
479 }
480 #else
481 #define dmar_parse_one_rhsa dmar_res_noop
482 #endif
483
484 static void __init
485 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
486 {
487 struct acpi_dmar_hardware_unit *drhd;
488 struct acpi_dmar_reserved_memory *rmrr;
489 struct acpi_dmar_atsr *atsr;
490 struct acpi_dmar_rhsa *rhsa;
491
492 switch (header->type) {
493 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
494 drhd = container_of(header, struct acpi_dmar_hardware_unit,
495 header);
496 pr_info("DRHD base: %#016Lx flags: %#x\n",
497 (unsigned long long)drhd->address, drhd->flags);
498 break;
499 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
500 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
501 header);
502 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
503 (unsigned long long)rmrr->base_address,
504 (unsigned long long)rmrr->end_address);
505 break;
506 case ACPI_DMAR_TYPE_ROOT_ATS:
507 atsr = container_of(header, struct acpi_dmar_atsr, header);
508 pr_info("ATSR flags: %#x\n", atsr->flags);
509 break;
510 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
511 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
512 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
513 (unsigned long long)rhsa->base_address,
514 rhsa->proximity_domain);
515 break;
516 case ACPI_DMAR_TYPE_NAMESPACE:
517 /* We don't print this here because we need to sanity-check
518 it first. So print it in dmar_parse_one_andd() instead. */
519 break;
520 }
521 }
522
523 /**
524 * dmar_table_detect - checks to see if the platform supports DMAR devices
525 */
526 static int __init dmar_table_detect(void)
527 {
528 acpi_status status = AE_OK;
529
530 /* if we could find DMAR table, then there are DMAR devices */
531 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
532 (struct acpi_table_header **)&dmar_tbl,
533 &dmar_tbl_size);
534
535 if (ACPI_SUCCESS(status) && !dmar_tbl) {
536 pr_warn("Unable to map DMAR\n");
537 status = AE_NOT_FOUND;
538 }
539
540 return (ACPI_SUCCESS(status) ? 1 : 0);
541 }
542
543 static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
544 size_t len, struct dmar_res_callback *cb)
545 {
546 int ret = 0;
547 struct acpi_dmar_header *iter, *next;
548 struct acpi_dmar_header *end = ((void *)start) + len;
549
550 for (iter = start; iter < end && ret == 0; iter = next) {
551 next = (void *)iter + iter->length;
552 if (iter->length == 0) {
553 /* Avoid looping forever on bad ACPI tables */
554 pr_debug(FW_BUG "Invalid 0-length structure\n");
555 break;
556 } else if (next > end) {
557 /* Avoid passing table end */
558 pr_warn(FW_BUG "Record passes table end\n");
559 ret = -EINVAL;
560 break;
561 }
562
563 if (cb->print_entry)
564 dmar_table_print_dmar_entry(iter);
565
566 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
567 /* continue for forward compatibility */
568 pr_debug("Unknown DMAR structure type %d\n",
569 iter->type);
570 } else if (cb->cb[iter->type]) {
571 ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
572 } else if (!cb->ignore_unhandled) {
573 pr_warn("No handler for DMAR structure type %d\n",
574 iter->type);
575 ret = -EINVAL;
576 }
577 }
578
579 return ret;
580 }
581
582 static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
583 struct dmar_res_callback *cb)
584 {
585 return dmar_walk_remapping_entries((void *)(dmar + 1),
586 dmar->header.length - sizeof(*dmar), cb);
587 }
588
589 /**
590 * parse_dmar_table - parses the DMA reporting table
591 */
592 static int __init
593 parse_dmar_table(void)
594 {
595 struct acpi_table_dmar *dmar;
596 int ret = 0;
597 int drhd_count = 0;
598 struct dmar_res_callback cb = {
599 .print_entry = true,
600 .ignore_unhandled = true,
601 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
602 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
603 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
604 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
605 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
606 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
607 };
608
609 /*
610 * Do it again, earlier dmar_tbl mapping could be mapped with
611 * fixed map.
612 */
613 dmar_table_detect();
614
615 /*
616 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
617 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
618 */
619 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
620
621 dmar = (struct acpi_table_dmar *)dmar_tbl;
622 if (!dmar)
623 return -ENODEV;
624
625 if (dmar->width < PAGE_SHIFT - 1) {
626 pr_warn("Invalid DMAR haw\n");
627 return -EINVAL;
628 }
629
630 pr_info("Host address width %d\n", dmar->width + 1);
631 ret = dmar_walk_dmar_table(dmar, &cb);
632 if (ret == 0 && drhd_count == 0)
633 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
634
635 return ret;
636 }
637
638 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
639 int cnt, struct pci_dev *dev)
640 {
641 int index;
642 struct device *tmp;
643
644 while (dev) {
645 for_each_active_dev_scope(devices, cnt, index, tmp)
646 if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
647 return 1;
648
649 /* Check our parent */
650 dev = dev->bus->self;
651 }
652
653 return 0;
654 }
655
656 struct dmar_drhd_unit *
657 dmar_find_matched_drhd_unit(struct pci_dev *dev)
658 {
659 struct dmar_drhd_unit *dmaru;
660 struct acpi_dmar_hardware_unit *drhd;
661
662 dev = pci_physfn(dev);
663
664 rcu_read_lock();
665 for_each_drhd_unit(dmaru) {
666 drhd = container_of(dmaru->hdr,
667 struct acpi_dmar_hardware_unit,
668 header);
669
670 if (dmaru->include_all &&
671 drhd->segment == pci_domain_nr(dev->bus))
672 goto out;
673
674 if (dmar_pci_device_match(dmaru->devices,
675 dmaru->devices_cnt, dev))
676 goto out;
677 }
678 dmaru = NULL;
679 out:
680 rcu_read_unlock();
681
682 return dmaru;
683 }
684
685 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
686 struct acpi_device *adev)
687 {
688 struct dmar_drhd_unit *dmaru;
689 struct acpi_dmar_hardware_unit *drhd;
690 struct acpi_dmar_device_scope *scope;
691 struct device *tmp;
692 int i;
693 struct acpi_dmar_pci_path *path;
694
695 for_each_drhd_unit(dmaru) {
696 drhd = container_of(dmaru->hdr,
697 struct acpi_dmar_hardware_unit,
698 header);
699
700 for (scope = (void *)(drhd + 1);
701 (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
702 scope = ((void *)scope) + scope->length) {
703 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
704 continue;
705 if (scope->enumeration_id != device_number)
706 continue;
707
708 path = (void *)(scope + 1);
709 pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
710 dev_name(&adev->dev), dmaru->reg_base_addr,
711 scope->bus, path->device, path->function);
712 for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
713 if (tmp == NULL) {
714 dmaru->devices[i].bus = scope->bus;
715 dmaru->devices[i].devfn = PCI_DEVFN(path->device,
716 path->function);
717 rcu_assign_pointer(dmaru->devices[i].dev,
718 get_device(&adev->dev));
719 return;
720 }
721 BUG_ON(i >= dmaru->devices_cnt);
722 }
723 }
724 pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
725 device_number, dev_name(&adev->dev));
726 }
727
728 static int __init dmar_acpi_dev_scope_init(void)
729 {
730 struct acpi_dmar_andd *andd;
731
732 if (dmar_tbl == NULL)
733 return -ENODEV;
734
735 for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
736 ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
737 andd = ((void *)andd) + andd->header.length) {
738 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
739 acpi_handle h;
740 struct acpi_device *adev;
741
742 if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
743 andd->device_name,
744 &h))) {
745 pr_err("Failed to find handle for ACPI object %s\n",
746 andd->device_name);
747 continue;
748 }
749 if (acpi_bus_get_device(h, &adev)) {
750 pr_err("Failed to get device for ACPI object %s\n",
751 andd->device_name);
752 continue;
753 }
754 dmar_acpi_insert_dev_scope(andd->device_number, adev);
755 }
756 }
757 return 0;
758 }
759
760 int __init dmar_dev_scope_init(void)
761 {
762 struct pci_dev *dev = NULL;
763 struct dmar_pci_notify_info *info;
764
765 if (dmar_dev_scope_status != 1)
766 return dmar_dev_scope_status;
767
768 if (list_empty(&dmar_drhd_units)) {
769 dmar_dev_scope_status = -ENODEV;
770 } else {
771 dmar_dev_scope_status = 0;
772
773 dmar_acpi_dev_scope_init();
774
775 for_each_pci_dev(dev) {
776 if (dev->is_virtfn)
777 continue;
778
779 info = dmar_alloc_pci_notify_info(dev,
780 BUS_NOTIFY_ADD_DEVICE);
781 if (!info) {
782 return dmar_dev_scope_status;
783 } else {
784 dmar_pci_bus_add_dev(info);
785 dmar_free_pci_notify_info(info);
786 }
787 }
788
789 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
790 }
791
792 return dmar_dev_scope_status;
793 }
794
795
796 int __init dmar_table_init(void)
797 {
798 static int dmar_table_initialized;
799 int ret;
800
801 if (dmar_table_initialized == 0) {
802 ret = parse_dmar_table();
803 if (ret < 0) {
804 if (ret != -ENODEV)
805 pr_info("Parse DMAR table failure.\n");
806 } else if (list_empty(&dmar_drhd_units)) {
807 pr_info("No DMAR devices found\n");
808 ret = -ENODEV;
809 }
810
811 if (ret < 0)
812 dmar_table_initialized = ret;
813 else
814 dmar_table_initialized = 1;
815 }
816
817 return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
818 }
819
820 static void warn_invalid_dmar(u64 addr, const char *message)
821 {
822 WARN_TAINT_ONCE(
823 1, TAINT_FIRMWARE_WORKAROUND,
824 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
825 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
826 addr, message,
827 dmi_get_system_info(DMI_BIOS_VENDOR),
828 dmi_get_system_info(DMI_BIOS_VERSION),
829 dmi_get_system_info(DMI_PRODUCT_VERSION));
830 }
831
832 static int __ref
833 dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
834 {
835 struct acpi_dmar_hardware_unit *drhd;
836 void __iomem *addr;
837 u64 cap, ecap;
838
839 drhd = (void *)entry;
840 if (!drhd->address) {
841 warn_invalid_dmar(0, "");
842 return -EINVAL;
843 }
844
845 if (arg)
846 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
847 else
848 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
849 if (!addr) {
850 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
851 return -EINVAL;
852 }
853
854 cap = dmar_readq(addr + DMAR_CAP_REG);
855 ecap = dmar_readq(addr + DMAR_ECAP_REG);
856
857 if (arg)
858 iounmap(addr);
859 else
860 early_iounmap(addr, VTD_PAGE_SIZE);
861
862 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
863 warn_invalid_dmar(drhd->address, " returns all ones");
864 return -EINVAL;
865 }
866
867 return 0;
868 }
869
870 int __init detect_intel_iommu(void)
871 {
872 int ret;
873 struct dmar_res_callback validate_drhd_cb = {
874 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
875 .ignore_unhandled = true,
876 };
877
878 down_write(&dmar_global_lock);
879 ret = dmar_table_detect();
880 if (ret)
881 ret = !dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
882 &validate_drhd_cb);
883 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
884 iommu_detected = 1;
885 /* Make sure ACS will be enabled */
886 pci_request_acs();
887 }
888
889 #ifdef CONFIG_X86
890 if (ret)
891 x86_init.iommu.iommu_init = intel_iommu_init;
892 #endif
893
894 early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
895 dmar_tbl = NULL;
896 up_write(&dmar_global_lock);
897
898 return ret ? 1 : -ENODEV;
899 }
900
901
902 static void unmap_iommu(struct intel_iommu *iommu)
903 {
904 iounmap(iommu->reg);
905 release_mem_region(iommu->reg_phys, iommu->reg_size);
906 }
907
908 /**
909 * map_iommu: map the iommu's registers
910 * @iommu: the iommu to map
911 * @phys_addr: the physical address of the base resgister
912 *
913 * Memory map the iommu's registers. Start w/ a single page, and
914 * possibly expand if that turns out to be insufficent.
915 */
916 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
917 {
918 int map_size, err=0;
919
920 iommu->reg_phys = phys_addr;
921 iommu->reg_size = VTD_PAGE_SIZE;
922
923 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
924 pr_err("Can't reserve memory\n");
925 err = -EBUSY;
926 goto out;
927 }
928
929 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
930 if (!iommu->reg) {
931 pr_err("Can't map the region\n");
932 err = -ENOMEM;
933 goto release;
934 }
935
936 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
937 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
938
939 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
940 err = -EINVAL;
941 warn_invalid_dmar(phys_addr, " returns all ones");
942 goto unmap;
943 }
944
945 /* the registers might be more than one page */
946 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
947 cap_max_fault_reg_offset(iommu->cap));
948 map_size = VTD_PAGE_ALIGN(map_size);
949 if (map_size > iommu->reg_size) {
950 iounmap(iommu->reg);
951 release_mem_region(iommu->reg_phys, iommu->reg_size);
952 iommu->reg_size = map_size;
953 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
954 iommu->name)) {
955 pr_err("Can't reserve memory\n");
956 err = -EBUSY;
957 goto out;
958 }
959 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
960 if (!iommu->reg) {
961 pr_err("Can't map the region\n");
962 err = -ENOMEM;
963 goto release;
964 }
965 }
966 err = 0;
967 goto out;
968
969 unmap:
970 iounmap(iommu->reg);
971 release:
972 release_mem_region(iommu->reg_phys, iommu->reg_size);
973 out:
974 return err;
975 }
976
977 static int dmar_alloc_seq_id(struct intel_iommu *iommu)
978 {
979 iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
980 DMAR_UNITS_SUPPORTED);
981 if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
982 iommu->seq_id = -1;
983 } else {
984 set_bit(iommu->seq_id, dmar_seq_ids);
985 sprintf(iommu->name, "dmar%d", iommu->seq_id);
986 }
987
988 return iommu->seq_id;
989 }
990
991 static void dmar_free_seq_id(struct intel_iommu *iommu)
992 {
993 if (iommu->seq_id >= 0) {
994 clear_bit(iommu->seq_id, dmar_seq_ids);
995 iommu->seq_id = -1;
996 }
997 }
998
999 static int alloc_iommu(struct dmar_drhd_unit *drhd)
1000 {
1001 struct intel_iommu *iommu;
1002 u32 ver, sts;
1003 int agaw = 0;
1004 int msagaw = 0;
1005 int err;
1006
1007 if (!drhd->reg_base_addr) {
1008 warn_invalid_dmar(0, "");
1009 return -EINVAL;
1010 }
1011
1012 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1013 if (!iommu)
1014 return -ENOMEM;
1015
1016 if (dmar_alloc_seq_id(iommu) < 0) {
1017 pr_err("Failed to allocate seq_id\n");
1018 err = -ENOSPC;
1019 goto error;
1020 }
1021
1022 err = map_iommu(iommu, drhd->reg_base_addr);
1023 if (err) {
1024 pr_err("Failed to map %s\n", iommu->name);
1025 goto error_free_seq_id;
1026 }
1027
1028 err = -EINVAL;
1029 agaw = iommu_calculate_agaw(iommu);
1030 if (agaw < 0) {
1031 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1032 iommu->seq_id);
1033 goto err_unmap;
1034 }
1035 msagaw = iommu_calculate_max_sagaw(iommu);
1036 if (msagaw < 0) {
1037 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1038 iommu->seq_id);
1039 goto err_unmap;
1040 }
1041 iommu->agaw = agaw;
1042 iommu->msagaw = msagaw;
1043 iommu->segment = drhd->segment;
1044
1045 iommu->node = -1;
1046
1047 ver = readl(iommu->reg + DMAR_VER_REG);
1048 pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1049 iommu->name,
1050 (unsigned long long)drhd->reg_base_addr,
1051 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1052 (unsigned long long)iommu->cap,
1053 (unsigned long long)iommu->ecap);
1054
1055 /* Reflect status in gcmd */
1056 sts = readl(iommu->reg + DMAR_GSTS_REG);
1057 if (sts & DMA_GSTS_IRES)
1058 iommu->gcmd |= DMA_GCMD_IRE;
1059 if (sts & DMA_GSTS_TES)
1060 iommu->gcmd |= DMA_GCMD_TE;
1061 if (sts & DMA_GSTS_QIES)
1062 iommu->gcmd |= DMA_GCMD_QIE;
1063
1064 raw_spin_lock_init(&iommu->register_lock);
1065
1066 if (intel_iommu_enabled) {
1067 iommu->iommu_dev = iommu_device_create(NULL, iommu,
1068 intel_iommu_groups,
1069 "%s", iommu->name);
1070
1071 if (IS_ERR(iommu->iommu_dev)) {
1072 err = PTR_ERR(iommu->iommu_dev);
1073 goto err_unmap;
1074 }
1075 }
1076
1077 drhd->iommu = iommu;
1078
1079 return 0;
1080
1081 err_unmap:
1082 unmap_iommu(iommu);
1083 error_free_seq_id:
1084 dmar_free_seq_id(iommu);
1085 error:
1086 kfree(iommu);
1087 return err;
1088 }
1089
1090 static void free_iommu(struct intel_iommu *iommu)
1091 {
1092 iommu_device_destroy(iommu->iommu_dev);
1093
1094 if (iommu->irq) {
1095 if (iommu->pr_irq) {
1096 free_irq(iommu->pr_irq, iommu);
1097 dmar_free_hwirq(iommu->pr_irq);
1098 iommu->pr_irq = 0;
1099 }
1100 free_irq(iommu->irq, iommu);
1101 dmar_free_hwirq(iommu->irq);
1102 iommu->irq = 0;
1103 }
1104
1105 if (iommu->qi) {
1106 free_page((unsigned long)iommu->qi->desc);
1107 kfree(iommu->qi->desc_status);
1108 kfree(iommu->qi);
1109 }
1110
1111 if (iommu->reg)
1112 unmap_iommu(iommu);
1113
1114 dmar_free_seq_id(iommu);
1115 kfree(iommu);
1116 }
1117
1118 /*
1119 * Reclaim all the submitted descriptors which have completed its work.
1120 */
1121 static inline void reclaim_free_desc(struct q_inval *qi)
1122 {
1123 while (qi->desc_status[qi->free_tail] == QI_DONE ||
1124 qi->desc_status[qi->free_tail] == QI_ABORT) {
1125 qi->desc_status[qi->free_tail] = QI_FREE;
1126 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1127 qi->free_cnt++;
1128 }
1129 }
1130
1131 static int qi_check_fault(struct intel_iommu *iommu, int index)
1132 {
1133 u32 fault;
1134 int head, tail;
1135 struct q_inval *qi = iommu->qi;
1136 int wait_index = (index + 1) % QI_LENGTH;
1137
1138 if (qi->desc_status[wait_index] == QI_ABORT)
1139 return -EAGAIN;
1140
1141 fault = readl(iommu->reg + DMAR_FSTS_REG);
1142
1143 /*
1144 * If IQE happens, the head points to the descriptor associated
1145 * with the error. No new descriptors are fetched until the IQE
1146 * is cleared.
1147 */
1148 if (fault & DMA_FSTS_IQE) {
1149 head = readl(iommu->reg + DMAR_IQH_REG);
1150 if ((head >> DMAR_IQ_SHIFT) == index) {
1151 pr_err("VT-d detected invalid descriptor: "
1152 "low=%llx, high=%llx\n",
1153 (unsigned long long)qi->desc[index].low,
1154 (unsigned long long)qi->desc[index].high);
1155 memcpy(&qi->desc[index], &qi->desc[wait_index],
1156 sizeof(struct qi_desc));
1157 __iommu_flush_cache(iommu, &qi->desc[index],
1158 sizeof(struct qi_desc));
1159 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1160 return -EINVAL;
1161 }
1162 }
1163
1164 /*
1165 * If ITE happens, all pending wait_desc commands are aborted.
1166 * No new descriptors are fetched until the ITE is cleared.
1167 */
1168 if (fault & DMA_FSTS_ITE) {
1169 head = readl(iommu->reg + DMAR_IQH_REG);
1170 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1171 head |= 1;
1172 tail = readl(iommu->reg + DMAR_IQT_REG);
1173 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1174
1175 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1176
1177 do {
1178 if (qi->desc_status[head] == QI_IN_USE)
1179 qi->desc_status[head] = QI_ABORT;
1180 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1181 } while (head != tail);
1182
1183 if (qi->desc_status[wait_index] == QI_ABORT)
1184 return -EAGAIN;
1185 }
1186
1187 if (fault & DMA_FSTS_ICE)
1188 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1189
1190 return 0;
1191 }
1192
1193 /*
1194 * Submit the queued invalidation descriptor to the remapping
1195 * hardware unit and wait for its completion.
1196 */
1197 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1198 {
1199 int rc;
1200 struct q_inval *qi = iommu->qi;
1201 struct qi_desc *hw, wait_desc;
1202 int wait_index, index;
1203 unsigned long flags;
1204
1205 if (!qi)
1206 return 0;
1207
1208 hw = qi->desc;
1209
1210 restart:
1211 rc = 0;
1212
1213 raw_spin_lock_irqsave(&qi->q_lock, flags);
1214 while (qi->free_cnt < 3) {
1215 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1216 cpu_relax();
1217 raw_spin_lock_irqsave(&qi->q_lock, flags);
1218 }
1219
1220 index = qi->free_head;
1221 wait_index = (index + 1) % QI_LENGTH;
1222
1223 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1224
1225 hw[index] = *desc;
1226
1227 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1228 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1229 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1230
1231 hw[wait_index] = wait_desc;
1232
1233 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
1234 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
1235
1236 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1237 qi->free_cnt -= 2;
1238
1239 /*
1240 * update the HW tail register indicating the presence of
1241 * new descriptors.
1242 */
1243 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
1244
1245 while (qi->desc_status[wait_index] != QI_DONE) {
1246 /*
1247 * We will leave the interrupts disabled, to prevent interrupt
1248 * context to queue another cmd while a cmd is already submitted
1249 * and waiting for completion on this cpu. This is to avoid
1250 * a deadlock where the interrupt context can wait indefinitely
1251 * for free slots in the queue.
1252 */
1253 rc = qi_check_fault(iommu, index);
1254 if (rc)
1255 break;
1256
1257 raw_spin_unlock(&qi->q_lock);
1258 cpu_relax();
1259 raw_spin_lock(&qi->q_lock);
1260 }
1261
1262 qi->desc_status[index] = QI_DONE;
1263
1264 reclaim_free_desc(qi);
1265 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1266
1267 if (rc == -EAGAIN)
1268 goto restart;
1269
1270 return rc;
1271 }
1272
1273 /*
1274 * Flush the global interrupt entry cache.
1275 */
1276 void qi_global_iec(struct intel_iommu *iommu)
1277 {
1278 struct qi_desc desc;
1279
1280 desc.low = QI_IEC_TYPE;
1281 desc.high = 0;
1282
1283 /* should never fail */
1284 qi_submit_sync(&desc, iommu);
1285 }
1286
1287 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1288 u64 type)
1289 {
1290 struct qi_desc desc;
1291
1292 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1293 | QI_CC_GRAN(type) | QI_CC_TYPE;
1294 desc.high = 0;
1295
1296 qi_submit_sync(&desc, iommu);
1297 }
1298
1299 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1300 unsigned int size_order, u64 type)
1301 {
1302 u8 dw = 0, dr = 0;
1303
1304 struct qi_desc desc;
1305 int ih = 0;
1306
1307 if (cap_write_drain(iommu->cap))
1308 dw = 1;
1309
1310 if (cap_read_drain(iommu->cap))
1311 dr = 1;
1312
1313 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1314 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1315 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1316 | QI_IOTLB_AM(size_order);
1317
1318 qi_submit_sync(&desc, iommu);
1319 }
1320
1321 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1322 u64 addr, unsigned mask)
1323 {
1324 struct qi_desc desc;
1325
1326 if (mask) {
1327 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1328 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1329 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1330 } else
1331 desc.high = QI_DEV_IOTLB_ADDR(addr);
1332
1333 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1334 qdep = 0;
1335
1336 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1337 QI_DIOTLB_TYPE;
1338
1339 qi_submit_sync(&desc, iommu);
1340 }
1341
1342 /*
1343 * Disable Queued Invalidation interface.
1344 */
1345 void dmar_disable_qi(struct intel_iommu *iommu)
1346 {
1347 unsigned long flags;
1348 u32 sts;
1349 cycles_t start_time = get_cycles();
1350
1351 if (!ecap_qis(iommu->ecap))
1352 return;
1353
1354 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1355
1356 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
1357 if (!(sts & DMA_GSTS_QIES))
1358 goto end;
1359
1360 /*
1361 * Give a chance to HW to complete the pending invalidation requests.
1362 */
1363 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1364 readl(iommu->reg + DMAR_IQH_REG)) &&
1365 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1366 cpu_relax();
1367
1368 iommu->gcmd &= ~DMA_GCMD_QIE;
1369 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1370
1371 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1372 !(sts & DMA_GSTS_QIES), sts);
1373 end:
1374 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1375 }
1376
1377 /*
1378 * Enable queued invalidation.
1379 */
1380 static void __dmar_enable_qi(struct intel_iommu *iommu)
1381 {
1382 u32 sts;
1383 unsigned long flags;
1384 struct q_inval *qi = iommu->qi;
1385
1386 qi->free_head = qi->free_tail = 0;
1387 qi->free_cnt = QI_LENGTH;
1388
1389 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1390
1391 /* write zero to the tail reg */
1392 writel(0, iommu->reg + DMAR_IQT_REG);
1393
1394 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1395
1396 iommu->gcmd |= DMA_GCMD_QIE;
1397 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1398
1399 /* Make sure hardware complete it */
1400 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1401
1402 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1403 }
1404
1405 /*
1406 * Enable Queued Invalidation interface. This is a must to support
1407 * interrupt-remapping. Also used by DMA-remapping, which replaces
1408 * register based IOTLB invalidation.
1409 */
1410 int dmar_enable_qi(struct intel_iommu *iommu)
1411 {
1412 struct q_inval *qi;
1413 struct page *desc_page;
1414
1415 if (!ecap_qis(iommu->ecap))
1416 return -ENOENT;
1417
1418 /*
1419 * queued invalidation is already setup and enabled.
1420 */
1421 if (iommu->qi)
1422 return 0;
1423
1424 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1425 if (!iommu->qi)
1426 return -ENOMEM;
1427
1428 qi = iommu->qi;
1429
1430
1431 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1432 if (!desc_page) {
1433 kfree(qi);
1434 iommu->qi = NULL;
1435 return -ENOMEM;
1436 }
1437
1438 qi->desc = page_address(desc_page);
1439
1440 qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1441 if (!qi->desc_status) {
1442 free_page((unsigned long) qi->desc);
1443 kfree(qi);
1444 iommu->qi = NULL;
1445 return -ENOMEM;
1446 }
1447
1448 raw_spin_lock_init(&qi->q_lock);
1449
1450 __dmar_enable_qi(iommu);
1451
1452 return 0;
1453 }
1454
1455 /* iommu interrupt handling. Most stuff are MSI-like. */
1456
1457 enum faulttype {
1458 DMA_REMAP,
1459 INTR_REMAP,
1460 UNKNOWN,
1461 };
1462
1463 static const char *dma_remap_fault_reasons[] =
1464 {
1465 "Software",
1466 "Present bit in root entry is clear",
1467 "Present bit in context entry is clear",
1468 "Invalid context entry",
1469 "Access beyond MGAW",
1470 "PTE Write access is not set",
1471 "PTE Read access is not set",
1472 "Next page table ptr is invalid",
1473 "Root table address invalid",
1474 "Context table ptr is invalid",
1475 "non-zero reserved fields in RTP",
1476 "non-zero reserved fields in CTP",
1477 "non-zero reserved fields in PTE",
1478 "PCE for translation request specifies blocking",
1479 };
1480
1481 static const char *irq_remap_fault_reasons[] =
1482 {
1483 "Detected reserved fields in the decoded interrupt-remapped request",
1484 "Interrupt index exceeded the interrupt-remapping table size",
1485 "Present field in the IRTE entry is clear",
1486 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1487 "Detected reserved fields in the IRTE entry",
1488 "Blocked a compatibility format interrupt request",
1489 "Blocked an interrupt request due to source-id verification failure",
1490 };
1491
1492 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1493 {
1494 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1495 ARRAY_SIZE(irq_remap_fault_reasons))) {
1496 *fault_type = INTR_REMAP;
1497 return irq_remap_fault_reasons[fault_reason - 0x20];
1498 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1499 *fault_type = DMA_REMAP;
1500 return dma_remap_fault_reasons[fault_reason];
1501 } else {
1502 *fault_type = UNKNOWN;
1503 return "Unknown";
1504 }
1505 }
1506
1507
1508 static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1509 {
1510 if (iommu->irq == irq)
1511 return DMAR_FECTL_REG;
1512 else if (iommu->pr_irq == irq)
1513 return DMAR_PECTL_REG;
1514 else
1515 BUG();
1516 }
1517
1518 void dmar_msi_unmask(struct irq_data *data)
1519 {
1520 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1521 int reg = dmar_msi_reg(iommu, data->irq);
1522 unsigned long flag;
1523
1524 /* unmask it */
1525 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1526 writel(0, iommu->reg + reg);
1527 /* Read a reg to force flush the post write */
1528 readl(iommu->reg + reg);
1529 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1530 }
1531
1532 void dmar_msi_mask(struct irq_data *data)
1533 {
1534 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1535 int reg = dmar_msi_reg(iommu, data->irq);
1536 unsigned long flag;
1537
1538 /* mask it */
1539 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1540 writel(DMA_FECTL_IM, iommu->reg + reg);
1541 /* Read a reg to force flush the post write */
1542 readl(iommu->reg + reg);
1543 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1544 }
1545
1546 void dmar_msi_write(int irq, struct msi_msg *msg)
1547 {
1548 struct intel_iommu *iommu = irq_get_handler_data(irq);
1549 int reg = dmar_msi_reg(iommu, irq);
1550 unsigned long flag;
1551
1552 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1553 writel(msg->data, iommu->reg + reg + 4);
1554 writel(msg->address_lo, iommu->reg + reg + 8);
1555 writel(msg->address_hi, iommu->reg + reg + 12);
1556 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1557 }
1558
1559 void dmar_msi_read(int irq, struct msi_msg *msg)
1560 {
1561 struct intel_iommu *iommu = irq_get_handler_data(irq);
1562 int reg = dmar_msi_reg(iommu, irq);
1563 unsigned long flag;
1564
1565 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1566 msg->data = readl(iommu->reg + reg + 4);
1567 msg->address_lo = readl(iommu->reg + reg + 8);
1568 msg->address_hi = readl(iommu->reg + reg + 12);
1569 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1570 }
1571
1572 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1573 u8 fault_reason, u16 source_id, unsigned long long addr)
1574 {
1575 const char *reason;
1576 int fault_type;
1577
1578 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1579
1580 if (fault_type == INTR_REMAP)
1581 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1582 "fault index %llx\n"
1583 "INTR-REMAP:[fault reason %02d] %s\n",
1584 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1585 PCI_FUNC(source_id & 0xFF), addr >> 48,
1586 fault_reason, reason);
1587 else
1588 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1589 "fault addr %llx \n"
1590 "DMAR:[fault reason %02d] %s\n",
1591 (type ? "DMA Read" : "DMA Write"),
1592 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1593 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1594 return 0;
1595 }
1596
1597 #define PRIMARY_FAULT_REG_LEN (16)
1598 irqreturn_t dmar_fault(int irq, void *dev_id)
1599 {
1600 struct intel_iommu *iommu = dev_id;
1601 int reg, fault_index;
1602 u32 fault_status;
1603 unsigned long flag;
1604
1605 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1606 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1607 if (fault_status)
1608 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1609
1610 /* TBD: ignore advanced fault log currently */
1611 if (!(fault_status & DMA_FSTS_PPF))
1612 goto unlock_exit;
1613
1614 fault_index = dma_fsts_fault_record_index(fault_status);
1615 reg = cap_fault_reg_offset(iommu->cap);
1616 while (1) {
1617 u8 fault_reason;
1618 u16 source_id;
1619 u64 guest_addr;
1620 int type;
1621 u32 data;
1622
1623 /* highest 32 bits */
1624 data = readl(iommu->reg + reg +
1625 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1626 if (!(data & DMA_FRCD_F))
1627 break;
1628
1629 fault_reason = dma_frcd_fault_reason(data);
1630 type = dma_frcd_type(data);
1631
1632 data = readl(iommu->reg + reg +
1633 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1634 source_id = dma_frcd_source_id(data);
1635
1636 guest_addr = dmar_readq(iommu->reg + reg +
1637 fault_index * PRIMARY_FAULT_REG_LEN);
1638 guest_addr = dma_frcd_page_addr(guest_addr);
1639 /* clear the fault */
1640 writel(DMA_FRCD_F, iommu->reg + reg +
1641 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1642
1643 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1644
1645 dmar_fault_do_one(iommu, type, fault_reason,
1646 source_id, guest_addr);
1647
1648 fault_index++;
1649 if (fault_index >= cap_num_fault_regs(iommu->cap))
1650 fault_index = 0;
1651 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1652 }
1653
1654 writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1655
1656 unlock_exit:
1657 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1658 return IRQ_HANDLED;
1659 }
1660
1661 int dmar_set_interrupt(struct intel_iommu *iommu)
1662 {
1663 int irq, ret;
1664
1665 /*
1666 * Check if the fault interrupt is already initialized.
1667 */
1668 if (iommu->irq)
1669 return 0;
1670
1671 irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
1672 if (irq > 0) {
1673 iommu->irq = irq;
1674 } else {
1675 pr_err("No free IRQ vectors\n");
1676 return -EINVAL;
1677 }
1678
1679 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1680 if (ret)
1681 pr_err("Can't request irq\n");
1682 return ret;
1683 }
1684
1685 int __init enable_drhd_fault_handling(void)
1686 {
1687 struct dmar_drhd_unit *drhd;
1688 struct intel_iommu *iommu;
1689
1690 /*
1691 * Enable fault control interrupt.
1692 */
1693 for_each_iommu(iommu, drhd) {
1694 u32 fault_status;
1695 int ret = dmar_set_interrupt(iommu);
1696
1697 if (ret) {
1698 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1699 (unsigned long long)drhd->reg_base_addr, ret);
1700 return -1;
1701 }
1702
1703 /*
1704 * Clear any previous faults.
1705 */
1706 dmar_fault(iommu->irq, iommu);
1707 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1708 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1709 }
1710
1711 return 0;
1712 }
1713
1714 /*
1715 * Re-enable Queued Invalidation interface.
1716 */
1717 int dmar_reenable_qi(struct intel_iommu *iommu)
1718 {
1719 if (!ecap_qis(iommu->ecap))
1720 return -ENOENT;
1721
1722 if (!iommu->qi)
1723 return -ENOENT;
1724
1725 /*
1726 * First disable queued invalidation.
1727 */
1728 dmar_disable_qi(iommu);
1729 /*
1730 * Then enable queued invalidation again. Since there is no pending
1731 * invalidation requests now, it's safe to re-enable queued
1732 * invalidation.
1733 */
1734 __dmar_enable_qi(iommu);
1735
1736 return 0;
1737 }
1738
1739 /*
1740 * Check interrupt remapping support in DMAR table description.
1741 */
1742 int __init dmar_ir_support(void)
1743 {
1744 struct acpi_table_dmar *dmar;
1745 dmar = (struct acpi_table_dmar *)dmar_tbl;
1746 if (!dmar)
1747 return 0;
1748 return dmar->flags & 0x1;
1749 }
1750
1751 /* Check whether DMAR units are in use */
1752 static inline bool dmar_in_use(void)
1753 {
1754 return irq_remapping_enabled || intel_iommu_enabled;
1755 }
1756
1757 static int __init dmar_free_unused_resources(void)
1758 {
1759 struct dmar_drhd_unit *dmaru, *dmaru_n;
1760
1761 if (dmar_in_use())
1762 return 0;
1763
1764 if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1765 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1766
1767 down_write(&dmar_global_lock);
1768 list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1769 list_del(&dmaru->list);
1770 dmar_free_drhd(dmaru);
1771 }
1772 up_write(&dmar_global_lock);
1773
1774 return 0;
1775 }
1776
1777 late_initcall(dmar_free_unused_resources);
1778 IOMMU_INIT_POST(detect_intel_iommu);
1779
1780 /*
1781 * DMAR Hotplug Support
1782 * For more details, please refer to Intel(R) Virtualization Technology
1783 * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
1784 * "Remapping Hardware Unit Hot Plug".
1785 */
1786 static u8 dmar_hp_uuid[] = {
1787 /* 0000 */ 0xA6, 0xA3, 0xC1, 0xD8, 0x9B, 0xBE, 0x9B, 0x4C,
1788 /* 0008 */ 0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF
1789 };
1790
1791 /*
1792 * Currently there's only one revision and BIOS will not check the revision id,
1793 * so use 0 for safety.
1794 */
1795 #define DMAR_DSM_REV_ID 0
1796 #define DMAR_DSM_FUNC_DRHD 1
1797 #define DMAR_DSM_FUNC_ATSR 2
1798 #define DMAR_DSM_FUNC_RHSA 3
1799
1800 static inline bool dmar_detect_dsm(acpi_handle handle, int func)
1801 {
1802 return acpi_check_dsm(handle, dmar_hp_uuid, DMAR_DSM_REV_ID, 1 << func);
1803 }
1804
1805 static int dmar_walk_dsm_resource(acpi_handle handle, int func,
1806 dmar_res_handler_t handler, void *arg)
1807 {
1808 int ret = -ENODEV;
1809 union acpi_object *obj;
1810 struct acpi_dmar_header *start;
1811 struct dmar_res_callback callback;
1812 static int res_type[] = {
1813 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
1814 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
1815 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
1816 };
1817
1818 if (!dmar_detect_dsm(handle, func))
1819 return 0;
1820
1821 obj = acpi_evaluate_dsm_typed(handle, dmar_hp_uuid, DMAR_DSM_REV_ID,
1822 func, NULL, ACPI_TYPE_BUFFER);
1823 if (!obj)
1824 return -ENODEV;
1825
1826 memset(&callback, 0, sizeof(callback));
1827 callback.cb[res_type[func]] = handler;
1828 callback.arg[res_type[func]] = arg;
1829 start = (struct acpi_dmar_header *)obj->buffer.pointer;
1830 ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
1831
1832 ACPI_FREE(obj);
1833
1834 return ret;
1835 }
1836
1837 static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
1838 {
1839 int ret;
1840 struct dmar_drhd_unit *dmaru;
1841
1842 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1843 if (!dmaru)
1844 return -ENODEV;
1845
1846 ret = dmar_ir_hotplug(dmaru, true);
1847 if (ret == 0)
1848 ret = dmar_iommu_hotplug(dmaru, true);
1849
1850 return ret;
1851 }
1852
1853 static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
1854 {
1855 int i, ret;
1856 struct device *dev;
1857 struct dmar_drhd_unit *dmaru;
1858
1859 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1860 if (!dmaru)
1861 return 0;
1862
1863 /*
1864 * All PCI devices managed by this unit should have been destroyed.
1865 */
1866 if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt)
1867 for_each_active_dev_scope(dmaru->devices,
1868 dmaru->devices_cnt, i, dev)
1869 return -EBUSY;
1870
1871 ret = dmar_ir_hotplug(dmaru, false);
1872 if (ret == 0)
1873 ret = dmar_iommu_hotplug(dmaru, false);
1874
1875 return ret;
1876 }
1877
1878 static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
1879 {
1880 struct dmar_drhd_unit *dmaru;
1881
1882 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1883 if (dmaru) {
1884 list_del_rcu(&dmaru->list);
1885 synchronize_rcu();
1886 dmar_free_drhd(dmaru);
1887 }
1888
1889 return 0;
1890 }
1891
1892 static int dmar_hotplug_insert(acpi_handle handle)
1893 {
1894 int ret;
1895 int drhd_count = 0;
1896
1897 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1898 &dmar_validate_one_drhd, (void *)1);
1899 if (ret)
1900 goto out;
1901
1902 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1903 &dmar_parse_one_drhd, (void *)&drhd_count);
1904 if (ret == 0 && drhd_count == 0) {
1905 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
1906 goto out;
1907 } else if (ret) {
1908 goto release_drhd;
1909 }
1910
1911 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
1912 &dmar_parse_one_rhsa, NULL);
1913 if (ret)
1914 goto release_drhd;
1915
1916 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1917 &dmar_parse_one_atsr, NULL);
1918 if (ret)
1919 goto release_atsr;
1920
1921 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1922 &dmar_hp_add_drhd, NULL);
1923 if (!ret)
1924 return 0;
1925
1926 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1927 &dmar_hp_remove_drhd, NULL);
1928 release_atsr:
1929 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1930 &dmar_release_one_atsr, NULL);
1931 release_drhd:
1932 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1933 &dmar_hp_release_drhd, NULL);
1934 out:
1935 return ret;
1936 }
1937
1938 static int dmar_hotplug_remove(acpi_handle handle)
1939 {
1940 int ret;
1941
1942 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1943 &dmar_check_one_atsr, NULL);
1944 if (ret)
1945 return ret;
1946
1947 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1948 &dmar_hp_remove_drhd, NULL);
1949 if (ret == 0) {
1950 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1951 &dmar_release_one_atsr, NULL));
1952 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1953 &dmar_hp_release_drhd, NULL));
1954 } else {
1955 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1956 &dmar_hp_add_drhd, NULL);
1957 }
1958
1959 return ret;
1960 }
1961
1962 static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
1963 void *context, void **retval)
1964 {
1965 acpi_handle *phdl = retval;
1966
1967 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1968 *phdl = handle;
1969 return AE_CTRL_TERMINATE;
1970 }
1971
1972 return AE_OK;
1973 }
1974
1975 static int dmar_device_hotplug(acpi_handle handle, bool insert)
1976 {
1977 int ret;
1978 acpi_handle tmp = NULL;
1979 acpi_status status;
1980
1981 if (!dmar_in_use())
1982 return 0;
1983
1984 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1985 tmp = handle;
1986 } else {
1987 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
1988 ACPI_UINT32_MAX,
1989 dmar_get_dsm_handle,
1990 NULL, NULL, &tmp);
1991 if (ACPI_FAILURE(status)) {
1992 pr_warn("Failed to locate _DSM method.\n");
1993 return -ENXIO;
1994 }
1995 }
1996 if (tmp == NULL)
1997 return 0;
1998
1999 down_write(&dmar_global_lock);
2000 if (insert)
2001 ret = dmar_hotplug_insert(tmp);
2002 else
2003 ret = dmar_hotplug_remove(tmp);
2004 up_write(&dmar_global_lock);
2005
2006 return ret;
2007 }
2008
2009 int dmar_device_add(acpi_handle handle)
2010 {
2011 return dmar_device_hotplug(handle, true);
2012 }
2013
2014 int dmar_device_remove(acpi_handle handle)
2015 {
2016 return dmar_device_hotplug(handle, false);
2017 }
Linux with added FPC-III support