?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
mm, debug: replace dump_flags() with the new printk formats
[linux/fpc-iii.git] / drivers / iommu / dmar.c
blob8ffd7568fc919e32b97354b2777db21be263d37f
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 *
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */
28
29 #define pr_fmt(fmt) "DMAR: " fmt
30
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/iova.h>
34 #include <linux/intel-iommu.h>
35 #include <linux/timer.h>
36 #include <linux/irq.h>
37 #include <linux/interrupt.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/slab.h>
41 #include <linux/iommu.h>
42 #include <asm/irq_remapping.h>
43 #include <asm/iommu_table.h>
44
45 #include "irq_remapping.h"
46
47 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
48 struct dmar_res_callback {
49 dmar_res_handler_t cb[ACPI_DMAR_TYPE_RESERVED];
50 void *arg[ACPI_DMAR_TYPE_RESERVED];
51 bool ignore_unhandled;
52 bool print_entry;
53 };
54
55 /*
56 * Assumptions:
57 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
58 * before IO devices managed by that unit.
59 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
60 * after IO devices managed by that unit.
61 * 3) Hotplug events are rare.
62 *
63 * Locking rules for DMA and interrupt remapping related global data structures:
64 * 1) Use dmar_global_lock in process context
65 * 2) Use RCU in interrupt context
66 */
67 DECLARE_RWSEM(dmar_global_lock);
68 LIST_HEAD(dmar_drhd_units);
69
70 struct acpi_table_header * __initdata dmar_tbl;
71 static acpi_size dmar_tbl_size;
72 static int dmar_dev_scope_status = 1;
73 static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
74
75 static int alloc_iommu(struct dmar_drhd_unit *drhd);
76 static void free_iommu(struct intel_iommu *iommu);
77
78 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
79 {
80 /*
81 * add INCLUDE_ALL at the tail, so scan the list will find it at
82 * the very end.
83 */
84 if (drhd->include_all)
85 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
86 else
87 list_add_rcu(&drhd->list, &dmar_drhd_units);
88 }
89
90 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
91 {
92 struct acpi_dmar_device_scope *scope;
93
94 *cnt = 0;
95 while (start < end) {
96 scope = start;
97 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
98 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
99 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
100 (*cnt)++;
101 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
102 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
103 pr_warn("Unsupported device scope\n");
104 }
105 start += scope->length;
106 }
107 if (*cnt == 0)
108 return NULL;
109
110 return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
111 }
112
113 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
114 {
115 int i;
116 struct device *tmp_dev;
117
118 if (*devices && *cnt) {
119 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
120 put_device(tmp_dev);
121 kfree(*devices);
122 }
123
124 *devices = NULL;
125 *cnt = 0;
126 }
127
128 /* Optimize out kzalloc()/kfree() for normal cases */
129 static char dmar_pci_notify_info_buf[64];
130
131 static struct dmar_pci_notify_info *
132 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
133 {
134 int level = 0;
135 size_t size;
136 struct pci_dev *tmp;
137 struct dmar_pci_notify_info *info;
138
139 BUG_ON(dev->is_virtfn);
140
141 /* Only generate path[] for device addition event */
142 if (event == BUS_NOTIFY_ADD_DEVICE)
143 for (tmp = dev; tmp; tmp = tmp->bus->self)
144 level++;
145
146 size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
147 if (size <= sizeof(dmar_pci_notify_info_buf)) {
148 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
149 } else {
150 info = kzalloc(size, GFP_KERNEL);
151 if (!info) {
152 pr_warn("Out of memory when allocating notify_info "
153 "for %s.\n", pci_name(dev));
154 if (dmar_dev_scope_status == 0)
155 dmar_dev_scope_status = -ENOMEM;
156 return NULL;
157 }
158 }
159
160 info->event = event;
161 info->dev = dev;
162 info->seg = pci_domain_nr(dev->bus);
163 info->level = level;
164 if (event == BUS_NOTIFY_ADD_DEVICE) {
165 for (tmp = dev; tmp; tmp = tmp->bus->self) {
166 level--;
167 info->path[level].bus = tmp->bus->number;
168 info->path[level].device = PCI_SLOT(tmp->devfn);
169 info->path[level].function = PCI_FUNC(tmp->devfn);
170 if (pci_is_root_bus(tmp->bus))
171 info->bus = tmp->bus->number;
172 }
173 }
174
175 return info;
176 }
177
178 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
179 {
180 if ((void *)info != dmar_pci_notify_info_buf)
181 kfree(info);
182 }
183
184 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
185 struct acpi_dmar_pci_path *path, int count)
186 {
187 int i;
188
189 if (info->bus != bus)
190 goto fallback;
191 if (info->level != count)
192 goto fallback;
193
194 for (i = 0; i < count; i++) {
195 if (path[i].device != info->path[i].device ||
196 path[i].function != info->path[i].function)
197 goto fallback;
198 }
199
200 return true;
201
202 fallback:
203
204 if (count != 1)
205 return false;
206
207 i = info->level - 1;
208 if (bus == info->path[i].bus &&
209 path[0].device == info->path[i].device &&
210 path[0].function == info->path[i].function) {
211 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
212 bus, path[0].device, path[0].function);
213 return true;
214 }
215
216 return false;
217 }
218
219 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
220 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
221 void *start, void*end, u16 segment,
222 struct dmar_dev_scope *devices,
223 int devices_cnt)
224 {
225 int i, level;
226 struct device *tmp, *dev = &info->dev->dev;
227 struct acpi_dmar_device_scope *scope;
228 struct acpi_dmar_pci_path *path;
229
230 if (segment != info->seg)
231 return 0;
232
233 for (; start < end; start += scope->length) {
234 scope = start;
235 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
236 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
237 continue;
238
239 path = (struct acpi_dmar_pci_path *)(scope + 1);
240 level = (scope->length - sizeof(*scope)) / sizeof(*path);
241 if (!dmar_match_pci_path(info, scope->bus, path, level))
242 continue;
243
244 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^
245 (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) {
246 pr_warn("Device scope type does not match for %s\n",
247 pci_name(info->dev));
248 return -EINVAL;
249 }
250
251 for_each_dev_scope(devices, devices_cnt, i, tmp)
252 if (tmp == NULL) {
253 devices[i].bus = info->dev->bus->number;
254 devices[i].devfn = info->dev->devfn;
255 rcu_assign_pointer(devices[i].dev,
256 get_device(dev));
257 return 1;
258 }
259 BUG_ON(i >= devices_cnt);
260 }
261
262 return 0;
263 }
264
265 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
266 struct dmar_dev_scope *devices, int count)
267 {
268 int index;
269 struct device *tmp;
270
271 if (info->seg != segment)
272 return 0;
273
274 for_each_active_dev_scope(devices, count, index, tmp)
275 if (tmp == &info->dev->dev) {
276 RCU_INIT_POINTER(devices[index].dev, NULL);
277 synchronize_rcu();
278 put_device(tmp);
279 return 1;
280 }
281
282 return 0;
283 }
284
285 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
286 {
287 int ret = 0;
288 struct dmar_drhd_unit *dmaru;
289 struct acpi_dmar_hardware_unit *drhd;
290
291 for_each_drhd_unit(dmaru) {
292 if (dmaru->include_all)
293 continue;
294
295 drhd = container_of(dmaru->hdr,
296 struct acpi_dmar_hardware_unit, header);
297 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
298 ((void *)drhd) + drhd->header.length,
299 dmaru->segment,
300 dmaru->devices, dmaru->devices_cnt);
301 if (ret != 0)
302 break;
303 }
304 if (ret >= 0)
305 ret = dmar_iommu_notify_scope_dev(info);
306 if (ret < 0 && dmar_dev_scope_status == 0)
307 dmar_dev_scope_status = ret;
308
309 return ret;
310 }
311
312 static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
313 {
314 struct dmar_drhd_unit *dmaru;
315
316 for_each_drhd_unit(dmaru)
317 if (dmar_remove_dev_scope(info, dmaru->segment,
318 dmaru->devices, dmaru->devices_cnt))
319 break;
320 dmar_iommu_notify_scope_dev(info);
321 }
322
323 static int dmar_pci_bus_notifier(struct notifier_block *nb,
324 unsigned long action, void *data)
325 {
326 struct pci_dev *pdev = to_pci_dev(data);
327 struct dmar_pci_notify_info *info;
328
329 /* Only care about add/remove events for physical functions */
330 if (pdev->is_virtfn)
331 return NOTIFY_DONE;
332 if (action != BUS_NOTIFY_ADD_DEVICE &&
333 action != BUS_NOTIFY_REMOVED_DEVICE)
334 return NOTIFY_DONE;
335
336 info = dmar_alloc_pci_notify_info(pdev, action);
337 if (!info)
338 return NOTIFY_DONE;
339
340 down_write(&dmar_global_lock);
341 if (action == BUS_NOTIFY_ADD_DEVICE)
342 dmar_pci_bus_add_dev(info);
343 else if (action == BUS_NOTIFY_REMOVED_DEVICE)
344 dmar_pci_bus_del_dev(info);
345 up_write(&dmar_global_lock);
346
347 dmar_free_pci_notify_info(info);
348
349 return NOTIFY_OK;
350 }
351
352 static struct notifier_block dmar_pci_bus_nb = {
353 .notifier_call = dmar_pci_bus_notifier,
354 .priority = INT_MIN,
355 };
356
357 static struct dmar_drhd_unit *
358 dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
359 {
360 struct dmar_drhd_unit *dmaru;
361
362 list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list)
363 if (dmaru->segment == drhd->segment &&
364 dmaru->reg_base_addr == drhd->address)
365 return dmaru;
366
367 return NULL;
368 }
369
370 /**
371 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
372 * structure which uniquely represent one DMA remapping hardware unit
373 * present in the platform
374 */
375 static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
376 {
377 struct acpi_dmar_hardware_unit *drhd;
378 struct dmar_drhd_unit *dmaru;
379 int ret = 0;
380
381 drhd = (struct acpi_dmar_hardware_unit *)header;
382 dmaru = dmar_find_dmaru(drhd);
383 if (dmaru)
384 goto out;
385
386 dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
387 if (!dmaru)
388 return -ENOMEM;
389
390 /*
391 * If header is allocated from slab by ACPI _DSM method, we need to
392 * copy the content because the memory buffer will be freed on return.
393 */
394 dmaru->hdr = (void *)(dmaru + 1);
395 memcpy(dmaru->hdr, header, header->length);
396 dmaru->reg_base_addr = drhd->address;
397 dmaru->segment = drhd->segment;
398 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
399 dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
400 ((void *)drhd) + drhd->header.length,
401 &dmaru->devices_cnt);
402 if (dmaru->devices_cnt && dmaru->devices == NULL) {
403 kfree(dmaru);
404 return -ENOMEM;
405 }
406
407 ret = alloc_iommu(dmaru);
408 if (ret) {
409 dmar_free_dev_scope(&dmaru->devices,
410 &dmaru->devices_cnt);
411 kfree(dmaru);
412 return ret;
413 }
414 dmar_register_drhd_unit(dmaru);
415
416 out:
417 if (arg)
418 (*(int *)arg)++;
419
420 return 0;
421 }
422
423 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
424 {
425 if (dmaru->devices && dmaru->devices_cnt)
426 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
427 if (dmaru->iommu)
428 free_iommu(dmaru->iommu);
429 kfree(dmaru);
430 }
431
432 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
433 void *arg)
434 {
435 struct acpi_dmar_andd *andd = (void *)header;
436
437 /* Check for NUL termination within the designated length */
438 if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
439 WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
440 "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
441 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
442 dmi_get_system_info(DMI_BIOS_VENDOR),
443 dmi_get_system_info(DMI_BIOS_VERSION),
444 dmi_get_system_info(DMI_PRODUCT_VERSION));
445 return -EINVAL;
446 }
447 pr_info("ANDD device: %x name: %s\n", andd->device_number,
448 andd->device_name);
449
450 return 0;
451 }
452
453 #ifdef CONFIG_ACPI_NUMA
454 static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
455 {
456 struct acpi_dmar_rhsa *rhsa;
457 struct dmar_drhd_unit *drhd;
458
459 rhsa = (struct acpi_dmar_rhsa *)header;
460 for_each_drhd_unit(drhd) {
461 if (drhd->reg_base_addr == rhsa->base_address) {
462 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
463
464 if (!node_online(node))
465 node = -1;
466 drhd->iommu->node = node;
467 return 0;
468 }
469 }
470 WARN_TAINT(
471 1, TAINT_FIRMWARE_WORKAROUND,
472 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
473 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
474 drhd->reg_base_addr,
475 dmi_get_system_info(DMI_BIOS_VENDOR),
476 dmi_get_system_info(DMI_BIOS_VERSION),
477 dmi_get_system_info(DMI_PRODUCT_VERSION));
478
479 return 0;
480 }
481 #else
482 #define dmar_parse_one_rhsa dmar_res_noop
483 #endif
484
485 static void __init
486 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
487 {
488 struct acpi_dmar_hardware_unit *drhd;
489 struct acpi_dmar_reserved_memory *rmrr;
490 struct acpi_dmar_atsr *atsr;
491 struct acpi_dmar_rhsa *rhsa;
492
493 switch (header->type) {
494 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
495 drhd = container_of(header, struct acpi_dmar_hardware_unit,
496 header);
497 pr_info("DRHD base: %#016Lx flags: %#x\n",
498 (unsigned long long)drhd->address, drhd->flags);
499 break;
500 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
501 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
502 header);
503 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
504 (unsigned long long)rmrr->base_address,
505 (unsigned long long)rmrr->end_address);
506 break;
507 case ACPI_DMAR_TYPE_ROOT_ATS:
508 atsr = container_of(header, struct acpi_dmar_atsr, header);
509 pr_info("ATSR flags: %#x\n", atsr->flags);
510 break;
511 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
512 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
513 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
514 (unsigned long long)rhsa->base_address,
515 rhsa->proximity_domain);
516 break;
517 case ACPI_DMAR_TYPE_NAMESPACE:
518 /* We don't print this here because we need to sanity-check
519 it first. So print it in dmar_parse_one_andd() instead. */
520 break;
521 }
522 }
523
524 /**
525 * dmar_table_detect - checks to see if the platform supports DMAR devices
526 */
527 static int __init dmar_table_detect(void)
528 {
529 acpi_status status = AE_OK;
530
531 /* if we could find DMAR table, then there are DMAR devices */
532 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
533 (struct acpi_table_header **)&dmar_tbl,
534 &dmar_tbl_size);
535
536 if (ACPI_SUCCESS(status) && !dmar_tbl) {
537 pr_warn("Unable to map DMAR\n");
538 status = AE_NOT_FOUND;
539 }
540
541 return (ACPI_SUCCESS(status) ? 1 : 0);
542 }
543
544 static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
545 size_t len, struct dmar_res_callback *cb)
546 {
547 int ret = 0;
548 struct acpi_dmar_header *iter, *next;
549 struct acpi_dmar_header *end = ((void *)start) + len;
550
551 for (iter = start; iter < end && ret == 0; iter = next) {
552 next = (void *)iter + iter->length;
553 if (iter->length == 0) {
554 /* Avoid looping forever on bad ACPI tables */
555 pr_debug(FW_BUG "Invalid 0-length structure\n");
556 break;
557 } else if (next > end) {
558 /* Avoid passing table end */
559 pr_warn(FW_BUG "Record passes table end\n");
560 ret = -EINVAL;
561 break;
562 }
563
564 if (cb->print_entry)
565 dmar_table_print_dmar_entry(iter);
566
567 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
568 /* continue for forward compatibility */
569 pr_debug("Unknown DMAR structure type %d\n",
570 iter->type);
571 } else if (cb->cb[iter->type]) {
572 ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
573 } else if (!cb->ignore_unhandled) {
574 pr_warn("No handler for DMAR structure type %d\n",
575 iter->type);
576 ret = -EINVAL;
577 }
578 }
579
580 return ret;
581 }
582
583 static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
584 struct dmar_res_callback *cb)
585 {
586 return dmar_walk_remapping_entries((void *)(dmar + 1),
587 dmar->header.length - sizeof(*dmar), cb);
588 }
589
590 /**
591 * parse_dmar_table - parses the DMA reporting table
592 */
593 static int __init
594 parse_dmar_table(void)
595 {
596 struct acpi_table_dmar *dmar;
597 int ret = 0;
598 int drhd_count = 0;
599 struct dmar_res_callback cb = {
600 .print_entry = true,
601 .ignore_unhandled = true,
602 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
603 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
604 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
605 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
606 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
607 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
608 };
609
610 /*
611 * Do it again, earlier dmar_tbl mapping could be mapped with
612 * fixed map.
613 */
614 dmar_table_detect();
615
616 /*
617 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
618 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
619 */
620 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
621
622 dmar = (struct acpi_table_dmar *)dmar_tbl;
623 if (!dmar)
624 return -ENODEV;
625
626 if (dmar->width < PAGE_SHIFT - 1) {
627 pr_warn("Invalid DMAR haw\n");
628 return -EINVAL;
629 }
630
631 pr_info("Host address width %d\n", dmar->width + 1);
632 ret = dmar_walk_dmar_table(dmar, &cb);
633 if (ret == 0 && drhd_count == 0)
634 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
635
636 return ret;
637 }
638
639 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
640 int cnt, struct pci_dev *dev)
641 {
642 int index;
643 struct device *tmp;
644
645 while (dev) {
646 for_each_active_dev_scope(devices, cnt, index, tmp)
647 if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
648 return 1;
649
650 /* Check our parent */
651 dev = dev->bus->self;
652 }
653
654 return 0;
655 }
656
657 struct dmar_drhd_unit *
658 dmar_find_matched_drhd_unit(struct pci_dev *dev)
659 {
660 struct dmar_drhd_unit *dmaru;
661 struct acpi_dmar_hardware_unit *drhd;
662
663 dev = pci_physfn(dev);
664
665 rcu_read_lock();
666 for_each_drhd_unit(dmaru) {
667 drhd = container_of(dmaru->hdr,
668 struct acpi_dmar_hardware_unit,
669 header);
670
671 if (dmaru->include_all &&
672 drhd->segment == pci_domain_nr(dev->bus))
673 goto out;
674
675 if (dmar_pci_device_match(dmaru->devices,
676 dmaru->devices_cnt, dev))
677 goto out;
678 }
679 dmaru = NULL;
680 out:
681 rcu_read_unlock();
682
683 return dmaru;
684 }
685
686 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
687 struct acpi_device *adev)
688 {
689 struct dmar_drhd_unit *dmaru;
690 struct acpi_dmar_hardware_unit *drhd;
691 struct acpi_dmar_device_scope *scope;
692 struct device *tmp;
693 int i;
694 struct acpi_dmar_pci_path *path;
695
696 for_each_drhd_unit(dmaru) {
697 drhd = container_of(dmaru->hdr,
698 struct acpi_dmar_hardware_unit,
699 header);
700
701 for (scope = (void *)(drhd + 1);
702 (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
703 scope = ((void *)scope) + scope->length) {
704 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
705 continue;
706 if (scope->enumeration_id != device_number)
707 continue;
708
709 path = (void *)(scope + 1);
710 pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
711 dev_name(&adev->dev), dmaru->reg_base_addr,
712 scope->bus, path->device, path->function);
713 for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
714 if (tmp == NULL) {
715 dmaru->devices[i].bus = scope->bus;
716 dmaru->devices[i].devfn = PCI_DEVFN(path->device,
717 path->function);
718 rcu_assign_pointer(dmaru->devices[i].dev,
719 get_device(&adev->dev));
720 return;
721 }
722 BUG_ON(i >= dmaru->devices_cnt);
723 }
724 }
725 pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
726 device_number, dev_name(&adev->dev));
727 }
728
729 static int __init dmar_acpi_dev_scope_init(void)
730 {
731 struct acpi_dmar_andd *andd;
732
733 if (dmar_tbl == NULL)
734 return -ENODEV;
735
736 for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
737 ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
738 andd = ((void *)andd) + andd->header.length) {
739 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
740 acpi_handle h;
741 struct acpi_device *adev;
742
743 if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
744 andd->device_name,
745 &h))) {
746 pr_err("Failed to find handle for ACPI object %s\n",
747 andd->device_name);
748 continue;
749 }
750 if (acpi_bus_get_device(h, &adev)) {
751 pr_err("Failed to get device for ACPI object %s\n",
752 andd->device_name);
753 continue;
754 }
755 dmar_acpi_insert_dev_scope(andd->device_number, adev);
756 }
757 }
758 return 0;
759 }
760
761 int __init dmar_dev_scope_init(void)
762 {
763 struct pci_dev *dev = NULL;
764 struct dmar_pci_notify_info *info;
765
766 if (dmar_dev_scope_status != 1)
767 return dmar_dev_scope_status;
768
769 if (list_empty(&dmar_drhd_units)) {
770 dmar_dev_scope_status = -ENODEV;
771 } else {
772 dmar_dev_scope_status = 0;
773
774 dmar_acpi_dev_scope_init();
775
776 for_each_pci_dev(dev) {
777 if (dev->is_virtfn)
778 continue;
779
780 info = dmar_alloc_pci_notify_info(dev,
781 BUS_NOTIFY_ADD_DEVICE);
782 if (!info) {
783 return dmar_dev_scope_status;
784 } else {
785 dmar_pci_bus_add_dev(info);
786 dmar_free_pci_notify_info(info);
787 }
788 }
789
790 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
791 }
792
793 return dmar_dev_scope_status;
794 }
795
796
797 int __init dmar_table_init(void)
798 {
799 static int dmar_table_initialized;
800 int ret;
801
802 if (dmar_table_initialized == 0) {
803 ret = parse_dmar_table();
804 if (ret < 0) {
805 if (ret != -ENODEV)
806 pr_info("Parse DMAR table failure.\n");
807 } else if (list_empty(&dmar_drhd_units)) {
808 pr_info("No DMAR devices found\n");
809 ret = -ENODEV;
810 }
811
812 if (ret < 0)
813 dmar_table_initialized = ret;
814 else
815 dmar_table_initialized = 1;
816 }
817
818 return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
819 }
820
821 static void warn_invalid_dmar(u64 addr, const char *message)
822 {
823 WARN_TAINT_ONCE(
824 1, TAINT_FIRMWARE_WORKAROUND,
825 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
826 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
827 addr, message,
828 dmi_get_system_info(DMI_BIOS_VENDOR),
829 dmi_get_system_info(DMI_BIOS_VERSION),
830 dmi_get_system_info(DMI_PRODUCT_VERSION));
831 }
832
833 static int __ref
834 dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
835 {
836 struct acpi_dmar_hardware_unit *drhd;
837 void __iomem *addr;
838 u64 cap, ecap;
839
840 drhd = (void *)entry;
841 if (!drhd->address) {
842 warn_invalid_dmar(0, "");
843 return -EINVAL;
844 }
845
846 if (arg)
847 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
848 else
849 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
850 if (!addr) {
851 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
852 return -EINVAL;
853 }
854
855 cap = dmar_readq(addr + DMAR_CAP_REG);
856 ecap = dmar_readq(addr + DMAR_ECAP_REG);
857
858 if (arg)
859 iounmap(addr);
860 else
861 early_iounmap(addr, VTD_PAGE_SIZE);
862
863 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
864 warn_invalid_dmar(drhd->address, " returns all ones");
865 return -EINVAL;
866 }
867
868 return 0;
869 }
870
871 int __init detect_intel_iommu(void)
872 {
873 int ret;
874 struct dmar_res_callback validate_drhd_cb = {
875 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
876 .ignore_unhandled = true,
877 };
878
879 down_write(&dmar_global_lock);
880 ret = dmar_table_detect();
881 if (ret)
882 ret = !dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
883 &validate_drhd_cb);
884 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
885 iommu_detected = 1;
886 /* Make sure ACS will be enabled */
887 pci_request_acs();
888 }
889
890 #ifdef CONFIG_X86
891 if (ret)
892 x86_init.iommu.iommu_init = intel_iommu_init;
893 #endif
894
895 early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
896 dmar_tbl = NULL;
897 up_write(&dmar_global_lock);
898
899 return ret ? 1 : -ENODEV;
900 }
901
902
903 static void unmap_iommu(struct intel_iommu *iommu)
904 {
905 iounmap(iommu->reg);
906 release_mem_region(iommu->reg_phys, iommu->reg_size);
907 }
908
909 /**
910 * map_iommu: map the iommu's registers
911 * @iommu: the iommu to map
912 * @phys_addr: the physical address of the base resgister
913 *
914 * Memory map the iommu's registers. Start w/ a single page, and
915 * possibly expand if that turns out to be insufficent.
916 */
917 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
918 {
919 int map_size, err=0;
920
921 iommu->reg_phys = phys_addr;
922 iommu->reg_size = VTD_PAGE_SIZE;
923
924 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
925 pr_err("Can't reserve memory\n");
926 err = -EBUSY;
927 goto out;
928 }
929
930 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
931 if (!iommu->reg) {
932 pr_err("Can't map the region\n");
933 err = -ENOMEM;
934 goto release;
935 }
936
937 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
938 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
939
940 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
941 err = -EINVAL;
942 warn_invalid_dmar(phys_addr, " returns all ones");
943 goto unmap;
944 }
945
946 /* the registers might be more than one page */
947 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
948 cap_max_fault_reg_offset(iommu->cap));
949 map_size = VTD_PAGE_ALIGN(map_size);
950 if (map_size > iommu->reg_size) {
951 iounmap(iommu->reg);
952 release_mem_region(iommu->reg_phys, iommu->reg_size);
953 iommu->reg_size = map_size;
954 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
955 iommu->name)) {
956 pr_err("Can't reserve memory\n");
957 err = -EBUSY;
958 goto out;
959 }
960 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
961 if (!iommu->reg) {
962 pr_err("Can't map the region\n");
963 err = -ENOMEM;
964 goto release;
965 }
966 }
967 err = 0;
968 goto out;
969
970 unmap:
971 iounmap(iommu->reg);
972 release:
973 release_mem_region(iommu->reg_phys, iommu->reg_size);
974 out:
975 return err;
976 }
977
978 static int dmar_alloc_seq_id(struct intel_iommu *iommu)
979 {
980 iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
981 DMAR_UNITS_SUPPORTED);
982 if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
983 iommu->seq_id = -1;
984 } else {
985 set_bit(iommu->seq_id, dmar_seq_ids);
986 sprintf(iommu->name, "dmar%d", iommu->seq_id);
987 }
988
989 return iommu->seq_id;
990 }
991
992 static void dmar_free_seq_id(struct intel_iommu *iommu)
993 {
994 if (iommu->seq_id >= 0) {
995 clear_bit(iommu->seq_id, dmar_seq_ids);
996 iommu->seq_id = -1;
997 }
998 }
999
1000 static int alloc_iommu(struct dmar_drhd_unit *drhd)
1001 {
1002 struct intel_iommu *iommu;
1003 u32 ver, sts;
1004 int agaw = 0;
1005 int msagaw = 0;
1006 int err;
1007
1008 if (!drhd->reg_base_addr) {
1009 warn_invalid_dmar(0, "");
1010 return -EINVAL;
1011 }
1012
1013 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1014 if (!iommu)
1015 return -ENOMEM;
1016
1017 if (dmar_alloc_seq_id(iommu) < 0) {
1018 pr_err("Failed to allocate seq_id\n");
1019 err = -ENOSPC;
1020 goto error;
1021 }
1022
1023 err = map_iommu(iommu, drhd->reg_base_addr);
1024 if (err) {
1025 pr_err("Failed to map %s\n", iommu->name);
1026 goto error_free_seq_id;
1027 }
1028
1029 err = -EINVAL;
1030 agaw = iommu_calculate_agaw(iommu);
1031 if (agaw < 0) {
1032 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1033 iommu->seq_id);
1034 goto err_unmap;
1035 }
1036 msagaw = iommu_calculate_max_sagaw(iommu);
1037 if (msagaw < 0) {
1038 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1039 iommu->seq_id);
1040 goto err_unmap;
1041 }
1042 iommu->agaw = agaw;
1043 iommu->msagaw = msagaw;
1044 iommu->segment = drhd->segment;
1045
1046 iommu->node = -1;
1047
1048 ver = readl(iommu->reg + DMAR_VER_REG);
1049 pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1050 iommu->name,
1051 (unsigned long long)drhd->reg_base_addr,
1052 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1053 (unsigned long long)iommu->cap,
1054 (unsigned long long)iommu->ecap);
1055
1056 /* Reflect status in gcmd */
1057 sts = readl(iommu->reg + DMAR_GSTS_REG);
1058 if (sts & DMA_GSTS_IRES)
1059 iommu->gcmd |= DMA_GCMD_IRE;
1060 if (sts & DMA_GSTS_TES)
1061 iommu->gcmd |= DMA_GCMD_TE;
1062 if (sts & DMA_GSTS_QIES)
1063 iommu->gcmd |= DMA_GCMD_QIE;
1064
1065 raw_spin_lock_init(&iommu->register_lock);
1066
1067 if (intel_iommu_enabled) {
1068 iommu->iommu_dev = iommu_device_create(NULL, iommu,
1069 intel_iommu_groups,
1070 "%s", iommu->name);
1071
1072 if (IS_ERR(iommu->iommu_dev)) {
1073 err = PTR_ERR(iommu->iommu_dev);
1074 goto err_unmap;
1075 }
1076 }
1077
1078 drhd->iommu = iommu;
1079
1080 return 0;
1081
1082 err_unmap:
1083 unmap_iommu(iommu);
1084 error_free_seq_id:
1085 dmar_free_seq_id(iommu);
1086 error:
1087 kfree(iommu);
1088 return err;
1089 }
1090
1091 static void free_iommu(struct intel_iommu *iommu)
1092 {
1093 iommu_device_destroy(iommu->iommu_dev);
1094
1095 if (iommu->irq) {
1096 if (iommu->pr_irq) {
1097 free_irq(iommu->pr_irq, iommu);
1098 dmar_free_hwirq(iommu->pr_irq);
1099 iommu->pr_irq = 0;
1100 }
1101 free_irq(iommu->irq, iommu);
1102 dmar_free_hwirq(iommu->irq);
1103 iommu->irq = 0;
1104 }
1105
1106 if (iommu->qi) {
1107 free_page((unsigned long)iommu->qi->desc);
1108 kfree(iommu->qi->desc_status);
1109 kfree(iommu->qi);
1110 }
1111
1112 if (iommu->reg)
1113 unmap_iommu(iommu);
1114
1115 dmar_free_seq_id(iommu);
1116 kfree(iommu);
1117 }
1118
1119 /*
1120 * Reclaim all the submitted descriptors which have completed its work.
1121 */
1122 static inline void reclaim_free_desc(struct q_inval *qi)
1123 {
1124 while (qi->desc_status[qi->free_tail] == QI_DONE ||
1125 qi->desc_status[qi->free_tail] == QI_ABORT) {
1126 qi->desc_status[qi->free_tail] = QI_FREE;
1127 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1128 qi->free_cnt++;
1129 }
1130 }
1131
1132 static int qi_check_fault(struct intel_iommu *iommu, int index)
1133 {
1134 u32 fault;
1135 int head, tail;
1136 struct q_inval *qi = iommu->qi;
1137 int wait_index = (index + 1) % QI_LENGTH;
1138
1139 if (qi->desc_status[wait_index] == QI_ABORT)
1140 return -EAGAIN;
1141
1142 fault = readl(iommu->reg + DMAR_FSTS_REG);
1143
1144 /*
1145 * If IQE happens, the head points to the descriptor associated
1146 * with the error. No new descriptors are fetched until the IQE
1147 * is cleared.
1148 */
1149 if (fault & DMA_FSTS_IQE) {
1150 head = readl(iommu->reg + DMAR_IQH_REG);
1151 if ((head >> DMAR_IQ_SHIFT) == index) {
1152 pr_err("VT-d detected invalid descriptor: "
1153 "low=%llx, high=%llx\n",
1154 (unsigned long long)qi->desc[index].low,
1155 (unsigned long long)qi->desc[index].high);
1156 memcpy(&qi->desc[index], &qi->desc[wait_index],
1157 sizeof(struct qi_desc));
1158 __iommu_flush_cache(iommu, &qi->desc[index],
1159 sizeof(struct qi_desc));
1160 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1161 return -EINVAL;
1162 }
1163 }
1164
1165 /*
1166 * If ITE happens, all pending wait_desc commands are aborted.
1167 * No new descriptors are fetched until the ITE is cleared.
1168 */
1169 if (fault & DMA_FSTS_ITE) {
1170 head = readl(iommu->reg + DMAR_IQH_REG);
1171 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1172 head |= 1;
1173 tail = readl(iommu->reg + DMAR_IQT_REG);
1174 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1175
1176 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1177
1178 do {
1179 if (qi->desc_status[head] == QI_IN_USE)
1180 qi->desc_status[head] = QI_ABORT;
1181 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1182 } while (head != tail);
1183
1184 if (qi->desc_status[wait_index] == QI_ABORT)
1185 return -EAGAIN;
1186 }
1187
1188 if (fault & DMA_FSTS_ICE)
1189 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1190
1191 return 0;
1192 }
1193
1194 /*
1195 * Submit the queued invalidation descriptor to the remapping
1196 * hardware unit and wait for its completion.
1197 */
1198 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1199 {
1200 int rc;
1201 struct q_inval *qi = iommu->qi;
1202 struct qi_desc *hw, wait_desc;
1203 int wait_index, index;
1204 unsigned long flags;
1205
1206 if (!qi)
1207 return 0;
1208
1209 hw = qi->desc;
1210
1211 restart:
1212 rc = 0;
1213
1214 raw_spin_lock_irqsave(&qi->q_lock, flags);
1215 while (qi->free_cnt < 3) {
1216 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1217 cpu_relax();
1218 raw_spin_lock_irqsave(&qi->q_lock, flags);
1219 }
1220
1221 index = qi->free_head;
1222 wait_index = (index + 1) % QI_LENGTH;
1223
1224 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1225
1226 hw[index] = *desc;
1227
1228 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1229 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1230 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1231
1232 hw[wait_index] = wait_desc;
1233
1234 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
1235 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
1236
1237 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1238 qi->free_cnt -= 2;
1239
1240 /*
1241 * update the HW tail register indicating the presence of
1242 * new descriptors.
1243 */
1244 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
1245
1246 while (qi->desc_status[wait_index] != QI_DONE) {
1247 /*
1248 * We will leave the interrupts disabled, to prevent interrupt
1249 * context to queue another cmd while a cmd is already submitted
1250 * and waiting for completion on this cpu. This is to avoid
1251 * a deadlock where the interrupt context can wait indefinitely
1252 * for free slots in the queue.
1253 */
1254 rc = qi_check_fault(iommu, index);
1255 if (rc)
1256 break;
1257
1258 raw_spin_unlock(&qi->q_lock);
1259 cpu_relax();
1260 raw_spin_lock(&qi->q_lock);
1261 }
1262
1263 qi->desc_status[index] = QI_DONE;
1264
1265 reclaim_free_desc(qi);
1266 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1267
1268 if (rc == -EAGAIN)
1269 goto restart;
1270
1271 return rc;
1272 }
1273
1274 /*
1275 * Flush the global interrupt entry cache.
1276 */
1277 void qi_global_iec(struct intel_iommu *iommu)
1278 {
1279 struct qi_desc desc;
1280
1281 desc.low = QI_IEC_TYPE;
1282 desc.high = 0;
1283
1284 /* should never fail */
1285 qi_submit_sync(&desc, iommu);
1286 }
1287
1288 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1289 u64 type)
1290 {
1291 struct qi_desc desc;
1292
1293 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1294 | QI_CC_GRAN(type) | QI_CC_TYPE;
1295 desc.high = 0;
1296
1297 qi_submit_sync(&desc, iommu);
1298 }
1299
1300 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1301 unsigned int size_order, u64 type)
1302 {
1303 u8 dw = 0, dr = 0;
1304
1305 struct qi_desc desc;
1306 int ih = 0;
1307
1308 if (cap_write_drain(iommu->cap))
1309 dw = 1;
1310
1311 if (cap_read_drain(iommu->cap))
1312 dr = 1;
1313
1314 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1315 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1316 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1317 | QI_IOTLB_AM(size_order);
1318
1319 qi_submit_sync(&desc, iommu);
1320 }
1321
1322 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1323 u64 addr, unsigned mask)
1324 {
1325 struct qi_desc desc;
1326
1327 if (mask) {
1328 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1329 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1330 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1331 } else
1332 desc.high = QI_DEV_IOTLB_ADDR(addr);
1333
1334 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1335 qdep = 0;
1336
1337 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1338 QI_DIOTLB_TYPE;
1339
1340 qi_submit_sync(&desc, iommu);
1341 }
1342
1343 /*
1344 * Disable Queued Invalidation interface.
1345 */
1346 void dmar_disable_qi(struct intel_iommu *iommu)
1347 {
1348 unsigned long flags;
1349 u32 sts;
1350 cycles_t start_time = get_cycles();
1351
1352 if (!ecap_qis(iommu->ecap))
1353 return;
1354
1355 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1356
1357 sts = readl(iommu->reg + DMAR_GSTS_REG);
1358 if (!(sts & DMA_GSTS_QIES))
1359 goto end;
1360
1361 /*
1362 * Give a chance to HW to complete the pending invalidation requests.
1363 */
1364 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1365 readl(iommu->reg + DMAR_IQH_REG)) &&
1366 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1367 cpu_relax();
1368
1369 iommu->gcmd &= ~DMA_GCMD_QIE;
1370 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1371
1372 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1373 !(sts & DMA_GSTS_QIES), sts);
1374 end:
1375 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1376 }
1377
1378 /*
1379 * Enable queued invalidation.
1380 */
1381 static void __dmar_enable_qi(struct intel_iommu *iommu)
1382 {
1383 u32 sts;
1384 unsigned long flags;
1385 struct q_inval *qi = iommu->qi;
1386
1387 qi->free_head = qi->free_tail = 0;
1388 qi->free_cnt = QI_LENGTH;
1389
1390 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1391
1392 /* write zero to the tail reg */
1393 writel(0, iommu->reg + DMAR_IQT_REG);
1394
1395 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1396
1397 iommu->gcmd |= DMA_GCMD_QIE;
1398 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1399
1400 /* Make sure hardware complete it */
1401 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1402
1403 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1404 }
1405
1406 /*
1407 * Enable Queued Invalidation interface. This is a must to support
1408 * interrupt-remapping. Also used by DMA-remapping, which replaces
1409 * register based IOTLB invalidation.
1410 */
1411 int dmar_enable_qi(struct intel_iommu *iommu)
1412 {
1413 struct q_inval *qi;
1414 struct page *desc_page;
1415
1416 if (!ecap_qis(iommu->ecap))
1417 return -ENOENT;
1418
1419 /*
1420 * queued invalidation is already setup and enabled.
1421 */
1422 if (iommu->qi)
1423 return 0;
1424
1425 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1426 if (!iommu->qi)
1427 return -ENOMEM;
1428
1429 qi = iommu->qi;
1430
1431
1432 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1433 if (!desc_page) {
1434 kfree(qi);
1435 iommu->qi = NULL;
1436 return -ENOMEM;
1437 }
1438
1439 qi->desc = page_address(desc_page);
1440
1441 qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1442 if (!qi->desc_status) {
1443 free_page((unsigned long) qi->desc);
1444 kfree(qi);
1445 iommu->qi = NULL;
1446 return -ENOMEM;
1447 }
1448
1449 raw_spin_lock_init(&qi->q_lock);
1450
1451 __dmar_enable_qi(iommu);
1452
1453 return 0;
1454 }
1455
1456 /* iommu interrupt handling. Most stuff are MSI-like. */
1457
1458 enum faulttype {
1459 DMA_REMAP,
1460 INTR_REMAP,
1461 UNKNOWN,
1462 };
1463
1464 static const char *dma_remap_fault_reasons[] =
1465 {
1466 "Software",
1467 "Present bit in root entry is clear",
1468 "Present bit in context entry is clear",
1469 "Invalid context entry",
1470 "Access beyond MGAW",
1471 "PTE Write access is not set",
1472 "PTE Read access is not set",
1473 "Next page table ptr is invalid",
1474 "Root table address invalid",
1475 "Context table ptr is invalid",
1476 "non-zero reserved fields in RTP",
1477 "non-zero reserved fields in CTP",
1478 "non-zero reserved fields in PTE",
1479 "PCE for translation request specifies blocking",
1480 };
1481
1482 static const char *irq_remap_fault_reasons[] =
1483 {
1484 "Detected reserved fields in the decoded interrupt-remapped request",
1485 "Interrupt index exceeded the interrupt-remapping table size",
1486 "Present field in the IRTE entry is clear",
1487 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1488 "Detected reserved fields in the IRTE entry",
1489 "Blocked a compatibility format interrupt request",
1490 "Blocked an interrupt request due to source-id verification failure",
1491 };
1492
1493 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1494 {
1495 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1496 ARRAY_SIZE(irq_remap_fault_reasons))) {
1497 *fault_type = INTR_REMAP;
1498 return irq_remap_fault_reasons[fault_reason - 0x20];
1499 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1500 *fault_type = DMA_REMAP;
1501 return dma_remap_fault_reasons[fault_reason];
1502 } else {
1503 *fault_type = UNKNOWN;
1504 return "Unknown";
1505 }
1506 }
1507
1508
1509 static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1510 {
1511 if (iommu->irq == irq)
1512 return DMAR_FECTL_REG;
1513 else if (iommu->pr_irq == irq)
1514 return DMAR_PECTL_REG;
1515 else
1516 BUG();
1517 }
1518
1519 void dmar_msi_unmask(struct irq_data *data)
1520 {
1521 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1522 int reg = dmar_msi_reg(iommu, data->irq);
1523 unsigned long flag;
1524
1525 /* unmask it */
1526 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1527 writel(0, iommu->reg + reg);
1528 /* Read a reg to force flush the post write */
1529 readl(iommu->reg + reg);
1530 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1531 }
1532
1533 void dmar_msi_mask(struct irq_data *data)
1534 {
1535 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1536 int reg = dmar_msi_reg(iommu, data->irq);
1537 unsigned long flag;
1538
1539 /* mask it */
1540 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1541 writel(DMA_FECTL_IM, iommu->reg + reg);
1542 /* Read a reg to force flush the post write */
1543 readl(iommu->reg + reg);
1544 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1545 }
1546
1547 void dmar_msi_write(int irq, struct msi_msg *msg)
1548 {
1549 struct intel_iommu *iommu = irq_get_handler_data(irq);
1550 int reg = dmar_msi_reg(iommu, irq);
1551 unsigned long flag;
1552
1553 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1554 writel(msg->data, iommu->reg + reg + 4);
1555 writel(msg->address_lo, iommu->reg + reg + 8);
1556 writel(msg->address_hi, iommu->reg + reg + 12);
1557 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1558 }
1559
1560 void dmar_msi_read(int irq, struct msi_msg *msg)
1561 {
1562 struct intel_iommu *iommu = irq_get_handler_data(irq);
1563 int reg = dmar_msi_reg(iommu, irq);
1564 unsigned long flag;
1565
1566 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1567 msg->data = readl(iommu->reg + reg + 4);
1568 msg->address_lo = readl(iommu->reg + reg + 8);
1569 msg->address_hi = readl(iommu->reg + reg + 12);
1570 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1571 }
1572
1573 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1574 u8 fault_reason, u16 source_id, unsigned long long addr)
1575 {
1576 const char *reason;
1577 int fault_type;
1578
1579 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1580
1581 if (fault_type == INTR_REMAP)
1582 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1583 "fault index %llx\n"
1584 "INTR-REMAP:[fault reason %02d] %s\n",
1585 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1586 PCI_FUNC(source_id & 0xFF), addr >> 48,
1587 fault_reason, reason);
1588 else
1589 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1590 "fault addr %llx \n"
1591 "DMAR:[fault reason %02d] %s\n",
1592 (type ? "DMA Read" : "DMA Write"),
1593 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1594 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1595 return 0;
1596 }
1597
1598 #define PRIMARY_FAULT_REG_LEN (16)
1599 irqreturn_t dmar_fault(int irq, void *dev_id)
1600 {
1601 struct intel_iommu *iommu = dev_id;
1602 int reg, fault_index;
1603 u32 fault_status;
1604 unsigned long flag;
1605
1606 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1607 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1608 if (fault_status)
1609 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1610
1611 /* TBD: ignore advanced fault log currently */
1612 if (!(fault_status & DMA_FSTS_PPF))
1613 goto unlock_exit;
1614
1615 fault_index = dma_fsts_fault_record_index(fault_status);
1616 reg = cap_fault_reg_offset(iommu->cap);
1617 while (1) {
1618 u8 fault_reason;
1619 u16 source_id;
1620 u64 guest_addr;
1621 int type;
1622 u32 data;
1623
1624 /* highest 32 bits */
1625 data = readl(iommu->reg + reg +
1626 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1627 if (!(data & DMA_FRCD_F))
1628 break;
1629
1630 fault_reason = dma_frcd_fault_reason(data);
1631 type = dma_frcd_type(data);
1632
1633 data = readl(iommu->reg + reg +
1634 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1635 source_id = dma_frcd_source_id(data);
1636
1637 guest_addr = dmar_readq(iommu->reg + reg +
1638 fault_index * PRIMARY_FAULT_REG_LEN);
1639 guest_addr = dma_frcd_page_addr(guest_addr);
1640 /* clear the fault */
1641 writel(DMA_FRCD_F, iommu->reg + reg +
1642 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1643
1644 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1645
1646 dmar_fault_do_one(iommu, type, fault_reason,
1647 source_id, guest_addr);
1648
1649 fault_index++;
1650 if (fault_index >= cap_num_fault_regs(iommu->cap))
1651 fault_index = 0;
1652 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1653 }
1654
1655 writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1656
1657 unlock_exit:
1658 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1659 return IRQ_HANDLED;
1660 }
1661
1662 int dmar_set_interrupt(struct intel_iommu *iommu)
1663 {
1664 int irq, ret;
1665
1666 /*
1667 * Check if the fault interrupt is already initialized.
1668 */
1669 if (iommu->irq)
1670 return 0;
1671
1672 irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
1673 if (irq > 0) {
1674 iommu->irq = irq;
1675 } else {
1676 pr_err("No free IRQ vectors\n");
1677 return -EINVAL;
1678 }
1679
1680 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1681 if (ret)
1682 pr_err("Can't request irq\n");
1683 return ret;
1684 }
1685
1686 int __init enable_drhd_fault_handling(void)
1687 {
1688 struct dmar_drhd_unit *drhd;
1689 struct intel_iommu *iommu;
1690
1691 /*
1692 * Enable fault control interrupt.
1693 */
1694 for_each_iommu(iommu, drhd) {
1695 u32 fault_status;
1696 int ret = dmar_set_interrupt(iommu);
1697
1698 if (ret) {
1699 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1700 (unsigned long long)drhd->reg_base_addr, ret);
1701 return -1;
1702 }
1703
1704 /*
1705 * Clear any previous faults.
1706 */
1707 dmar_fault(iommu->irq, iommu);
1708 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1709 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1710 }
1711
1712 return 0;
1713 }
1714
1715 /*
1716 * Re-enable Queued Invalidation interface.
1717 */
1718 int dmar_reenable_qi(struct intel_iommu *iommu)
1719 {
1720 if (!ecap_qis(iommu->ecap))
1721 return -ENOENT;
1722
1723 if (!iommu->qi)
1724 return -ENOENT;
1725
1726 /*
1727 * First disable queued invalidation.
1728 */
1729 dmar_disable_qi(iommu);
1730 /*
1731 * Then enable queued invalidation again. Since there is no pending
1732 * invalidation requests now, it's safe to re-enable queued
1733 * invalidation.
1734 */
1735 __dmar_enable_qi(iommu);
1736
1737 return 0;
1738 }
1739
1740 /*
1741 * Check interrupt remapping support in DMAR table description.
1742 */
1743 int __init dmar_ir_support(void)
1744 {
1745 struct acpi_table_dmar *dmar;
1746 dmar = (struct acpi_table_dmar *)dmar_tbl;
1747 if (!dmar)
1748 return 0;
1749 return dmar->flags & 0x1;
1750 }
1751
1752 /* Check whether DMAR units are in use */
1753 static inline bool dmar_in_use(void)
1754 {
1755 return irq_remapping_enabled || intel_iommu_enabled;
1756 }
1757
1758 static int __init dmar_free_unused_resources(void)
1759 {
1760 struct dmar_drhd_unit *dmaru, *dmaru_n;
1761
1762 if (dmar_in_use())
1763 return 0;
1764
1765 if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1766 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1767
1768 down_write(&dmar_global_lock);
1769 list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1770 list_del(&dmaru->list);
1771 dmar_free_drhd(dmaru);
1772 }
1773 up_write(&dmar_global_lock);
1774
1775 return 0;
1776 }
1777
1778 late_initcall(dmar_free_unused_resources);
1779 IOMMU_INIT_POST(detect_intel_iommu);
1780
1781 /*
1782 * DMAR Hotplug Support
1783 * For more details, please refer to Intel(R) Virtualization Technology
1784 * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
1785 * "Remapping Hardware Unit Hot Plug".
1786 */
1787 static u8 dmar_hp_uuid[] = {
1788 /* 0000 */ 0xA6, 0xA3, 0xC1, 0xD8, 0x9B, 0xBE, 0x9B, 0x4C,
1789 /* 0008 */ 0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF
1790 };
1791
1792 /*
1793 * Currently there's only one revision and BIOS will not check the revision id,
1794 * so use 0 for safety.
1795 */
1796 #define DMAR_DSM_REV_ID 0
1797 #define DMAR_DSM_FUNC_DRHD 1
1798 #define DMAR_DSM_FUNC_ATSR 2
1799 #define DMAR_DSM_FUNC_RHSA 3
1800
1801 static inline bool dmar_detect_dsm(acpi_handle handle, int func)
1802 {
1803 return acpi_check_dsm(handle, dmar_hp_uuid, DMAR_DSM_REV_ID, 1 << func);
1804 }
1805
1806 static int dmar_walk_dsm_resource(acpi_handle handle, int func,
1807 dmar_res_handler_t handler, void *arg)
1808 {
1809 int ret = -ENODEV;
1810 union acpi_object *obj;
1811 struct acpi_dmar_header *start;
1812 struct dmar_res_callback callback;
1813 static int res_type[] = {
1814 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
1815 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
1816 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
1817 };
1818
1819 if (!dmar_detect_dsm(handle, func))
1820 return 0;
1821
1822 obj = acpi_evaluate_dsm_typed(handle, dmar_hp_uuid, DMAR_DSM_REV_ID,
1823 func, NULL, ACPI_TYPE_BUFFER);
1824 if (!obj)
1825 return -ENODEV;
1826
1827 memset(&callback, 0, sizeof(callback));
1828 callback.cb[res_type[func]] = handler;
1829 callback.arg[res_type[func]] = arg;
1830 start = (struct acpi_dmar_header *)obj->buffer.pointer;
1831 ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
1832
1833 ACPI_FREE(obj);
1834
1835 return ret;
1836 }
1837
1838 static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
1839 {
1840 int ret;
1841 struct dmar_drhd_unit *dmaru;
1842
1843 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1844 if (!dmaru)
1845 return -ENODEV;
1846
1847 ret = dmar_ir_hotplug(dmaru, true);
1848 if (ret == 0)
1849 ret = dmar_iommu_hotplug(dmaru, true);
1850
1851 return ret;
1852 }
1853
1854 static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
1855 {
1856 int i, ret;
1857 struct device *dev;
1858 struct dmar_drhd_unit *dmaru;
1859
1860 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1861 if (!dmaru)
1862 return 0;
1863
1864 /*
1865 * All PCI devices managed by this unit should have been destroyed.
1866 */
1867 if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt)
1868 for_each_active_dev_scope(dmaru->devices,
1869 dmaru->devices_cnt, i, dev)
1870 return -EBUSY;
1871
1872 ret = dmar_ir_hotplug(dmaru, false);
1873 if (ret == 0)
1874 ret = dmar_iommu_hotplug(dmaru, false);
1875
1876 return ret;
1877 }
1878
1879 static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
1880 {
1881 struct dmar_drhd_unit *dmaru;
1882
1883 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1884 if (dmaru) {
1885 list_del_rcu(&dmaru->list);
1886 synchronize_rcu();
1887 dmar_free_drhd(dmaru);
1888 }
1889
1890 return 0;
1891 }
1892
1893 static int dmar_hotplug_insert(acpi_handle handle)
1894 {
1895 int ret;
1896 int drhd_count = 0;
1897
1898 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1899 &dmar_validate_one_drhd, (void *)1);
1900 if (ret)
1901 goto out;
1902
1903 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1904 &dmar_parse_one_drhd, (void *)&drhd_count);
1905 if (ret == 0 && drhd_count == 0) {
1906 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
1907 goto out;
1908 } else if (ret) {
1909 goto release_drhd;
1910 }
1911
1912 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
1913 &dmar_parse_one_rhsa, NULL);
1914 if (ret)
1915 goto release_drhd;
1916
1917 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1918 &dmar_parse_one_atsr, NULL);
1919 if (ret)
1920 goto release_atsr;
1921
1922 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1923 &dmar_hp_add_drhd, NULL);
1924 if (!ret)
1925 return 0;
1926
1927 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1928 &dmar_hp_remove_drhd, NULL);
1929 release_atsr:
1930 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1931 &dmar_release_one_atsr, NULL);
1932 release_drhd:
1933 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1934 &dmar_hp_release_drhd, NULL);
1935 out:
1936 return ret;
1937 }
1938
1939 static int dmar_hotplug_remove(acpi_handle handle)
1940 {
1941 int ret;
1942
1943 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1944 &dmar_check_one_atsr, NULL);
1945 if (ret)
1946 return ret;
1947
1948 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1949 &dmar_hp_remove_drhd, NULL);
1950 if (ret == 0) {
1951 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1952 &dmar_release_one_atsr, NULL));
1953 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1954 &dmar_hp_release_drhd, NULL));
1955 } else {
1956 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1957 &dmar_hp_add_drhd, NULL);
1958 }
1959
1960 return ret;
1961 }
1962
1963 static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
1964 void *context, void **retval)
1965 {
1966 acpi_handle *phdl = retval;
1967
1968 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1969 *phdl = handle;
1970 return AE_CTRL_TERMINATE;
1971 }
1972
1973 return AE_OK;
1974 }
1975
1976 static int dmar_device_hotplug(acpi_handle handle, bool insert)
1977 {
1978 int ret;
1979 acpi_handle tmp = NULL;
1980 acpi_status status;
1981
1982 if (!dmar_in_use())
1983 return 0;
1984
1985 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1986 tmp = handle;
1987 } else {
1988 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
1989 ACPI_UINT32_MAX,
1990 dmar_get_dsm_handle,
1991 NULL, NULL, &tmp);
1992 if (ACPI_FAILURE(status)) {
1993 pr_warn("Failed to locate _DSM method.\n");
1994 return -ENXIO;
1995 }
1996 }
1997 if (tmp == NULL)
1998 return 0;
1999
2000 down_write(&dmar_global_lock);
2001 if (insert)
2002 ret = dmar_hotplug_insert(tmp);
2003 else
2004 ret = dmar_hotplug_remove(tmp);
2005 up_write(&dmar_global_lock);
2006
2007 return ret;
2008 }
2009
2010 int dmar_device_add(acpi_handle handle)
2011 {
2012 return dmar_device_hotplug(handle, true);
2013 }
2014
2015 int dmar_device_remove(acpi_handle handle)
2016 {
2017 return dmar_device_hotplug(handle, false);
2018 }
Linux with added FPC-III support