?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
perf intel-pt: Factor out intel_pt_8b_tsc()
[linux/fpc-iii.git] / drivers / iommu / dmar.c
blob6d969a172fbb3491df6b1d38cc6f3b33cd99a4d0
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 *
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */
28
29 #define pr_fmt(fmt) "DMAR: " fmt
30
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/iova.h>
34 #include <linux/intel-iommu.h>
35 #include <linux/timer.h>
36 #include <linux/irq.h>
37 #include <linux/interrupt.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/slab.h>
41 #include <linux/iommu.h>
42 #include <linux/numa.h>
43 #include <asm/irq_remapping.h>
44 #include <asm/iommu_table.h>
45
46 #include "irq_remapping.h"
47
48 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
49 struct dmar_res_callback {
50 dmar_res_handler_t cb[ACPI_DMAR_TYPE_RESERVED];
51 void *arg[ACPI_DMAR_TYPE_RESERVED];
52 bool ignore_unhandled;
53 bool print_entry;
54 };
55
56 /*
57 * Assumptions:
58 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
59 * before IO devices managed by that unit.
60 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
61 * after IO devices managed by that unit.
62 * 3) Hotplug events are rare.
63 *
64 * Locking rules for DMA and interrupt remapping related global data structures:
65 * 1) Use dmar_global_lock in process context
66 * 2) Use RCU in interrupt context
67 */
68 DECLARE_RWSEM(dmar_global_lock);
69 LIST_HEAD(dmar_drhd_units);
70
71 struct acpi_table_header * __initdata dmar_tbl;
72 static int dmar_dev_scope_status = 1;
73 static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
74
75 static int alloc_iommu(struct dmar_drhd_unit *drhd);
76 static void free_iommu(struct intel_iommu *iommu);
77
78 extern const struct iommu_ops intel_iommu_ops;
79
80 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
81 {
82 /*
83 * add INCLUDE_ALL at the tail, so scan the list will find it at
84 * the very end.
85 */
86 if (drhd->include_all)
87 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
88 else
89 list_add_rcu(&drhd->list, &dmar_drhd_units);
90 }
91
92 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
93 {
94 struct acpi_dmar_device_scope *scope;
95
96 *cnt = 0;
97 while (start < end) {
98 scope = start;
99 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
100 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
101 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
102 (*cnt)++;
103 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
104 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
105 pr_warn("Unsupported device scope\n");
106 }
107 start += scope->length;
108 }
109 if (*cnt == 0)
110 return NULL;
111
112 return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
113 }
114
115 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
116 {
117 int i;
118 struct device *tmp_dev;
119
120 if (*devices && *cnt) {
121 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
122 put_device(tmp_dev);
123 kfree(*devices);
124 }
125
126 *devices = NULL;
127 *cnt = 0;
128 }
129
130 /* Optimize out kzalloc()/kfree() for normal cases */
131 static char dmar_pci_notify_info_buf[64];
132
133 static struct dmar_pci_notify_info *
134 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
135 {
136 int level = 0;
137 size_t size;
138 struct pci_dev *tmp;
139 struct dmar_pci_notify_info *info;
140
141 BUG_ON(dev->is_virtfn);
142
143 /* Only generate path[] for device addition event */
144 if (event == BUS_NOTIFY_ADD_DEVICE)
145 for (tmp = dev; tmp; tmp = tmp->bus->self)
146 level++;
147
148 size = struct_size(info, path, level);
149 if (size <= sizeof(dmar_pci_notify_info_buf)) {
150 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
151 } else {
152 info = kzalloc(size, GFP_KERNEL);
153 if (!info) {
154 pr_warn("Out of memory when allocating notify_info "
155 "for %s.\n", pci_name(dev));
156 if (dmar_dev_scope_status == 0)
157 dmar_dev_scope_status = -ENOMEM;
158 return NULL;
159 }
160 }
161
162 info->event = event;
163 info->dev = dev;
164 info->seg = pci_domain_nr(dev->bus);
165 info->level = level;
166 if (event == BUS_NOTIFY_ADD_DEVICE) {
167 for (tmp = dev; tmp; tmp = tmp->bus->self) {
168 level--;
169 info->path[level].bus = tmp->bus->number;
170 info->path[level].device = PCI_SLOT(tmp->devfn);
171 info->path[level].function = PCI_FUNC(tmp->devfn);
172 if (pci_is_root_bus(tmp->bus))
173 info->bus = tmp->bus->number;
174 }
175 }
176
177 return info;
178 }
179
180 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
181 {
182 if ((void *)info != dmar_pci_notify_info_buf)
183 kfree(info);
184 }
185
186 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
187 struct acpi_dmar_pci_path *path, int count)
188 {
189 int i;
190
191 if (info->bus != bus)
192 goto fallback;
193 if (info->level != count)
194 goto fallback;
195
196 for (i = 0; i < count; i++) {
197 if (path[i].device != info->path[i].device ||
198 path[i].function != info->path[i].function)
199 goto fallback;
200 }
201
202 return true;
203
204 fallback:
205
206 if (count != 1)
207 return false;
208
209 i = info->level - 1;
210 if (bus == info->path[i].bus &&
211 path[0].device == info->path[i].device &&
212 path[0].function == info->path[i].function) {
213 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
214 bus, path[0].device, path[0].function);
215 return true;
216 }
217
218 return false;
219 }
220
221 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
222 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
223 void *start, void*end, u16 segment,
224 struct dmar_dev_scope *devices,
225 int devices_cnt)
226 {
227 int i, level;
228 struct device *tmp, *dev = &info->dev->dev;
229 struct acpi_dmar_device_scope *scope;
230 struct acpi_dmar_pci_path *path;
231
232 if (segment != info->seg)
233 return 0;
234
235 for (; start < end; start += scope->length) {
236 scope = start;
237 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
238 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
239 continue;
240
241 path = (struct acpi_dmar_pci_path *)(scope + 1);
242 level = (scope->length - sizeof(*scope)) / sizeof(*path);
243 if (!dmar_match_pci_path(info, scope->bus, path, level))
244 continue;
245
246 /*
247 * We expect devices with endpoint scope to have normal PCI
248 * headers, and devices with bridge scope to have bridge PCI
249 * headers. However PCI NTB devices may be listed in the
250 * DMAR table with bridge scope, even though they have a
251 * normal PCI header. NTB devices are identified by class
252 * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch
253 * for this special case.
254 */
255 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
256 info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) ||
257 (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
258 (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
259 info->dev->class >> 8 != PCI_CLASS_BRIDGE_OTHER))) {
260 pr_warn("Device scope type does not match for %s\n",
261 pci_name(info->dev));
262 return -EINVAL;
263 }
264
265 for_each_dev_scope(devices, devices_cnt, i, tmp)
266 if (tmp == NULL) {
267 devices[i].bus = info->dev->bus->number;
268 devices[i].devfn = info->dev->devfn;
269 rcu_assign_pointer(devices[i].dev,
270 get_device(dev));
271 return 1;
272 }
273 BUG_ON(i >= devices_cnt);
274 }
275
276 return 0;
277 }
278
279 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
280 struct dmar_dev_scope *devices, int count)
281 {
282 int index;
283 struct device *tmp;
284
285 if (info->seg != segment)
286 return 0;
287
288 for_each_active_dev_scope(devices, count, index, tmp)
289 if (tmp == &info->dev->dev) {
290 RCU_INIT_POINTER(devices[index].dev, NULL);
291 synchronize_rcu();
292 put_device(tmp);
293 return 1;
294 }
295
296 return 0;
297 }
298
299 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
300 {
301 int ret = 0;
302 struct dmar_drhd_unit *dmaru;
303 struct acpi_dmar_hardware_unit *drhd;
304
305 for_each_drhd_unit(dmaru) {
306 if (dmaru->include_all)
307 continue;
308
309 drhd = container_of(dmaru->hdr,
310 struct acpi_dmar_hardware_unit, header);
311 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
312 ((void *)drhd) + drhd->header.length,
313 dmaru->segment,
314 dmaru->devices, dmaru->devices_cnt);
315 if (ret)
316 break;
317 }
318 if (ret >= 0)
319 ret = dmar_iommu_notify_scope_dev(info);
320 if (ret < 0 && dmar_dev_scope_status == 0)
321 dmar_dev_scope_status = ret;
322
323 return ret;
324 }
325
326 static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
327 {
328 struct dmar_drhd_unit *dmaru;
329
330 for_each_drhd_unit(dmaru)
331 if (dmar_remove_dev_scope(info, dmaru->segment,
332 dmaru->devices, dmaru->devices_cnt))
333 break;
334 dmar_iommu_notify_scope_dev(info);
335 }
336
337 static int dmar_pci_bus_notifier(struct notifier_block *nb,
338 unsigned long action, void *data)
339 {
340 struct pci_dev *pdev = to_pci_dev(data);
341 struct dmar_pci_notify_info *info;
342
343 /* Only care about add/remove events for physical functions.
344 * For VFs we actually do the lookup based on the corresponding
345 * PF in device_to_iommu() anyway. */
346 if (pdev->is_virtfn)
347 return NOTIFY_DONE;
348 if (action != BUS_NOTIFY_ADD_DEVICE &&
349 action != BUS_NOTIFY_REMOVED_DEVICE)
350 return NOTIFY_DONE;
351
352 info = dmar_alloc_pci_notify_info(pdev, action);
353 if (!info)
354 return NOTIFY_DONE;
355
356 down_write(&dmar_global_lock);
357 if (action == BUS_NOTIFY_ADD_DEVICE)
358 dmar_pci_bus_add_dev(info);
359 else if (action == BUS_NOTIFY_REMOVED_DEVICE)
360 dmar_pci_bus_del_dev(info);
361 up_write(&dmar_global_lock);
362
363 dmar_free_pci_notify_info(info);
364
365 return NOTIFY_OK;
366 }
367
368 static struct notifier_block dmar_pci_bus_nb = {
369 .notifier_call = dmar_pci_bus_notifier,
370 .priority = INT_MIN,
371 };
372
373 static struct dmar_drhd_unit *
374 dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
375 {
376 struct dmar_drhd_unit *dmaru;
377
378 list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list)
379 if (dmaru->segment == drhd->segment &&
380 dmaru->reg_base_addr == drhd->address)
381 return dmaru;
382
383 return NULL;
384 }
385
386 /**
387 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
388 * structure which uniquely represent one DMA remapping hardware unit
389 * present in the platform
390 */
391 static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
392 {
393 struct acpi_dmar_hardware_unit *drhd;
394 struct dmar_drhd_unit *dmaru;
395 int ret;
396
397 drhd = (struct acpi_dmar_hardware_unit *)header;
398 dmaru = dmar_find_dmaru(drhd);
399 if (dmaru)
400 goto out;
401
402 dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
403 if (!dmaru)
404 return -ENOMEM;
405
406 /*
407 * If header is allocated from slab by ACPI _DSM method, we need to
408 * copy the content because the memory buffer will be freed on return.
409 */
410 dmaru->hdr = (void *)(dmaru + 1);
411 memcpy(dmaru->hdr, header, header->length);
412 dmaru->reg_base_addr = drhd->address;
413 dmaru->segment = drhd->segment;
414 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
415 dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
416 ((void *)drhd) + drhd->header.length,
417 &dmaru->devices_cnt);
418 if (dmaru->devices_cnt && dmaru->devices == NULL) {
419 kfree(dmaru);
420 return -ENOMEM;
421 }
422
423 ret = alloc_iommu(dmaru);
424 if (ret) {
425 dmar_free_dev_scope(&dmaru->devices,
426 &dmaru->devices_cnt);
427 kfree(dmaru);
428 return ret;
429 }
430 dmar_register_drhd_unit(dmaru);
431
432 out:
433 if (arg)
434 (*(int *)arg)++;
435
436 return 0;
437 }
438
439 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
440 {
441 if (dmaru->devices && dmaru->devices_cnt)
442 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
443 if (dmaru->iommu)
444 free_iommu(dmaru->iommu);
445 kfree(dmaru);
446 }
447
448 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
449 void *arg)
450 {
451 struct acpi_dmar_andd *andd = (void *)header;
452
453 /* Check for NUL termination within the designated length */
454 if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
455 WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
456 "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
457 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
458 dmi_get_system_info(DMI_BIOS_VENDOR),
459 dmi_get_system_info(DMI_BIOS_VERSION),
460 dmi_get_system_info(DMI_PRODUCT_VERSION));
461 return -EINVAL;
462 }
463 pr_info("ANDD device: %x name: %s\n", andd->device_number,
464 andd->device_name);
465
466 return 0;
467 }
468
469 #ifdef CONFIG_ACPI_NUMA
470 static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
471 {
472 struct acpi_dmar_rhsa *rhsa;
473 struct dmar_drhd_unit *drhd;
474
475 rhsa = (struct acpi_dmar_rhsa *)header;
476 for_each_drhd_unit(drhd) {
477 if (drhd->reg_base_addr == rhsa->base_address) {
478 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
479
480 if (!node_online(node))
481 node = NUMA_NO_NODE;
482 drhd->iommu->node = node;
483 return 0;
484 }
485 }
486 WARN_TAINT(
487 1, TAINT_FIRMWARE_WORKAROUND,
488 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
489 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
490 drhd->reg_base_addr,
491 dmi_get_system_info(DMI_BIOS_VENDOR),
492 dmi_get_system_info(DMI_BIOS_VERSION),
493 dmi_get_system_info(DMI_PRODUCT_VERSION));
494
495 return 0;
496 }
497 #else
498 #define dmar_parse_one_rhsa dmar_res_noop
499 #endif
500
501 static void
502 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
503 {
504 struct acpi_dmar_hardware_unit *drhd;
505 struct acpi_dmar_reserved_memory *rmrr;
506 struct acpi_dmar_atsr *atsr;
507 struct acpi_dmar_rhsa *rhsa;
508
509 switch (header->type) {
510 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
511 drhd = container_of(header, struct acpi_dmar_hardware_unit,
512 header);
513 pr_info("DRHD base: %#016Lx flags: %#x\n",
514 (unsigned long long)drhd->address, drhd->flags);
515 break;
516 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
517 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
518 header);
519 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
520 (unsigned long long)rmrr->base_address,
521 (unsigned long long)rmrr->end_address);
522 break;
523 case ACPI_DMAR_TYPE_ROOT_ATS:
524 atsr = container_of(header, struct acpi_dmar_atsr, header);
525 pr_info("ATSR flags: %#x\n", atsr->flags);
526 break;
527 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
528 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
529 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
530 (unsigned long long)rhsa->base_address,
531 rhsa->proximity_domain);
532 break;
533 case ACPI_DMAR_TYPE_NAMESPACE:
534 /* We don't print this here because we need to sanity-check
535 it first. So print it in dmar_parse_one_andd() instead. */
536 break;
537 }
538 }
539
540 /**
541 * dmar_table_detect - checks to see if the platform supports DMAR devices
542 */
543 static int __init dmar_table_detect(void)
544 {
545 acpi_status status = AE_OK;
546
547 /* if we could find DMAR table, then there are DMAR devices */
548 status = acpi_get_table(ACPI_SIG_DMAR, 0, &dmar_tbl);
549
550 if (ACPI_SUCCESS(status) && !dmar_tbl) {
551 pr_warn("Unable to map DMAR\n");
552 status = AE_NOT_FOUND;
553 }
554
555 return ACPI_SUCCESS(status) ? 0 : -ENOENT;
556 }
557
558 static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
559 size_t len, struct dmar_res_callback *cb)
560 {
561 struct acpi_dmar_header *iter, *next;
562 struct acpi_dmar_header *end = ((void *)start) + len;
563
564 for (iter = start; iter < end; iter = next) {
565 next = (void *)iter + iter->length;
566 if (iter->length == 0) {
567 /* Avoid looping forever on bad ACPI tables */
568 pr_debug(FW_BUG "Invalid 0-length structure\n");
569 break;
570 } else if (next > end) {
571 /* Avoid passing table end */
572 pr_warn(FW_BUG "Record passes table end\n");
573 return -EINVAL;
574 }
575
576 if (cb->print_entry)
577 dmar_table_print_dmar_entry(iter);
578
579 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
580 /* continue for forward compatibility */
581 pr_debug("Unknown DMAR structure type %d\n",
582 iter->type);
583 } else if (cb->cb[iter->type]) {
584 int ret;
585
586 ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
587 if (ret)
588 return ret;
589 } else if (!cb->ignore_unhandled) {
590 pr_warn("No handler for DMAR structure type %d\n",
591 iter->type);
592 return -EINVAL;
593 }
594 }
595
596 return 0;
597 }
598
599 static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
600 struct dmar_res_callback *cb)
601 {
602 return dmar_walk_remapping_entries((void *)(dmar + 1),
603 dmar->header.length - sizeof(*dmar), cb);
604 }
605
606 /**
607 * parse_dmar_table - parses the DMA reporting table
608 */
609 static int __init
610 parse_dmar_table(void)
611 {
612 struct acpi_table_dmar *dmar;
613 int drhd_count = 0;
614 int ret;
615 struct dmar_res_callback cb = {
616 .print_entry = true,
617 .ignore_unhandled = true,
618 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
619 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
620 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
621 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
622 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
623 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
624 };
625
626 /*
627 * Do it again, earlier dmar_tbl mapping could be mapped with
628 * fixed map.
629 */
630 dmar_table_detect();
631
632 /*
633 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
634 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
635 */
636 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
637
638 dmar = (struct acpi_table_dmar *)dmar_tbl;
639 if (!dmar)
640 return -ENODEV;
641
642 if (dmar->width < PAGE_SHIFT - 1) {
643 pr_warn("Invalid DMAR haw\n");
644 return -EINVAL;
645 }
646
647 pr_info("Host address width %d\n", dmar->width + 1);
648 ret = dmar_walk_dmar_table(dmar, &cb);
649 if (ret == 0 && drhd_count == 0)
650 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
651
652 return ret;
653 }
654
655 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
656 int cnt, struct pci_dev *dev)
657 {
658 int index;
659 struct device *tmp;
660
661 while (dev) {
662 for_each_active_dev_scope(devices, cnt, index, tmp)
663 if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
664 return 1;
665
666 /* Check our parent */
667 dev = dev->bus->self;
668 }
669
670 return 0;
671 }
672
673 struct dmar_drhd_unit *
674 dmar_find_matched_drhd_unit(struct pci_dev *dev)
675 {
676 struct dmar_drhd_unit *dmaru;
677 struct acpi_dmar_hardware_unit *drhd;
678
679 dev = pci_physfn(dev);
680
681 rcu_read_lock();
682 for_each_drhd_unit(dmaru) {
683 drhd = container_of(dmaru->hdr,
684 struct acpi_dmar_hardware_unit,
685 header);
686
687 if (dmaru->include_all &&
688 drhd->segment == pci_domain_nr(dev->bus))
689 goto out;
690
691 if (dmar_pci_device_match(dmaru->devices,
692 dmaru->devices_cnt, dev))
693 goto out;
694 }
695 dmaru = NULL;
696 out:
697 rcu_read_unlock();
698
699 return dmaru;
700 }
701
702 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
703 struct acpi_device *adev)
704 {
705 struct dmar_drhd_unit *dmaru;
706 struct acpi_dmar_hardware_unit *drhd;
707 struct acpi_dmar_device_scope *scope;
708 struct device *tmp;
709 int i;
710 struct acpi_dmar_pci_path *path;
711
712 for_each_drhd_unit(dmaru) {
713 drhd = container_of(dmaru->hdr,
714 struct acpi_dmar_hardware_unit,
715 header);
716
717 for (scope = (void *)(drhd + 1);
718 (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
719 scope = ((void *)scope) + scope->length) {
720 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
721 continue;
722 if (scope->enumeration_id != device_number)
723 continue;
724
725 path = (void *)(scope + 1);
726 pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
727 dev_name(&adev->dev), dmaru->reg_base_addr,
728 scope->bus, path->device, path->function);
729 for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
730 if (tmp == NULL) {
731 dmaru->devices[i].bus = scope->bus;
732 dmaru->devices[i].devfn = PCI_DEVFN(path->device,
733 path->function);
734 rcu_assign_pointer(dmaru->devices[i].dev,
735 get_device(&adev->dev));
736 return;
737 }
738 BUG_ON(i >= dmaru->devices_cnt);
739 }
740 }
741 pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
742 device_number, dev_name(&adev->dev));
743 }
744
745 static int __init dmar_acpi_dev_scope_init(void)
746 {
747 struct acpi_dmar_andd *andd;
748
749 if (dmar_tbl == NULL)
750 return -ENODEV;
751
752 for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
753 ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
754 andd = ((void *)andd) + andd->header.length) {
755 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
756 acpi_handle h;
757 struct acpi_device *adev;
758
759 if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
760 andd->device_name,
761 &h))) {
762 pr_err("Failed to find handle for ACPI object %s\n",
763 andd->device_name);
764 continue;
765 }
766 if (acpi_bus_get_device(h, &adev)) {
767 pr_err("Failed to get device for ACPI object %s\n",
768 andd->device_name);
769 continue;
770 }
771 dmar_acpi_insert_dev_scope(andd->device_number, adev);
772 }
773 }
774 return 0;
775 }
776
777 int __init dmar_dev_scope_init(void)
778 {
779 struct pci_dev *dev = NULL;
780 struct dmar_pci_notify_info *info;
781
782 if (dmar_dev_scope_status != 1)
783 return dmar_dev_scope_status;
784
785 if (list_empty(&dmar_drhd_units)) {
786 dmar_dev_scope_status = -ENODEV;
787 } else {
788 dmar_dev_scope_status = 0;
789
790 dmar_acpi_dev_scope_init();
791
792 for_each_pci_dev(dev) {
793 if (dev->is_virtfn)
794 continue;
795
796 info = dmar_alloc_pci_notify_info(dev,
797 BUS_NOTIFY_ADD_DEVICE);
798 if (!info) {
799 return dmar_dev_scope_status;
800 } else {
801 dmar_pci_bus_add_dev(info);
802 dmar_free_pci_notify_info(info);
803 }
804 }
805 }
806
807 return dmar_dev_scope_status;
808 }
809
810 void __init dmar_register_bus_notifier(void)
811 {
812 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
813 }
814
815
816 int __init dmar_table_init(void)
817 {
818 static int dmar_table_initialized;
819 int ret;
820
821 if (dmar_table_initialized == 0) {
822 ret = parse_dmar_table();
823 if (ret < 0) {
824 if (ret != -ENODEV)
825 pr_info("Parse DMAR table failure.\n");
826 } else if (list_empty(&dmar_drhd_units)) {
827 pr_info("No DMAR devices found\n");
828 ret = -ENODEV;
829 }
830
831 if (ret < 0)
832 dmar_table_initialized = ret;
833 else
834 dmar_table_initialized = 1;
835 }
836
837 return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
838 }
839
840 static void warn_invalid_dmar(u64 addr, const char *message)
841 {
842 WARN_TAINT_ONCE(
843 1, TAINT_FIRMWARE_WORKAROUND,
844 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
845 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
846 addr, message,
847 dmi_get_system_info(DMI_BIOS_VENDOR),
848 dmi_get_system_info(DMI_BIOS_VERSION),
849 dmi_get_system_info(DMI_PRODUCT_VERSION));
850 }
851
852 static int __ref
853 dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
854 {
855 struct acpi_dmar_hardware_unit *drhd;
856 void __iomem *addr;
857 u64 cap, ecap;
858
859 drhd = (void *)entry;
860 if (!drhd->address) {
861 warn_invalid_dmar(0, "");
862 return -EINVAL;
863 }
864
865 if (arg)
866 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
867 else
868 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
869 if (!addr) {
870 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
871 return -EINVAL;
872 }
873
874 cap = dmar_readq(addr + DMAR_CAP_REG);
875 ecap = dmar_readq(addr + DMAR_ECAP_REG);
876
877 if (arg)
878 iounmap(addr);
879 else
880 early_iounmap(addr, VTD_PAGE_SIZE);
881
882 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
883 warn_invalid_dmar(drhd->address, " returns all ones");
884 return -EINVAL;
885 }
886
887 return 0;
888 }
889
890 int __init detect_intel_iommu(void)
891 {
892 int ret;
893 struct dmar_res_callback validate_drhd_cb = {
894 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
895 .ignore_unhandled = true,
896 };
897
898 down_write(&dmar_global_lock);
899 ret = dmar_table_detect();
900 if (!ret)
901 ret = dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
902 &validate_drhd_cb);
903 if (!ret && !no_iommu && !iommu_detected && !dmar_disabled) {
904 iommu_detected = 1;
905 /* Make sure ACS will be enabled */
906 pci_request_acs();
907 }
908
909 #ifdef CONFIG_X86
910 if (!ret)
911 x86_init.iommu.iommu_init = intel_iommu_init;
912 #endif
913
914 if (dmar_tbl) {
915 acpi_put_table(dmar_tbl);
916 dmar_tbl = NULL;
917 }
918 up_write(&dmar_global_lock);
919
920 return ret ? ret : 1;
921 }
922
923 static void unmap_iommu(struct intel_iommu *iommu)
924 {
925 iounmap(iommu->reg);
926 release_mem_region(iommu->reg_phys, iommu->reg_size);
927 }
928
929 /**
930 * map_iommu: map the iommu's registers
931 * @iommu: the iommu to map
932 * @phys_addr: the physical address of the base resgister
933 *
934 * Memory map the iommu's registers. Start w/ a single page, and
935 * possibly expand if that turns out to be insufficent.
936 */
937 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
938 {
939 int map_size, err=0;
940
941 iommu->reg_phys = phys_addr;
942 iommu->reg_size = VTD_PAGE_SIZE;
943
944 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
945 pr_err("Can't reserve memory\n");
946 err = -EBUSY;
947 goto out;
948 }
949
950 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
951 if (!iommu->reg) {
952 pr_err("Can't map the region\n");
953 err = -ENOMEM;
954 goto release;
955 }
956
957 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
958 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
959
960 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
961 err = -EINVAL;
962 warn_invalid_dmar(phys_addr, " returns all ones");
963 goto unmap;
964 }
965
966 /* the registers might be more than one page */
967 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
968 cap_max_fault_reg_offset(iommu->cap));
969 map_size = VTD_PAGE_ALIGN(map_size);
970 if (map_size > iommu->reg_size) {
971 iounmap(iommu->reg);
972 release_mem_region(iommu->reg_phys, iommu->reg_size);
973 iommu->reg_size = map_size;
974 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
975 iommu->name)) {
976 pr_err("Can't reserve memory\n");
977 err = -EBUSY;
978 goto out;
979 }
980 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
981 if (!iommu->reg) {
982 pr_err("Can't map the region\n");
983 err = -ENOMEM;
984 goto release;
985 }
986 }
987 err = 0;
988 goto out;
989
990 unmap:
991 iounmap(iommu->reg);
992 release:
993 release_mem_region(iommu->reg_phys, iommu->reg_size);
994 out:
995 return err;
996 }
997
998 static int dmar_alloc_seq_id(struct intel_iommu *iommu)
999 {
1000 iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
1001 DMAR_UNITS_SUPPORTED);
1002 if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
1003 iommu->seq_id = -1;
1004 } else {
1005 set_bit(iommu->seq_id, dmar_seq_ids);
1006 sprintf(iommu->name, "dmar%d", iommu->seq_id);
1007 }
1008
1009 return iommu->seq_id;
1010 }
1011
1012 static void dmar_free_seq_id(struct intel_iommu *iommu)
1013 {
1014 if (iommu->seq_id >= 0) {
1015 clear_bit(iommu->seq_id, dmar_seq_ids);
1016 iommu->seq_id = -1;
1017 }
1018 }
1019
1020 static int alloc_iommu(struct dmar_drhd_unit *drhd)
1021 {
1022 struct intel_iommu *iommu;
1023 u32 ver, sts;
1024 int agaw = 0;
1025 int msagaw = 0;
1026 int err;
1027
1028 if (!drhd->reg_base_addr) {
1029 warn_invalid_dmar(0, "");
1030 return -EINVAL;
1031 }
1032
1033 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1034 if (!iommu)
1035 return -ENOMEM;
1036
1037 if (dmar_alloc_seq_id(iommu) < 0) {
1038 pr_err("Failed to allocate seq_id\n");
1039 err = -ENOSPC;
1040 goto error;
1041 }
1042
1043 err = map_iommu(iommu, drhd->reg_base_addr);
1044 if (err) {
1045 pr_err("Failed to map %s\n", iommu->name);
1046 goto error_free_seq_id;
1047 }
1048
1049 err = -EINVAL;
1050 agaw = iommu_calculate_agaw(iommu);
1051 if (agaw < 0) {
1052 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1053 iommu->seq_id);
1054 goto err_unmap;
1055 }
1056 msagaw = iommu_calculate_max_sagaw(iommu);
1057 if (msagaw < 0) {
1058 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1059 iommu->seq_id);
1060 goto err_unmap;
1061 }
1062 iommu->agaw = agaw;
1063 iommu->msagaw = msagaw;
1064 iommu->segment = drhd->segment;
1065
1066 iommu->node = NUMA_NO_NODE;
1067
1068 ver = readl(iommu->reg + DMAR_VER_REG);
1069 pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1070 iommu->name,
1071 (unsigned long long)drhd->reg_base_addr,
1072 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1073 (unsigned long long)iommu->cap,
1074 (unsigned long long)iommu->ecap);
1075
1076 /* Reflect status in gcmd */
1077 sts = readl(iommu->reg + DMAR_GSTS_REG);
1078 if (sts & DMA_GSTS_IRES)
1079 iommu->gcmd |= DMA_GCMD_IRE;
1080 if (sts & DMA_GSTS_TES)
1081 iommu->gcmd |= DMA_GCMD_TE;
1082 if (sts & DMA_GSTS_QIES)
1083 iommu->gcmd |= DMA_GCMD_QIE;
1084
1085 raw_spin_lock_init(&iommu->register_lock);
1086
1087 if (intel_iommu_enabled) {
1088 err = iommu_device_sysfs_add(&iommu->iommu, NULL,
1089 intel_iommu_groups,
1090 "%s", iommu->name);
1091 if (err)
1092 goto err_unmap;
1093
1094 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
1095
1096 err = iommu_device_register(&iommu->iommu);
1097 if (err)
1098 goto err_unmap;
1099 }
1100
1101 drhd->iommu = iommu;
1102
1103 return 0;
1104
1105 err_unmap:
1106 unmap_iommu(iommu);
1107 error_free_seq_id:
1108 dmar_free_seq_id(iommu);
1109 error:
1110 kfree(iommu);
1111 return err;
1112 }
1113
1114 static void free_iommu(struct intel_iommu *iommu)
1115 {
1116 if (intel_iommu_enabled) {
1117 iommu_device_unregister(&iommu->iommu);
1118 iommu_device_sysfs_remove(&iommu->iommu);
1119 }
1120
1121 if (iommu->irq) {
1122 if (iommu->pr_irq) {
1123 free_irq(iommu->pr_irq, iommu);
1124 dmar_free_hwirq(iommu->pr_irq);
1125 iommu->pr_irq = 0;
1126 }
1127 free_irq(iommu->irq, iommu);
1128 dmar_free_hwirq(iommu->irq);
1129 iommu->irq = 0;
1130 }
1131
1132 if (iommu->qi) {
1133 free_page((unsigned long)iommu->qi->desc);
1134 kfree(iommu->qi->desc_status);
1135 kfree(iommu->qi);
1136 }
1137
1138 if (iommu->reg)
1139 unmap_iommu(iommu);
1140
1141 dmar_free_seq_id(iommu);
1142 kfree(iommu);
1143 }
1144
1145 /*
1146 * Reclaim all the submitted descriptors which have completed its work.
1147 */
1148 static inline void reclaim_free_desc(struct q_inval *qi)
1149 {
1150 while (qi->desc_status[qi->free_tail] == QI_DONE ||
1151 qi->desc_status[qi->free_tail] == QI_ABORT) {
1152 qi->desc_status[qi->free_tail] = QI_FREE;
1153 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1154 qi->free_cnt++;
1155 }
1156 }
1157
1158 static int qi_check_fault(struct intel_iommu *iommu, int index)
1159 {
1160 u32 fault;
1161 int head, tail;
1162 struct q_inval *qi = iommu->qi;
1163 int wait_index = (index + 1) % QI_LENGTH;
1164 int shift = qi_shift(iommu);
1165
1166 if (qi->desc_status[wait_index] == QI_ABORT)
1167 return -EAGAIN;
1168
1169 fault = readl(iommu->reg + DMAR_FSTS_REG);
1170
1171 /*
1172 * If IQE happens, the head points to the descriptor associated
1173 * with the error. No new descriptors are fetched until the IQE
1174 * is cleared.
1175 */
1176 if (fault & DMA_FSTS_IQE) {
1177 head = readl(iommu->reg + DMAR_IQH_REG);
1178 if ((head >> shift) == index) {
1179 struct qi_desc *desc = qi->desc + head;
1180
1181 /*
1182 * desc->qw2 and desc->qw3 are either reserved or
1183 * used by software as private data. We won't print
1184 * out these two qw's for security consideration.
1185 */
1186 pr_err("VT-d detected invalid descriptor: qw0 = %llx, qw1 = %llx\n",
1187 (unsigned long long)desc->qw0,
1188 (unsigned long long)desc->qw1);
1189 memcpy(desc, qi->desc + (wait_index << shift),
1190 1 << shift);
1191 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1192 return -EINVAL;
1193 }
1194 }
1195
1196 /*
1197 * If ITE happens, all pending wait_desc commands are aborted.
1198 * No new descriptors are fetched until the ITE is cleared.
1199 */
1200 if (fault & DMA_FSTS_ITE) {
1201 head = readl(iommu->reg + DMAR_IQH_REG);
1202 head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1203 head |= 1;
1204 tail = readl(iommu->reg + DMAR_IQT_REG);
1205 tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1206
1207 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1208
1209 do {
1210 if (qi->desc_status[head] == QI_IN_USE)
1211 qi->desc_status[head] = QI_ABORT;
1212 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1213 } while (head != tail);
1214
1215 if (qi->desc_status[wait_index] == QI_ABORT)
1216 return -EAGAIN;
1217 }
1218
1219 if (fault & DMA_FSTS_ICE)
1220 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1221
1222 return 0;
1223 }
1224
1225 /*
1226 * Submit the queued invalidation descriptor to the remapping
1227 * hardware unit and wait for its completion.
1228 */
1229 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1230 {
1231 int rc;
1232 struct q_inval *qi = iommu->qi;
1233 int offset, shift, length;
1234 struct qi_desc wait_desc;
1235 int wait_index, index;
1236 unsigned long flags;
1237
1238 if (!qi)
1239 return 0;
1240
1241 restart:
1242 rc = 0;
1243
1244 raw_spin_lock_irqsave(&qi->q_lock, flags);
1245 while (qi->free_cnt < 3) {
1246 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1247 cpu_relax();
1248 raw_spin_lock_irqsave(&qi->q_lock, flags);
1249 }
1250
1251 index = qi->free_head;
1252 wait_index = (index + 1) % QI_LENGTH;
1253 shift = qi_shift(iommu);
1254 length = 1 << shift;
1255
1256 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1257
1258 offset = index << shift;
1259 memcpy(qi->desc + offset, desc, length);
1260 wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
1261 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1262 wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
1263 wait_desc.qw2 = 0;
1264 wait_desc.qw3 = 0;
1265
1266 offset = wait_index << shift;
1267 memcpy(qi->desc + offset, &wait_desc, length);
1268
1269 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1270 qi->free_cnt -= 2;
1271
1272 /*
1273 * update the HW tail register indicating the presence of
1274 * new descriptors.
1275 */
1276 writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);
1277
1278 while (qi->desc_status[wait_index] != QI_DONE) {
1279 /*
1280 * We will leave the interrupts disabled, to prevent interrupt
1281 * context to queue another cmd while a cmd is already submitted
1282 * and waiting for completion on this cpu. This is to avoid
1283 * a deadlock where the interrupt context can wait indefinitely
1284 * for free slots in the queue.
1285 */
1286 rc = qi_check_fault(iommu, index);
1287 if (rc)
1288 break;
1289
1290 raw_spin_unlock(&qi->q_lock);
1291 cpu_relax();
1292 raw_spin_lock(&qi->q_lock);
1293 }
1294
1295 qi->desc_status[index] = QI_DONE;
1296
1297 reclaim_free_desc(qi);
1298 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1299
1300 if (rc == -EAGAIN)
1301 goto restart;
1302
1303 return rc;
1304 }
1305
1306 /*
1307 * Flush the global interrupt entry cache.
1308 */
1309 void qi_global_iec(struct intel_iommu *iommu)
1310 {
1311 struct qi_desc desc;
1312
1313 desc.qw0 = QI_IEC_TYPE;
1314 desc.qw1 = 0;
1315 desc.qw2 = 0;
1316 desc.qw3 = 0;
1317
1318 /* should never fail */
1319 qi_submit_sync(&desc, iommu);
1320 }
1321
1322 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1323 u64 type)
1324 {
1325 struct qi_desc desc;
1326
1327 desc.qw0 = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1328 | QI_CC_GRAN(type) | QI_CC_TYPE;
1329 desc.qw1 = 0;
1330 desc.qw2 = 0;
1331 desc.qw3 = 0;
1332
1333 qi_submit_sync(&desc, iommu);
1334 }
1335
1336 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1337 unsigned int size_order, u64 type)
1338 {
1339 u8 dw = 0, dr = 0;
1340
1341 struct qi_desc desc;
1342 int ih = 0;
1343
1344 if (cap_write_drain(iommu->cap))
1345 dw = 1;
1346
1347 if (cap_read_drain(iommu->cap))
1348 dr = 1;
1349
1350 desc.qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1351 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1352 desc.qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1353 | QI_IOTLB_AM(size_order);
1354 desc.qw2 = 0;
1355 desc.qw3 = 0;
1356
1357 qi_submit_sync(&desc, iommu);
1358 }
1359
1360 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1361 u16 qdep, u64 addr, unsigned mask)
1362 {
1363 struct qi_desc desc;
1364
1365 if (mask) {
1366 WARN_ON_ONCE(addr & ((1ULL << (VTD_PAGE_SHIFT + mask)) - 1));
1367 addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1368 desc.qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1369 } else
1370 desc.qw1 = QI_DEV_IOTLB_ADDR(addr);
1371
1372 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1373 qdep = 0;
1374
1375 desc.qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1376 QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
1377 desc.qw2 = 0;
1378 desc.qw3 = 0;
1379
1380 qi_submit_sync(&desc, iommu);
1381 }
1382
1383 /*
1384 * Disable Queued Invalidation interface.
1385 */
1386 void dmar_disable_qi(struct intel_iommu *iommu)
1387 {
1388 unsigned long flags;
1389 u32 sts;
1390 cycles_t start_time = get_cycles();
1391
1392 if (!ecap_qis(iommu->ecap))
1393 return;
1394
1395 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1396
1397 sts = readl(iommu->reg + DMAR_GSTS_REG);
1398 if (!(sts & DMA_GSTS_QIES))
1399 goto end;
1400
1401 /*
1402 * Give a chance to HW to complete the pending invalidation requests.
1403 */
1404 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1405 readl(iommu->reg + DMAR_IQH_REG)) &&
1406 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1407 cpu_relax();
1408
1409 iommu->gcmd &= ~DMA_GCMD_QIE;
1410 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1411
1412 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1413 !(sts & DMA_GSTS_QIES), sts);
1414 end:
1415 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1416 }
1417
1418 /*
1419 * Enable queued invalidation.
1420 */
1421 static void __dmar_enable_qi(struct intel_iommu *iommu)
1422 {
1423 u32 sts;
1424 unsigned long flags;
1425 struct q_inval *qi = iommu->qi;
1426 u64 val = virt_to_phys(qi->desc);
1427
1428 qi->free_head = qi->free_tail = 0;
1429 qi->free_cnt = QI_LENGTH;
1430
1431 /*
1432 * Set DW=1 and QS=1 in IQA_REG when Scalable Mode capability
1433 * is present.
1434 */
1435 if (ecap_smts(iommu->ecap))
1436 val |= (1 << 11) | 1;
1437
1438 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1439
1440 /* write zero to the tail reg */
1441 writel(0, iommu->reg + DMAR_IQT_REG);
1442
1443 dmar_writeq(iommu->reg + DMAR_IQA_REG, val);
1444
1445 iommu->gcmd |= DMA_GCMD_QIE;
1446 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1447
1448 /* Make sure hardware complete it */
1449 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1450
1451 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1452 }
1453
1454 /*
1455 * Enable Queued Invalidation interface. This is a must to support
1456 * interrupt-remapping. Also used by DMA-remapping, which replaces
1457 * register based IOTLB invalidation.
1458 */
1459 int dmar_enable_qi(struct intel_iommu *iommu)
1460 {
1461 struct q_inval *qi;
1462 struct page *desc_page;
1463
1464 if (!ecap_qis(iommu->ecap))
1465 return -ENOENT;
1466
1467 /*
1468 * queued invalidation is already setup and enabled.
1469 */
1470 if (iommu->qi)
1471 return 0;
1472
1473 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1474 if (!iommu->qi)
1475 return -ENOMEM;
1476
1477 qi = iommu->qi;
1478
1479 /*
1480 * Need two pages to accommodate 256 descriptors of 256 bits each
1481 * if the remapping hardware supports scalable mode translation.
1482 */
1483 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO,
1484 !!ecap_smts(iommu->ecap));
1485 if (!desc_page) {
1486 kfree(qi);
1487 iommu->qi = NULL;
1488 return -ENOMEM;
1489 }
1490
1491 qi->desc = page_address(desc_page);
1492
1493 qi->desc_status = kcalloc(QI_LENGTH, sizeof(int), GFP_ATOMIC);
1494 if (!qi->desc_status) {
1495 free_page((unsigned long) qi->desc);
1496 kfree(qi);
1497 iommu->qi = NULL;
1498 return -ENOMEM;
1499 }
1500
1501 raw_spin_lock_init(&qi->q_lock);
1502
1503 __dmar_enable_qi(iommu);
1504
1505 return 0;
1506 }
1507
1508 /* iommu interrupt handling. Most stuff are MSI-like. */
1509
1510 enum faulttype {
1511 DMA_REMAP,
1512 INTR_REMAP,
1513 UNKNOWN,
1514 };
1515
1516 static const char *dma_remap_fault_reasons[] =
1517 {
1518 "Software",
1519 "Present bit in root entry is clear",
1520 "Present bit in context entry is clear",
1521 "Invalid context entry",
1522 "Access beyond MGAW",
1523 "PTE Write access is not set",
1524 "PTE Read access is not set",
1525 "Next page table ptr is invalid",
1526 "Root table address invalid",
1527 "Context table ptr is invalid",
1528 "non-zero reserved fields in RTP",
1529 "non-zero reserved fields in CTP",
1530 "non-zero reserved fields in PTE",
1531 "PCE for translation request specifies blocking",
1532 };
1533
1534 static const char *irq_remap_fault_reasons[] =
1535 {
1536 "Detected reserved fields in the decoded interrupt-remapped request",
1537 "Interrupt index exceeded the interrupt-remapping table size",
1538 "Present field in the IRTE entry is clear",
1539 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1540 "Detected reserved fields in the IRTE entry",
1541 "Blocked a compatibility format interrupt request",
1542 "Blocked an interrupt request due to source-id verification failure",
1543 };
1544
1545 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1546 {
1547 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1548 ARRAY_SIZE(irq_remap_fault_reasons))) {
1549 *fault_type = INTR_REMAP;
1550 return irq_remap_fault_reasons[fault_reason - 0x20];
1551 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1552 *fault_type = DMA_REMAP;
1553 return dma_remap_fault_reasons[fault_reason];
1554 } else {
1555 *fault_type = UNKNOWN;
1556 return "Unknown";
1557 }
1558 }
1559
1560
1561 static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1562 {
1563 if (iommu->irq == irq)
1564 return DMAR_FECTL_REG;
1565 else if (iommu->pr_irq == irq)
1566 return DMAR_PECTL_REG;
1567 else
1568 BUG();
1569 }
1570
1571 void dmar_msi_unmask(struct irq_data *data)
1572 {
1573 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1574 int reg = dmar_msi_reg(iommu, data->irq);
1575 unsigned long flag;
1576
1577 /* unmask it */
1578 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1579 writel(0, iommu->reg + reg);
1580 /* Read a reg to force flush the post write */
1581 readl(iommu->reg + reg);
1582 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1583 }
1584
1585 void dmar_msi_mask(struct irq_data *data)
1586 {
1587 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1588 int reg = dmar_msi_reg(iommu, data->irq);
1589 unsigned long flag;
1590
1591 /* mask it */
1592 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1593 writel(DMA_FECTL_IM, iommu->reg + reg);
1594 /* Read a reg to force flush the post write */
1595 readl(iommu->reg + reg);
1596 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1597 }
1598
1599 void dmar_msi_write(int irq, struct msi_msg *msg)
1600 {
1601 struct intel_iommu *iommu = irq_get_handler_data(irq);
1602 int reg = dmar_msi_reg(iommu, irq);
1603 unsigned long flag;
1604
1605 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1606 writel(msg->data, iommu->reg + reg + 4);
1607 writel(msg->address_lo, iommu->reg + reg + 8);
1608 writel(msg->address_hi, iommu->reg + reg + 12);
1609 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1610 }
1611
1612 void dmar_msi_read(int irq, struct msi_msg *msg)
1613 {
1614 struct intel_iommu *iommu = irq_get_handler_data(irq);
1615 int reg = dmar_msi_reg(iommu, irq);
1616 unsigned long flag;
1617
1618 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1619 msg->data = readl(iommu->reg + reg + 4);
1620 msg->address_lo = readl(iommu->reg + reg + 8);
1621 msg->address_hi = readl(iommu->reg + reg + 12);
1622 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1623 }
1624
1625 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1626 u8 fault_reason, u16 source_id, unsigned long long addr)
1627 {
1628 const char *reason;
1629 int fault_type;
1630
1631 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1632
1633 if (fault_type == INTR_REMAP)
1634 pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index %llx [fault reason %02d] %s\n",
1635 source_id >> 8, PCI_SLOT(source_id & 0xFF),
1636 PCI_FUNC(source_id & 0xFF), addr >> 48,
1637 fault_reason, reason);
1638 else
1639 pr_err("[%s] Request device [%02x:%02x.%d] fault addr %llx [fault reason %02d] %s\n",
1640 type ? "DMA Read" : "DMA Write",
1641 source_id >> 8, PCI_SLOT(source_id & 0xFF),
1642 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1643 return 0;
1644 }
1645
1646 #define PRIMARY_FAULT_REG_LEN (16)
1647 irqreturn_t dmar_fault(int irq, void *dev_id)
1648 {
1649 struct intel_iommu *iommu = dev_id;
1650 int reg, fault_index;
1651 u32 fault_status;
1652 unsigned long flag;
1653 static DEFINE_RATELIMIT_STATE(rs,
1654 DEFAULT_RATELIMIT_INTERVAL,
1655 DEFAULT_RATELIMIT_BURST);
1656
1657 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1658 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1659 if (fault_status && __ratelimit(&rs))
1660 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1661
1662 /* TBD: ignore advanced fault log currently */
1663 if (!(fault_status & DMA_FSTS_PPF))
1664 goto unlock_exit;
1665
1666 fault_index = dma_fsts_fault_record_index(fault_status);
1667 reg = cap_fault_reg_offset(iommu->cap);
1668 while (1) {
1669 /* Disable printing, simply clear the fault when ratelimited */
1670 bool ratelimited = !__ratelimit(&rs);
1671 u8 fault_reason;
1672 u16 source_id;
1673 u64 guest_addr;
1674 int type;
1675 u32 data;
1676
1677 /* highest 32 bits */
1678 data = readl(iommu->reg + reg +
1679 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1680 if (!(data & DMA_FRCD_F))
1681 break;
1682
1683 if (!ratelimited) {
1684 fault_reason = dma_frcd_fault_reason(data);
1685 type = dma_frcd_type(data);
1686
1687 data = readl(iommu->reg + reg +
1688 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1689 source_id = dma_frcd_source_id(data);
1690
1691 guest_addr = dmar_readq(iommu->reg + reg +
1692 fault_index * PRIMARY_FAULT_REG_LEN);
1693 guest_addr = dma_frcd_page_addr(guest_addr);
1694 }
1695
1696 /* clear the fault */
1697 writel(DMA_FRCD_F, iommu->reg + reg +
1698 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1699
1700 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1701
1702 if (!ratelimited)
1703 dmar_fault_do_one(iommu, type, fault_reason,
1704 source_id, guest_addr);
1705
1706 fault_index++;
1707 if (fault_index >= cap_num_fault_regs(iommu->cap))
1708 fault_index = 0;
1709 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1710 }
1711
1712 writel(DMA_FSTS_PFO | DMA_FSTS_PPF | DMA_FSTS_PRO,
1713 iommu->reg + DMAR_FSTS_REG);
1714
1715 unlock_exit:
1716 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1717 return IRQ_HANDLED;
1718 }
1719
1720 int dmar_set_interrupt(struct intel_iommu *iommu)
1721 {
1722 int irq, ret;
1723
1724 /*
1725 * Check if the fault interrupt is already initialized.
1726 */
1727 if (iommu->irq)
1728 return 0;
1729
1730 irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
1731 if (irq > 0) {
1732 iommu->irq = irq;
1733 } else {
1734 pr_err("No free IRQ vectors\n");
1735 return -EINVAL;
1736 }
1737
1738 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1739 if (ret)
1740 pr_err("Can't request irq\n");
1741 return ret;
1742 }
1743
1744 int __init enable_drhd_fault_handling(void)
1745 {
1746 struct dmar_drhd_unit *drhd;
1747 struct intel_iommu *iommu;
1748
1749 /*
1750 * Enable fault control interrupt.
1751 */
1752 for_each_iommu(iommu, drhd) {
1753 u32 fault_status;
1754 int ret = dmar_set_interrupt(iommu);
1755
1756 if (ret) {
1757 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1758 (unsigned long long)drhd->reg_base_addr, ret);
1759 return -1;
1760 }
1761
1762 /*
1763 * Clear any previous faults.
1764 */
1765 dmar_fault(iommu->irq, iommu);
1766 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1767 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1768 }
1769
1770 return 0;
1771 }
1772
1773 /*
1774 * Re-enable Queued Invalidation interface.
1775 */
1776 int dmar_reenable_qi(struct intel_iommu *iommu)
1777 {
1778 if (!ecap_qis(iommu->ecap))
1779 return -ENOENT;
1780
1781 if (!iommu->qi)
1782 return -ENOENT;
1783
1784 /*
1785 * First disable queued invalidation.
1786 */
1787 dmar_disable_qi(iommu);
1788 /*
1789 * Then enable queued invalidation again. Since there is no pending
1790 * invalidation requests now, it's safe to re-enable queued
1791 * invalidation.
1792 */
1793 __dmar_enable_qi(iommu);
1794
1795 return 0;
1796 }
1797
1798 /*
1799 * Check interrupt remapping support in DMAR table description.
1800 */
1801 int __init dmar_ir_support(void)
1802 {
1803 struct acpi_table_dmar *dmar;
1804 dmar = (struct acpi_table_dmar *)dmar_tbl;
1805 if (!dmar)
1806 return 0;
1807 return dmar->flags & 0x1;
1808 }
1809
1810 /* Check whether DMAR units are in use */
1811 static inline bool dmar_in_use(void)
1812 {
1813 return irq_remapping_enabled || intel_iommu_enabled;
1814 }
1815
1816 static int __init dmar_free_unused_resources(void)
1817 {
1818 struct dmar_drhd_unit *dmaru, *dmaru_n;
1819
1820 if (dmar_in_use())
1821 return 0;
1822
1823 if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1824 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1825
1826 down_write(&dmar_global_lock);
1827 list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1828 list_del(&dmaru->list);
1829 dmar_free_drhd(dmaru);
1830 }
1831 up_write(&dmar_global_lock);
1832
1833 return 0;
1834 }
1835
1836 late_initcall(dmar_free_unused_resources);
1837 IOMMU_INIT_POST(detect_intel_iommu);
1838
1839 /*
1840 * DMAR Hotplug Support
1841 * For more details, please refer to Intel(R) Virtualization Technology
1842 * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
1843 * "Remapping Hardware Unit Hot Plug".
1844 */
1845 static guid_t dmar_hp_guid =
1846 GUID_INIT(0xD8C1A3A6, 0xBE9B, 0x4C9B,
1847 0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF);
1848
1849 /*
1850 * Currently there's only one revision and BIOS will not check the revision id,
1851 * so use 0 for safety.
1852 */
1853 #define DMAR_DSM_REV_ID 0
1854 #define DMAR_DSM_FUNC_DRHD 1
1855 #define DMAR_DSM_FUNC_ATSR 2
1856 #define DMAR_DSM_FUNC_RHSA 3
1857
1858 static inline bool dmar_detect_dsm(acpi_handle handle, int func)
1859 {
1860 return acpi_check_dsm(handle, &dmar_hp_guid, DMAR_DSM_REV_ID, 1 << func);
1861 }
1862
1863 static int dmar_walk_dsm_resource(acpi_handle handle, int func,
1864 dmar_res_handler_t handler, void *arg)
1865 {
1866 int ret = -ENODEV;
1867 union acpi_object *obj;
1868 struct acpi_dmar_header *start;
1869 struct dmar_res_callback callback;
1870 static int res_type[] = {
1871 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
1872 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
1873 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
1874 };
1875
1876 if (!dmar_detect_dsm(handle, func))
1877 return 0;
1878
1879 obj = acpi_evaluate_dsm_typed(handle, &dmar_hp_guid, DMAR_DSM_REV_ID,
1880 func, NULL, ACPI_TYPE_BUFFER);
1881 if (!obj)
1882 return -ENODEV;
1883
1884 memset(&callback, 0, sizeof(callback));
1885 callback.cb[res_type[func]] = handler;
1886 callback.arg[res_type[func]] = arg;
1887 start = (struct acpi_dmar_header *)obj->buffer.pointer;
1888 ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
1889
1890 ACPI_FREE(obj);
1891
1892 return ret;
1893 }
1894
1895 static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
1896 {
1897 int ret;
1898 struct dmar_drhd_unit *dmaru;
1899
1900 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1901 if (!dmaru)
1902 return -ENODEV;
1903
1904 ret = dmar_ir_hotplug(dmaru, true);
1905 if (ret == 0)
1906 ret = dmar_iommu_hotplug(dmaru, true);
1907
1908 return ret;
1909 }
1910
1911 static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
1912 {
1913 int i, ret;
1914 struct device *dev;
1915 struct dmar_drhd_unit *dmaru;
1916
1917 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1918 if (!dmaru)
1919 return 0;
1920
1921 /*
1922 * All PCI devices managed by this unit should have been destroyed.
1923 */
1924 if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) {
1925 for_each_active_dev_scope(dmaru->devices,
1926 dmaru->devices_cnt, i, dev)
1927 return -EBUSY;
1928 }
1929
1930 ret = dmar_ir_hotplug(dmaru, false);
1931 if (ret == 0)
1932 ret = dmar_iommu_hotplug(dmaru, false);
1933
1934 return ret;
1935 }
1936
1937 static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
1938 {
1939 struct dmar_drhd_unit *dmaru;
1940
1941 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1942 if (dmaru) {
1943 list_del_rcu(&dmaru->list);
1944 synchronize_rcu();
1945 dmar_free_drhd(dmaru);
1946 }
1947
1948 return 0;
1949 }
1950
1951 static int dmar_hotplug_insert(acpi_handle handle)
1952 {
1953 int ret;
1954 int drhd_count = 0;
1955
1956 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1957 &dmar_validate_one_drhd, (void *)1);
1958 if (ret)
1959 goto out;
1960
1961 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1962 &dmar_parse_one_drhd, (void *)&drhd_count);
1963 if (ret == 0 && drhd_count == 0) {
1964 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
1965 goto out;
1966 } else if (ret) {
1967 goto release_drhd;
1968 }
1969
1970 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
1971 &dmar_parse_one_rhsa, NULL);
1972 if (ret)
1973 goto release_drhd;
1974
1975 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1976 &dmar_parse_one_atsr, NULL);
1977 if (ret)
1978 goto release_atsr;
1979
1980 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1981 &dmar_hp_add_drhd, NULL);
1982 if (!ret)
1983 return 0;
1984
1985 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1986 &dmar_hp_remove_drhd, NULL);
1987 release_atsr:
1988 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1989 &dmar_release_one_atsr, NULL);
1990 release_drhd:
1991 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1992 &dmar_hp_release_drhd, NULL);
1993 out:
1994 return ret;
1995 }
1996
1997 static int dmar_hotplug_remove(acpi_handle handle)
1998 {
1999 int ret;
2000
2001 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2002 &dmar_check_one_atsr, NULL);
2003 if (ret)
2004 return ret;
2005
2006 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2007 &dmar_hp_remove_drhd, NULL);
2008 if (ret == 0) {
2009 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2010 &dmar_release_one_atsr, NULL));
2011 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2012 &dmar_hp_release_drhd, NULL));
2013 } else {
2014 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2015 &dmar_hp_add_drhd, NULL);
2016 }
2017
2018 return ret;
2019 }
2020
2021 static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
2022 void *context, void **retval)
2023 {
2024 acpi_handle *phdl = retval;
2025
2026 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2027 *phdl = handle;
2028 return AE_CTRL_TERMINATE;
2029 }
2030
2031 return AE_OK;
2032 }
2033
2034 static int dmar_device_hotplug(acpi_handle handle, bool insert)
2035 {
2036 int ret;
2037 acpi_handle tmp = NULL;
2038 acpi_status status;
2039
2040 if (!dmar_in_use())
2041 return 0;
2042
2043 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2044 tmp = handle;
2045 } else {
2046 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
2047 ACPI_UINT32_MAX,
2048 dmar_get_dsm_handle,
2049 NULL, NULL, &tmp);
2050 if (ACPI_FAILURE(status)) {
2051 pr_warn("Failed to locate _DSM method.\n");
2052 return -ENXIO;
2053 }
2054 }
2055 if (tmp == NULL)
2056 return 0;
2057
2058 down_write(&dmar_global_lock);
2059 if (insert)
2060 ret = dmar_hotplug_insert(tmp);
2061 else
2062 ret = dmar_hotplug_remove(tmp);
2063 up_write(&dmar_global_lock);
2064
2065 return ret;
2066 }
2067
2068 int dmar_device_add(acpi_handle handle)
2069 {
2070 return dmar_device_hotplug(handle, true);
2071 }
2072
2073 int dmar_device_remove(acpi_handle handle)
2074 {
2075 return dmar_device_hotplug(handle, false);
2076 }
2077
2078 /*
2079 * dmar_platform_optin - Is %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in DMAR table
2080 *
2081 * Returns true if the platform has %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in
2082 * the ACPI DMAR table. This means that the platform boot firmware has made
2083 * sure no device can issue DMA outside of RMRR regions.
2084 */
2085 bool dmar_platform_optin(void)
2086 {
2087 struct acpi_table_dmar *dmar;
2088 acpi_status status;
2089 bool ret;
2090
2091 status = acpi_get_table(ACPI_SIG_DMAR, 0,
2092 (struct acpi_table_header **)&dmar);
2093 if (ACPI_FAILURE(status))
2094 return false;
2095
2096 ret = !!(dmar->flags & DMAR_PLATFORM_OPT_IN);
2097 acpi_put_table((struct acpi_table_header *)dmar);
2098
2099 return ret;
2100 }
2101 EXPORT_SYMBOL_GPL(dmar_platform_optin);
Linux with added FPC-III support