?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
mm/zsmalloc: allocate exactly size of struct zs_pool
[linux/fpc-iii.git] / drivers / iommu / dmar.c
blobc5c61cabd6e3ea3508c39b7d799476676fb85d32
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 *
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */
28
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */
30
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/iova.h>
34 #include <linux/intel-iommu.h>
35 #include <linux/timer.h>
36 #include <linux/irq.h>
37 #include <linux/interrupt.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/slab.h>
41 #include <linux/iommu.h>
42 #include <asm/irq_remapping.h>
43 #include <asm/iommu_table.h>
44
45 #include "irq_remapping.h"
46
47 /*
48 * Assumptions:
49 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
50 * before IO devices managed by that unit.
51 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
52 * after IO devices managed by that unit.
53 * 3) Hotplug events are rare.
54 *
55 * Locking rules for DMA and interrupt remapping related global data structures:
56 * 1) Use dmar_global_lock in process context
57 * 2) Use RCU in interrupt context
58 */
59 DECLARE_RWSEM(dmar_global_lock);
60 LIST_HEAD(dmar_drhd_units);
61
62 struct acpi_table_header * __initdata dmar_tbl;
63 static acpi_size dmar_tbl_size;
64 static int dmar_dev_scope_status = 1;
65
66 static int alloc_iommu(struct dmar_drhd_unit *drhd);
67 static void free_iommu(struct intel_iommu *iommu);
68
69 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
70 {
71 /*
72 * add INCLUDE_ALL at the tail, so scan the list will find it at
73 * the very end.
74 */
75 if (drhd->include_all)
76 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
77 else
78 list_add_rcu(&drhd->list, &dmar_drhd_units);
79 }
80
81 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
82 {
83 struct acpi_dmar_device_scope *scope;
84
85 *cnt = 0;
86 while (start < end) {
87 scope = start;
88 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
89 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
90 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
91 (*cnt)++;
92 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
93 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
94 pr_warn("Unsupported device scope\n");
95 }
96 start += scope->length;
97 }
98 if (*cnt == 0)
99 return NULL;
100
101 return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
102 }
103
104 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
105 {
106 int i;
107 struct device *tmp_dev;
108
109 if (*devices && *cnt) {
110 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
111 put_device(tmp_dev);
112 kfree(*devices);
113 }
114
115 *devices = NULL;
116 *cnt = 0;
117 }
118
119 /* Optimize out kzalloc()/kfree() for normal cases */
120 static char dmar_pci_notify_info_buf[64];
121
122 static struct dmar_pci_notify_info *
123 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
124 {
125 int level = 0;
126 size_t size;
127 struct pci_dev *tmp;
128 struct dmar_pci_notify_info *info;
129
130 BUG_ON(dev->is_virtfn);
131
132 /* Only generate path[] for device addition event */
133 if (event == BUS_NOTIFY_ADD_DEVICE)
134 for (tmp = dev; tmp; tmp = tmp->bus->self)
135 level++;
136
137 size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
138 if (size <= sizeof(dmar_pci_notify_info_buf)) {
139 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
140 } else {
141 info = kzalloc(size, GFP_KERNEL);
142 if (!info) {
143 pr_warn("Out of memory when allocating notify_info "
144 "for %s.\n", pci_name(dev));
145 if (dmar_dev_scope_status == 0)
146 dmar_dev_scope_status = -ENOMEM;
147 return NULL;
148 }
149 }
150
151 info->event = event;
152 info->dev = dev;
153 info->seg = pci_domain_nr(dev->bus);
154 info->level = level;
155 if (event == BUS_NOTIFY_ADD_DEVICE) {
156 for (tmp = dev; tmp; tmp = tmp->bus->self) {
157 level--;
158 info->path[level].bus = tmp->bus->number;
159 info->path[level].device = PCI_SLOT(tmp->devfn);
160 info->path[level].function = PCI_FUNC(tmp->devfn);
161 if (pci_is_root_bus(tmp->bus))
162 info->bus = tmp->bus->number;
163 }
164 }
165
166 return info;
167 }
168
169 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
170 {
171 if ((void *)info != dmar_pci_notify_info_buf)
172 kfree(info);
173 }
174
175 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
176 struct acpi_dmar_pci_path *path, int count)
177 {
178 int i;
179
180 if (info->bus != bus)
181 goto fallback;
182 if (info->level != count)
183 goto fallback;
184
185 for (i = 0; i < count; i++) {
186 if (path[i].device != info->path[i].device ||
187 path[i].function != info->path[i].function)
188 goto fallback;
189 }
190
191 return true;
192
193 fallback:
194
195 if (count != 1)
196 return false;
197
198 i = info->level - 1;
199 if (bus == info->path[i].bus &&
200 path[0].device == info->path[i].device &&
201 path[0].function == info->path[i].function) {
202 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
203 bus, path[0].device, path[0].function);
204 return true;
205 }
206
207 return false;
208 }
209
210 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
211 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
212 void *start, void*end, u16 segment,
213 struct dmar_dev_scope *devices,
214 int devices_cnt)
215 {
216 int i, level;
217 struct device *tmp, *dev = &info->dev->dev;
218 struct acpi_dmar_device_scope *scope;
219 struct acpi_dmar_pci_path *path;
220
221 if (segment != info->seg)
222 return 0;
223
224 for (; start < end; start += scope->length) {
225 scope = start;
226 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
227 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
228 continue;
229
230 path = (struct acpi_dmar_pci_path *)(scope + 1);
231 level = (scope->length - sizeof(*scope)) / sizeof(*path);
232 if (!dmar_match_pci_path(info, scope->bus, path, level))
233 continue;
234
235 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^
236 (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) {
237 pr_warn("Device scope type does not match for %s\n",
238 pci_name(info->dev));
239 return -EINVAL;
240 }
241
242 for_each_dev_scope(devices, devices_cnt, i, tmp)
243 if (tmp == NULL) {
244 devices[i].bus = info->dev->bus->number;
245 devices[i].devfn = info->dev->devfn;
246 rcu_assign_pointer(devices[i].dev,
247 get_device(dev));
248 return 1;
249 }
250 BUG_ON(i >= devices_cnt);
251 }
252
253 return 0;
254 }
255
256 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
257 struct dmar_dev_scope *devices, int count)
258 {
259 int index;
260 struct device *tmp;
261
262 if (info->seg != segment)
263 return 0;
264
265 for_each_active_dev_scope(devices, count, index, tmp)
266 if (tmp == &info->dev->dev) {
267 RCU_INIT_POINTER(devices[index].dev, NULL);
268 synchronize_rcu();
269 put_device(tmp);
270 return 1;
271 }
272
273 return 0;
274 }
275
276 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
277 {
278 int ret = 0;
279 struct dmar_drhd_unit *dmaru;
280 struct acpi_dmar_hardware_unit *drhd;
281
282 for_each_drhd_unit(dmaru) {
283 if (dmaru->include_all)
284 continue;
285
286 drhd = container_of(dmaru->hdr,
287 struct acpi_dmar_hardware_unit, header);
288 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
289 ((void *)drhd) + drhd->header.length,
290 dmaru->segment,
291 dmaru->devices, dmaru->devices_cnt);
292 if (ret != 0)
293 break;
294 }
295 if (ret >= 0)
296 ret = dmar_iommu_notify_scope_dev(info);
297 if (ret < 0 && dmar_dev_scope_status == 0)
298 dmar_dev_scope_status = ret;
299
300 return ret;
301 }
302
303 static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
304 {
305 struct dmar_drhd_unit *dmaru;
306
307 for_each_drhd_unit(dmaru)
308 if (dmar_remove_dev_scope(info, dmaru->segment,
309 dmaru->devices, dmaru->devices_cnt))
310 break;
311 dmar_iommu_notify_scope_dev(info);
312 }
313
314 static int dmar_pci_bus_notifier(struct notifier_block *nb,
315 unsigned long action, void *data)
316 {
317 struct pci_dev *pdev = to_pci_dev(data);
318 struct dmar_pci_notify_info *info;
319
320 /* Only care about add/remove events for physical functions */
321 if (pdev->is_virtfn)
322 return NOTIFY_DONE;
323 if (action != BUS_NOTIFY_ADD_DEVICE && action != BUS_NOTIFY_DEL_DEVICE)
324 return NOTIFY_DONE;
325
326 info = dmar_alloc_pci_notify_info(pdev, action);
327 if (!info)
328 return NOTIFY_DONE;
329
330 down_write(&dmar_global_lock);
331 if (action == BUS_NOTIFY_ADD_DEVICE)
332 dmar_pci_bus_add_dev(info);
333 else if (action == BUS_NOTIFY_DEL_DEVICE)
334 dmar_pci_bus_del_dev(info);
335 up_write(&dmar_global_lock);
336
337 dmar_free_pci_notify_info(info);
338
339 return NOTIFY_OK;
340 }
341
342 static struct notifier_block dmar_pci_bus_nb = {
343 .notifier_call = dmar_pci_bus_notifier,
344 .priority = INT_MIN,
345 };
346
347 /**
348 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
349 * structure which uniquely represent one DMA remapping hardware unit
350 * present in the platform
351 */
352 static int __init
353 dmar_parse_one_drhd(struct acpi_dmar_header *header)
354 {
355 struct acpi_dmar_hardware_unit *drhd;
356 struct dmar_drhd_unit *dmaru;
357 int ret = 0;
358
359 drhd = (struct acpi_dmar_hardware_unit *)header;
360 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
361 if (!dmaru)
362 return -ENOMEM;
363
364 dmaru->hdr = header;
365 dmaru->reg_base_addr = drhd->address;
366 dmaru->segment = drhd->segment;
367 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
368 dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
369 ((void *)drhd) + drhd->header.length,
370 &dmaru->devices_cnt);
371 if (dmaru->devices_cnt && dmaru->devices == NULL) {
372 kfree(dmaru);
373 return -ENOMEM;
374 }
375
376 ret = alloc_iommu(dmaru);
377 if (ret) {
378 dmar_free_dev_scope(&dmaru->devices,
379 &dmaru->devices_cnt);
380 kfree(dmaru);
381 return ret;
382 }
383 dmar_register_drhd_unit(dmaru);
384 return 0;
385 }
386
387 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
388 {
389 if (dmaru->devices && dmaru->devices_cnt)
390 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
391 if (dmaru->iommu)
392 free_iommu(dmaru->iommu);
393 kfree(dmaru);
394 }
395
396 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header)
397 {
398 struct acpi_dmar_andd *andd = (void *)header;
399
400 /* Check for NUL termination within the designated length */
401 if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
402 WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
403 "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
404 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
405 dmi_get_system_info(DMI_BIOS_VENDOR),
406 dmi_get_system_info(DMI_BIOS_VERSION),
407 dmi_get_system_info(DMI_PRODUCT_VERSION));
408 return -EINVAL;
409 }
410 pr_info("ANDD device: %x name: %s\n", andd->device_number,
411 andd->device_name);
412
413 return 0;
414 }
415
416 #ifdef CONFIG_ACPI_NUMA
417 static int __init
418 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
419 {
420 struct acpi_dmar_rhsa *rhsa;
421 struct dmar_drhd_unit *drhd;
422
423 rhsa = (struct acpi_dmar_rhsa *)header;
424 for_each_drhd_unit(drhd) {
425 if (drhd->reg_base_addr == rhsa->base_address) {
426 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
427
428 if (!node_online(node))
429 node = -1;
430 drhd->iommu->node = node;
431 return 0;
432 }
433 }
434 WARN_TAINT(
435 1, TAINT_FIRMWARE_WORKAROUND,
436 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
437 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
438 drhd->reg_base_addr,
439 dmi_get_system_info(DMI_BIOS_VENDOR),
440 dmi_get_system_info(DMI_BIOS_VERSION),
441 dmi_get_system_info(DMI_PRODUCT_VERSION));
442
443 return 0;
444 }
445 #endif
446
447 static void __init
448 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
449 {
450 struct acpi_dmar_hardware_unit *drhd;
451 struct acpi_dmar_reserved_memory *rmrr;
452 struct acpi_dmar_atsr *atsr;
453 struct acpi_dmar_rhsa *rhsa;
454
455 switch (header->type) {
456 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
457 drhd = container_of(header, struct acpi_dmar_hardware_unit,
458 header);
459 pr_info("DRHD base: %#016Lx flags: %#x\n",
460 (unsigned long long)drhd->address, drhd->flags);
461 break;
462 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
463 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
464 header);
465 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
466 (unsigned long long)rmrr->base_address,
467 (unsigned long long)rmrr->end_address);
468 break;
469 case ACPI_DMAR_TYPE_ROOT_ATS:
470 atsr = container_of(header, struct acpi_dmar_atsr, header);
471 pr_info("ATSR flags: %#x\n", atsr->flags);
472 break;
473 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
474 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
475 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
476 (unsigned long long)rhsa->base_address,
477 rhsa->proximity_domain);
478 break;
479 case ACPI_DMAR_TYPE_NAMESPACE:
480 /* We don't print this here because we need to sanity-check
481 it first. So print it in dmar_parse_one_andd() instead. */
482 break;
483 }
484 }
485
486 /**
487 * dmar_table_detect - checks to see if the platform supports DMAR devices
488 */
489 static int __init dmar_table_detect(void)
490 {
491 acpi_status status = AE_OK;
492
493 /* if we could find DMAR table, then there are DMAR devices */
494 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
495 (struct acpi_table_header **)&dmar_tbl,
496 &dmar_tbl_size);
497
498 if (ACPI_SUCCESS(status) && !dmar_tbl) {
499 pr_warn("Unable to map DMAR\n");
500 status = AE_NOT_FOUND;
501 }
502
503 return (ACPI_SUCCESS(status) ? 1 : 0);
504 }
505
506 /**
507 * parse_dmar_table - parses the DMA reporting table
508 */
509 static int __init
510 parse_dmar_table(void)
511 {
512 struct acpi_table_dmar *dmar;
513 struct acpi_dmar_header *entry_header;
514 int ret = 0;
515 int drhd_count = 0;
516
517 /*
518 * Do it again, earlier dmar_tbl mapping could be mapped with
519 * fixed map.
520 */
521 dmar_table_detect();
522
523 /*
524 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
525 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
526 */
527 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
528
529 dmar = (struct acpi_table_dmar *)dmar_tbl;
530 if (!dmar)
531 return -ENODEV;
532
533 if (dmar->width < PAGE_SHIFT - 1) {
534 pr_warn("Invalid DMAR haw\n");
535 return -EINVAL;
536 }
537
538 pr_info("Host address width %d\n", dmar->width + 1);
539
540 entry_header = (struct acpi_dmar_header *)(dmar + 1);
541 while (((unsigned long)entry_header) <
542 (((unsigned long)dmar) + dmar_tbl->length)) {
543 /* Avoid looping forever on bad ACPI tables */
544 if (entry_header->length == 0) {
545 pr_warn("Invalid 0-length structure\n");
546 ret = -EINVAL;
547 break;
548 }
549
550 dmar_table_print_dmar_entry(entry_header);
551
552 switch (entry_header->type) {
553 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
554 drhd_count++;
555 ret = dmar_parse_one_drhd(entry_header);
556 break;
557 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
558 ret = dmar_parse_one_rmrr(entry_header);
559 break;
560 case ACPI_DMAR_TYPE_ROOT_ATS:
561 ret = dmar_parse_one_atsr(entry_header);
562 break;
563 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
564 #ifdef CONFIG_ACPI_NUMA
565 ret = dmar_parse_one_rhsa(entry_header);
566 #endif
567 break;
568 case ACPI_DMAR_TYPE_NAMESPACE:
569 ret = dmar_parse_one_andd(entry_header);
570 break;
571 default:
572 pr_warn("Unknown DMAR structure type %d\n",
573 entry_header->type);
574 ret = 0; /* for forward compatibility */
575 break;
576 }
577 if (ret)
578 break;
579
580 entry_header = ((void *)entry_header + entry_header->length);
581 }
582 if (drhd_count == 0)
583 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
584 return ret;
585 }
586
587 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
588 int cnt, struct pci_dev *dev)
589 {
590 int index;
591 struct device *tmp;
592
593 while (dev) {
594 for_each_active_dev_scope(devices, cnt, index, tmp)
595 if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
596 return 1;
597
598 /* Check our parent */
599 dev = dev->bus->self;
600 }
601
602 return 0;
603 }
604
605 struct dmar_drhd_unit *
606 dmar_find_matched_drhd_unit(struct pci_dev *dev)
607 {
608 struct dmar_drhd_unit *dmaru;
609 struct acpi_dmar_hardware_unit *drhd;
610
611 dev = pci_physfn(dev);
612
613 rcu_read_lock();
614 for_each_drhd_unit(dmaru) {
615 drhd = container_of(dmaru->hdr,
616 struct acpi_dmar_hardware_unit,
617 header);
618
619 if (dmaru->include_all &&
620 drhd->segment == pci_domain_nr(dev->bus))
621 goto out;
622
623 if (dmar_pci_device_match(dmaru->devices,
624 dmaru->devices_cnt, dev))
625 goto out;
626 }
627 dmaru = NULL;
628 out:
629 rcu_read_unlock();
630
631 return dmaru;
632 }
633
634 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
635 struct acpi_device *adev)
636 {
637 struct dmar_drhd_unit *dmaru;
638 struct acpi_dmar_hardware_unit *drhd;
639 struct acpi_dmar_device_scope *scope;
640 struct device *tmp;
641 int i;
642 struct acpi_dmar_pci_path *path;
643
644 for_each_drhd_unit(dmaru) {
645 drhd = container_of(dmaru->hdr,
646 struct acpi_dmar_hardware_unit,
647 header);
648
649 for (scope = (void *)(drhd + 1);
650 (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
651 scope = ((void *)scope) + scope->length) {
652 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
653 continue;
654 if (scope->enumeration_id != device_number)
655 continue;
656
657 path = (void *)(scope + 1);
658 pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
659 dev_name(&adev->dev), dmaru->reg_base_addr,
660 scope->bus, path->device, path->function);
661 for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
662 if (tmp == NULL) {
663 dmaru->devices[i].bus = scope->bus;
664 dmaru->devices[i].devfn = PCI_DEVFN(path->device,
665 path->function);
666 rcu_assign_pointer(dmaru->devices[i].dev,
667 get_device(&adev->dev));
668 return;
669 }
670 BUG_ON(i >= dmaru->devices_cnt);
671 }
672 }
673 pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
674 device_number, dev_name(&adev->dev));
675 }
676
677 static int __init dmar_acpi_dev_scope_init(void)
678 {
679 struct acpi_dmar_andd *andd;
680
681 if (dmar_tbl == NULL)
682 return -ENODEV;
683
684 for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
685 ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
686 andd = ((void *)andd) + andd->header.length) {
687 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
688 acpi_handle h;
689 struct acpi_device *adev;
690
691 if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
692 andd->device_name,
693 &h))) {
694 pr_err("Failed to find handle for ACPI object %s\n",
695 andd->device_name);
696 continue;
697 }
698 if (acpi_bus_get_device(h, &adev)) {
699 pr_err("Failed to get device for ACPI object %s\n",
700 andd->device_name);
701 continue;
702 }
703 dmar_acpi_insert_dev_scope(andd->device_number, adev);
704 }
705 }
706 return 0;
707 }
708
709 int __init dmar_dev_scope_init(void)
710 {
711 struct pci_dev *dev = NULL;
712 struct dmar_pci_notify_info *info;
713
714 if (dmar_dev_scope_status != 1)
715 return dmar_dev_scope_status;
716
717 if (list_empty(&dmar_drhd_units)) {
718 dmar_dev_scope_status = -ENODEV;
719 } else {
720 dmar_dev_scope_status = 0;
721
722 dmar_acpi_dev_scope_init();
723
724 for_each_pci_dev(dev) {
725 if (dev->is_virtfn)
726 continue;
727
728 info = dmar_alloc_pci_notify_info(dev,
729 BUS_NOTIFY_ADD_DEVICE);
730 if (!info) {
731 return dmar_dev_scope_status;
732 } else {
733 dmar_pci_bus_add_dev(info);
734 dmar_free_pci_notify_info(info);
735 }
736 }
737
738 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
739 }
740
741 return dmar_dev_scope_status;
742 }
743
744
745 int __init dmar_table_init(void)
746 {
747 static int dmar_table_initialized;
748 int ret;
749
750 if (dmar_table_initialized == 0) {
751 ret = parse_dmar_table();
752 if (ret < 0) {
753 if (ret != -ENODEV)
754 pr_info("parse DMAR table failure.\n");
755 } else if (list_empty(&dmar_drhd_units)) {
756 pr_info("No DMAR devices found\n");
757 ret = -ENODEV;
758 }
759
760 if (ret < 0)
761 dmar_table_initialized = ret;
762 else
763 dmar_table_initialized = 1;
764 }
765
766 return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
767 }
768
769 static void warn_invalid_dmar(u64 addr, const char *message)
770 {
771 WARN_TAINT_ONCE(
772 1, TAINT_FIRMWARE_WORKAROUND,
773 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
774 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
775 addr, message,
776 dmi_get_system_info(DMI_BIOS_VENDOR),
777 dmi_get_system_info(DMI_BIOS_VERSION),
778 dmi_get_system_info(DMI_PRODUCT_VERSION));
779 }
780
781 static int __init check_zero_address(void)
782 {
783 struct acpi_table_dmar *dmar;
784 struct acpi_dmar_header *entry_header;
785 struct acpi_dmar_hardware_unit *drhd;
786
787 dmar = (struct acpi_table_dmar *)dmar_tbl;
788 entry_header = (struct acpi_dmar_header *)(dmar + 1);
789
790 while (((unsigned long)entry_header) <
791 (((unsigned long)dmar) + dmar_tbl->length)) {
792 /* Avoid looping forever on bad ACPI tables */
793 if (entry_header->length == 0) {
794 pr_warn("Invalid 0-length structure\n");
795 return 0;
796 }
797
798 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
799 void __iomem *addr;
800 u64 cap, ecap;
801
802 drhd = (void *)entry_header;
803 if (!drhd->address) {
804 warn_invalid_dmar(0, "");
805 goto failed;
806 }
807
808 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
809 if (!addr ) {
810 printk("IOMMU: can't validate: %llx\n", drhd->address);
811 goto failed;
812 }
813 cap = dmar_readq(addr + DMAR_CAP_REG);
814 ecap = dmar_readq(addr + DMAR_ECAP_REG);
815 early_iounmap(addr, VTD_PAGE_SIZE);
816 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
817 warn_invalid_dmar(drhd->address,
818 " returns all ones");
819 goto failed;
820 }
821 }
822
823 entry_header = ((void *)entry_header + entry_header->length);
824 }
825 return 1;
826
827 failed:
828 return 0;
829 }
830
831 int __init detect_intel_iommu(void)
832 {
833 int ret;
834
835 down_write(&dmar_global_lock);
836 ret = dmar_table_detect();
837 if (ret)
838 ret = check_zero_address();
839 {
840 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
841 iommu_detected = 1;
842 /* Make sure ACS will be enabled */
843 pci_request_acs();
844 }
845
846 #ifdef CONFIG_X86
847 if (ret)
848 x86_init.iommu.iommu_init = intel_iommu_init;
849 #endif
850 }
851 early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
852 dmar_tbl = NULL;
853 up_write(&dmar_global_lock);
854
855 return ret ? 1 : -ENODEV;
856 }
857
858
859 static void unmap_iommu(struct intel_iommu *iommu)
860 {
861 iounmap(iommu->reg);
862 release_mem_region(iommu->reg_phys, iommu->reg_size);
863 }
864
865 /**
866 * map_iommu: map the iommu's registers
867 * @iommu: the iommu to map
868 * @phys_addr: the physical address of the base resgister
869 *
870 * Memory map the iommu's registers. Start w/ a single page, and
871 * possibly expand if that turns out to be insufficent.
872 */
873 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
874 {
875 int map_size, err=0;
876
877 iommu->reg_phys = phys_addr;
878 iommu->reg_size = VTD_PAGE_SIZE;
879
880 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
881 pr_err("IOMMU: can't reserve memory\n");
882 err = -EBUSY;
883 goto out;
884 }
885
886 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
887 if (!iommu->reg) {
888 pr_err("IOMMU: can't map the region\n");
889 err = -ENOMEM;
890 goto release;
891 }
892
893 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
894 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
895
896 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
897 err = -EINVAL;
898 warn_invalid_dmar(phys_addr, " returns all ones");
899 goto unmap;
900 }
901
902 /* the registers might be more than one page */
903 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
904 cap_max_fault_reg_offset(iommu->cap));
905 map_size = VTD_PAGE_ALIGN(map_size);
906 if (map_size > iommu->reg_size) {
907 iounmap(iommu->reg);
908 release_mem_region(iommu->reg_phys, iommu->reg_size);
909 iommu->reg_size = map_size;
910 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
911 iommu->name)) {
912 pr_err("IOMMU: can't reserve memory\n");
913 err = -EBUSY;
914 goto out;
915 }
916 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
917 if (!iommu->reg) {
918 pr_err("IOMMU: can't map the region\n");
919 err = -ENOMEM;
920 goto release;
921 }
922 }
923 err = 0;
924 goto out;
925
926 unmap:
927 iounmap(iommu->reg);
928 release:
929 release_mem_region(iommu->reg_phys, iommu->reg_size);
930 out:
931 return err;
932 }
933
934 static int alloc_iommu(struct dmar_drhd_unit *drhd)
935 {
936 struct intel_iommu *iommu;
937 u32 ver, sts;
938 static int iommu_allocated = 0;
939 int agaw = 0;
940 int msagaw = 0;
941 int err;
942
943 if (!drhd->reg_base_addr) {
944 warn_invalid_dmar(0, "");
945 return -EINVAL;
946 }
947
948 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
949 if (!iommu)
950 return -ENOMEM;
951
952 iommu->seq_id = iommu_allocated++;
953 sprintf (iommu->name, "dmar%d", iommu->seq_id);
954
955 err = map_iommu(iommu, drhd->reg_base_addr);
956 if (err) {
957 pr_err("IOMMU: failed to map %s\n", iommu->name);
958 goto error;
959 }
960
961 err = -EINVAL;
962 agaw = iommu_calculate_agaw(iommu);
963 if (agaw < 0) {
964 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
965 iommu->seq_id);
966 goto err_unmap;
967 }
968 msagaw = iommu_calculate_max_sagaw(iommu);
969 if (msagaw < 0) {
970 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
971 iommu->seq_id);
972 goto err_unmap;
973 }
974 iommu->agaw = agaw;
975 iommu->msagaw = msagaw;
976 iommu->segment = drhd->segment;
977
978 iommu->node = -1;
979
980 ver = readl(iommu->reg + DMAR_VER_REG);
981 pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
982 iommu->seq_id,
983 (unsigned long long)drhd->reg_base_addr,
984 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
985 (unsigned long long)iommu->cap,
986 (unsigned long long)iommu->ecap);
987
988 /* Reflect status in gcmd */
989 sts = readl(iommu->reg + DMAR_GSTS_REG);
990 if (sts & DMA_GSTS_IRES)
991 iommu->gcmd |= DMA_GCMD_IRE;
992 if (sts & DMA_GSTS_TES)
993 iommu->gcmd |= DMA_GCMD_TE;
994 if (sts & DMA_GSTS_QIES)
995 iommu->gcmd |= DMA_GCMD_QIE;
996
997 raw_spin_lock_init(&iommu->register_lock);
998
999 drhd->iommu = iommu;
1000
1001 if (intel_iommu_enabled)
1002 iommu->iommu_dev = iommu_device_create(NULL, iommu,
1003 intel_iommu_groups,
1004 iommu->name);
1005
1006 return 0;
1007
1008 err_unmap:
1009 unmap_iommu(iommu);
1010 error:
1011 kfree(iommu);
1012 return err;
1013 }
1014
1015 static void free_iommu(struct intel_iommu *iommu)
1016 {
1017 iommu_device_destroy(iommu->iommu_dev);
1018
1019 if (iommu->irq) {
1020 free_irq(iommu->irq, iommu);
1021 irq_set_handler_data(iommu->irq, NULL);
1022 dmar_free_hwirq(iommu->irq);
1023 }
1024
1025 if (iommu->qi) {
1026 free_page((unsigned long)iommu->qi->desc);
1027 kfree(iommu->qi->desc_status);
1028 kfree(iommu->qi);
1029 }
1030
1031 if (iommu->reg)
1032 unmap_iommu(iommu);
1033
1034 kfree(iommu);
1035 }
1036
1037 /*
1038 * Reclaim all the submitted descriptors which have completed its work.
1039 */
1040 static inline void reclaim_free_desc(struct q_inval *qi)
1041 {
1042 while (qi->desc_status[qi->free_tail] == QI_DONE ||
1043 qi->desc_status[qi->free_tail] == QI_ABORT) {
1044 qi->desc_status[qi->free_tail] = QI_FREE;
1045 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1046 qi->free_cnt++;
1047 }
1048 }
1049
1050 static int qi_check_fault(struct intel_iommu *iommu, int index)
1051 {
1052 u32 fault;
1053 int head, tail;
1054 struct q_inval *qi = iommu->qi;
1055 int wait_index = (index + 1) % QI_LENGTH;
1056
1057 if (qi->desc_status[wait_index] == QI_ABORT)
1058 return -EAGAIN;
1059
1060 fault = readl(iommu->reg + DMAR_FSTS_REG);
1061
1062 /*
1063 * If IQE happens, the head points to the descriptor associated
1064 * with the error. No new descriptors are fetched until the IQE
1065 * is cleared.
1066 */
1067 if (fault & DMA_FSTS_IQE) {
1068 head = readl(iommu->reg + DMAR_IQH_REG);
1069 if ((head >> DMAR_IQ_SHIFT) == index) {
1070 pr_err("VT-d detected invalid descriptor: "
1071 "low=%llx, high=%llx\n",
1072 (unsigned long long)qi->desc[index].low,
1073 (unsigned long long)qi->desc[index].high);
1074 memcpy(&qi->desc[index], &qi->desc[wait_index],
1075 sizeof(struct qi_desc));
1076 __iommu_flush_cache(iommu, &qi->desc[index],
1077 sizeof(struct qi_desc));
1078 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1079 return -EINVAL;
1080 }
1081 }
1082
1083 /*
1084 * If ITE happens, all pending wait_desc commands are aborted.
1085 * No new descriptors are fetched until the ITE is cleared.
1086 */
1087 if (fault & DMA_FSTS_ITE) {
1088 head = readl(iommu->reg + DMAR_IQH_REG);
1089 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1090 head |= 1;
1091 tail = readl(iommu->reg + DMAR_IQT_REG);
1092 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1093
1094 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1095
1096 do {
1097 if (qi->desc_status[head] == QI_IN_USE)
1098 qi->desc_status[head] = QI_ABORT;
1099 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1100 } while (head != tail);
1101
1102 if (qi->desc_status[wait_index] == QI_ABORT)
1103 return -EAGAIN;
1104 }
1105
1106 if (fault & DMA_FSTS_ICE)
1107 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1108
1109 return 0;
1110 }
1111
1112 /*
1113 * Submit the queued invalidation descriptor to the remapping
1114 * hardware unit and wait for its completion.
1115 */
1116 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1117 {
1118 int rc;
1119 struct q_inval *qi = iommu->qi;
1120 struct qi_desc *hw, wait_desc;
1121 int wait_index, index;
1122 unsigned long flags;
1123
1124 if (!qi)
1125 return 0;
1126
1127 hw = qi->desc;
1128
1129 restart:
1130 rc = 0;
1131
1132 raw_spin_lock_irqsave(&qi->q_lock, flags);
1133 while (qi->free_cnt < 3) {
1134 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1135 cpu_relax();
1136 raw_spin_lock_irqsave(&qi->q_lock, flags);
1137 }
1138
1139 index = qi->free_head;
1140 wait_index = (index + 1) % QI_LENGTH;
1141
1142 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1143
1144 hw[index] = *desc;
1145
1146 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1147 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1148 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1149
1150 hw[wait_index] = wait_desc;
1151
1152 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
1153 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
1154
1155 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1156 qi->free_cnt -= 2;
1157
1158 /*
1159 * update the HW tail register indicating the presence of
1160 * new descriptors.
1161 */
1162 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
1163
1164 while (qi->desc_status[wait_index] != QI_DONE) {
1165 /*
1166 * We will leave the interrupts disabled, to prevent interrupt
1167 * context to queue another cmd while a cmd is already submitted
1168 * and waiting for completion on this cpu. This is to avoid
1169 * a deadlock where the interrupt context can wait indefinitely
1170 * for free slots in the queue.
1171 */
1172 rc = qi_check_fault(iommu, index);
1173 if (rc)
1174 break;
1175
1176 raw_spin_unlock(&qi->q_lock);
1177 cpu_relax();
1178 raw_spin_lock(&qi->q_lock);
1179 }
1180
1181 qi->desc_status[index] = QI_DONE;
1182
1183 reclaim_free_desc(qi);
1184 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1185
1186 if (rc == -EAGAIN)
1187 goto restart;
1188
1189 return rc;
1190 }
1191
1192 /*
1193 * Flush the global interrupt entry cache.
1194 */
1195 void qi_global_iec(struct intel_iommu *iommu)
1196 {
1197 struct qi_desc desc;
1198
1199 desc.low = QI_IEC_TYPE;
1200 desc.high = 0;
1201
1202 /* should never fail */
1203 qi_submit_sync(&desc, iommu);
1204 }
1205
1206 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1207 u64 type)
1208 {
1209 struct qi_desc desc;
1210
1211 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1212 | QI_CC_GRAN(type) | QI_CC_TYPE;
1213 desc.high = 0;
1214
1215 qi_submit_sync(&desc, iommu);
1216 }
1217
1218 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1219 unsigned int size_order, u64 type)
1220 {
1221 u8 dw = 0, dr = 0;
1222
1223 struct qi_desc desc;
1224 int ih = 0;
1225
1226 if (cap_write_drain(iommu->cap))
1227 dw = 1;
1228
1229 if (cap_read_drain(iommu->cap))
1230 dr = 1;
1231
1232 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1233 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1234 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1235 | QI_IOTLB_AM(size_order);
1236
1237 qi_submit_sync(&desc, iommu);
1238 }
1239
1240 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1241 u64 addr, unsigned mask)
1242 {
1243 struct qi_desc desc;
1244
1245 if (mask) {
1246 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1247 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1248 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1249 } else
1250 desc.high = QI_DEV_IOTLB_ADDR(addr);
1251
1252 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1253 qdep = 0;
1254
1255 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1256 QI_DIOTLB_TYPE;
1257
1258 qi_submit_sync(&desc, iommu);
1259 }
1260
1261 /*
1262 * Disable Queued Invalidation interface.
1263 */
1264 void dmar_disable_qi(struct intel_iommu *iommu)
1265 {
1266 unsigned long flags;
1267 u32 sts;
1268 cycles_t start_time = get_cycles();
1269
1270 if (!ecap_qis(iommu->ecap))
1271 return;
1272
1273 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1274
1275 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
1276 if (!(sts & DMA_GSTS_QIES))
1277 goto end;
1278
1279 /*
1280 * Give a chance to HW to complete the pending invalidation requests.
1281 */
1282 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1283 readl(iommu->reg + DMAR_IQH_REG)) &&
1284 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1285 cpu_relax();
1286
1287 iommu->gcmd &= ~DMA_GCMD_QIE;
1288 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1289
1290 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1291 !(sts & DMA_GSTS_QIES), sts);
1292 end:
1293 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1294 }
1295
1296 /*
1297 * Enable queued invalidation.
1298 */
1299 static void __dmar_enable_qi(struct intel_iommu *iommu)
1300 {
1301 u32 sts;
1302 unsigned long flags;
1303 struct q_inval *qi = iommu->qi;
1304
1305 qi->free_head = qi->free_tail = 0;
1306 qi->free_cnt = QI_LENGTH;
1307
1308 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1309
1310 /* write zero to the tail reg */
1311 writel(0, iommu->reg + DMAR_IQT_REG);
1312
1313 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1314
1315 iommu->gcmd |= DMA_GCMD_QIE;
1316 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1317
1318 /* Make sure hardware complete it */
1319 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1320
1321 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1322 }
1323
1324 /*
1325 * Enable Queued Invalidation interface. This is a must to support
1326 * interrupt-remapping. Also used by DMA-remapping, which replaces
1327 * register based IOTLB invalidation.
1328 */
1329 int dmar_enable_qi(struct intel_iommu *iommu)
1330 {
1331 struct q_inval *qi;
1332 struct page *desc_page;
1333
1334 if (!ecap_qis(iommu->ecap))
1335 return -ENOENT;
1336
1337 /*
1338 * queued invalidation is already setup and enabled.
1339 */
1340 if (iommu->qi)
1341 return 0;
1342
1343 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1344 if (!iommu->qi)
1345 return -ENOMEM;
1346
1347 qi = iommu->qi;
1348
1349
1350 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1351 if (!desc_page) {
1352 kfree(qi);
1353 iommu->qi = NULL;
1354 return -ENOMEM;
1355 }
1356
1357 qi->desc = page_address(desc_page);
1358
1359 qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1360 if (!qi->desc_status) {
1361 free_page((unsigned long) qi->desc);
1362 kfree(qi);
1363 iommu->qi = NULL;
1364 return -ENOMEM;
1365 }
1366
1367 raw_spin_lock_init(&qi->q_lock);
1368
1369 __dmar_enable_qi(iommu);
1370
1371 return 0;
1372 }
1373
1374 /* iommu interrupt handling. Most stuff are MSI-like. */
1375
1376 enum faulttype {
1377 DMA_REMAP,
1378 INTR_REMAP,
1379 UNKNOWN,
1380 };
1381
1382 static const char *dma_remap_fault_reasons[] =
1383 {
1384 "Software",
1385 "Present bit in root entry is clear",
1386 "Present bit in context entry is clear",
1387 "Invalid context entry",
1388 "Access beyond MGAW",
1389 "PTE Write access is not set",
1390 "PTE Read access is not set",
1391 "Next page table ptr is invalid",
1392 "Root table address invalid",
1393 "Context table ptr is invalid",
1394 "non-zero reserved fields in RTP",
1395 "non-zero reserved fields in CTP",
1396 "non-zero reserved fields in PTE",
1397 "PCE for translation request specifies blocking",
1398 };
1399
1400 static const char *irq_remap_fault_reasons[] =
1401 {
1402 "Detected reserved fields in the decoded interrupt-remapped request",
1403 "Interrupt index exceeded the interrupt-remapping table size",
1404 "Present field in the IRTE entry is clear",
1405 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1406 "Detected reserved fields in the IRTE entry",
1407 "Blocked a compatibility format interrupt request",
1408 "Blocked an interrupt request due to source-id verification failure",
1409 };
1410
1411 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1412 {
1413 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1414 ARRAY_SIZE(irq_remap_fault_reasons))) {
1415 *fault_type = INTR_REMAP;
1416 return irq_remap_fault_reasons[fault_reason - 0x20];
1417 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1418 *fault_type = DMA_REMAP;
1419 return dma_remap_fault_reasons[fault_reason];
1420 } else {
1421 *fault_type = UNKNOWN;
1422 return "Unknown";
1423 }
1424 }
1425
1426 void dmar_msi_unmask(struct irq_data *data)
1427 {
1428 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1429 unsigned long flag;
1430
1431 /* unmask it */
1432 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1433 writel(0, iommu->reg + DMAR_FECTL_REG);
1434 /* Read a reg to force flush the post write */
1435 readl(iommu->reg + DMAR_FECTL_REG);
1436 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1437 }
1438
1439 void dmar_msi_mask(struct irq_data *data)
1440 {
1441 unsigned long flag;
1442 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1443
1444 /* mask it */
1445 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1446 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1447 /* Read a reg to force flush the post write */
1448 readl(iommu->reg + DMAR_FECTL_REG);
1449 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1450 }
1451
1452 void dmar_msi_write(int irq, struct msi_msg *msg)
1453 {
1454 struct intel_iommu *iommu = irq_get_handler_data(irq);
1455 unsigned long flag;
1456
1457 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1458 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1459 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1460 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1461 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1462 }
1463
1464 void dmar_msi_read(int irq, struct msi_msg *msg)
1465 {
1466 struct intel_iommu *iommu = irq_get_handler_data(irq);
1467 unsigned long flag;
1468
1469 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1470 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1471 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1472 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1473 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1474 }
1475
1476 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1477 u8 fault_reason, u16 source_id, unsigned long long addr)
1478 {
1479 const char *reason;
1480 int fault_type;
1481
1482 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1483
1484 if (fault_type == INTR_REMAP)
1485 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1486 "fault index %llx\n"
1487 "INTR-REMAP:[fault reason %02d] %s\n",
1488 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1489 PCI_FUNC(source_id & 0xFF), addr >> 48,
1490 fault_reason, reason);
1491 else
1492 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1493 "fault addr %llx \n"
1494 "DMAR:[fault reason %02d] %s\n",
1495 (type ? "DMA Read" : "DMA Write"),
1496 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1497 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1498 return 0;
1499 }
1500
1501 #define PRIMARY_FAULT_REG_LEN (16)
1502 irqreturn_t dmar_fault(int irq, void *dev_id)
1503 {
1504 struct intel_iommu *iommu = dev_id;
1505 int reg, fault_index;
1506 u32 fault_status;
1507 unsigned long flag;
1508
1509 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1510 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1511 if (fault_status)
1512 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1513
1514 /* TBD: ignore advanced fault log currently */
1515 if (!(fault_status & DMA_FSTS_PPF))
1516 goto unlock_exit;
1517
1518 fault_index = dma_fsts_fault_record_index(fault_status);
1519 reg = cap_fault_reg_offset(iommu->cap);
1520 while (1) {
1521 u8 fault_reason;
1522 u16 source_id;
1523 u64 guest_addr;
1524 int type;
1525 u32 data;
1526
1527 /* highest 32 bits */
1528 data = readl(iommu->reg + reg +
1529 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1530 if (!(data & DMA_FRCD_F))
1531 break;
1532
1533 fault_reason = dma_frcd_fault_reason(data);
1534 type = dma_frcd_type(data);
1535
1536 data = readl(iommu->reg + reg +
1537 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1538 source_id = dma_frcd_source_id(data);
1539
1540 guest_addr = dmar_readq(iommu->reg + reg +
1541 fault_index * PRIMARY_FAULT_REG_LEN);
1542 guest_addr = dma_frcd_page_addr(guest_addr);
1543 /* clear the fault */
1544 writel(DMA_FRCD_F, iommu->reg + reg +
1545 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1546
1547 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1548
1549 dmar_fault_do_one(iommu, type, fault_reason,
1550 source_id, guest_addr);
1551
1552 fault_index++;
1553 if (fault_index >= cap_num_fault_regs(iommu->cap))
1554 fault_index = 0;
1555 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1556 }
1557
1558 writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1559
1560 unlock_exit:
1561 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1562 return IRQ_HANDLED;
1563 }
1564
1565 int dmar_set_interrupt(struct intel_iommu *iommu)
1566 {
1567 int irq, ret;
1568
1569 /*
1570 * Check if the fault interrupt is already initialized.
1571 */
1572 if (iommu->irq)
1573 return 0;
1574
1575 irq = dmar_alloc_hwirq();
1576 if (irq <= 0) {
1577 pr_err("IOMMU: no free vectors\n");
1578 return -EINVAL;
1579 }
1580
1581 irq_set_handler_data(irq, iommu);
1582 iommu->irq = irq;
1583
1584 ret = arch_setup_dmar_msi(irq);
1585 if (ret) {
1586 irq_set_handler_data(irq, NULL);
1587 iommu->irq = 0;
1588 dmar_free_hwirq(irq);
1589 return ret;
1590 }
1591
1592 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1593 if (ret)
1594 pr_err("IOMMU: can't request irq\n");
1595 return ret;
1596 }
1597
1598 int __init enable_drhd_fault_handling(void)
1599 {
1600 struct dmar_drhd_unit *drhd;
1601 struct intel_iommu *iommu;
1602
1603 /*
1604 * Enable fault control interrupt.
1605 */
1606 for_each_iommu(iommu, drhd) {
1607 u32 fault_status;
1608 int ret = dmar_set_interrupt(iommu);
1609
1610 if (ret) {
1611 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1612 (unsigned long long)drhd->reg_base_addr, ret);
1613 return -1;
1614 }
1615
1616 /*
1617 * Clear any previous faults.
1618 */
1619 dmar_fault(iommu->irq, iommu);
1620 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1621 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1622 }
1623
1624 return 0;
1625 }
1626
1627 /*
1628 * Re-enable Queued Invalidation interface.
1629 */
1630 int dmar_reenable_qi(struct intel_iommu *iommu)
1631 {
1632 if (!ecap_qis(iommu->ecap))
1633 return -ENOENT;
1634
1635 if (!iommu->qi)
1636 return -ENOENT;
1637
1638 /*
1639 * First disable queued invalidation.
1640 */
1641 dmar_disable_qi(iommu);
1642 /*
1643 * Then enable queued invalidation again. Since there is no pending
1644 * invalidation requests now, it's safe to re-enable queued
1645 * invalidation.
1646 */
1647 __dmar_enable_qi(iommu);
1648
1649 return 0;
1650 }
1651
1652 /*
1653 * Check interrupt remapping support in DMAR table description.
1654 */
1655 int __init dmar_ir_support(void)
1656 {
1657 struct acpi_table_dmar *dmar;
1658 dmar = (struct acpi_table_dmar *)dmar_tbl;
1659 if (!dmar)
1660 return 0;
1661 return dmar->flags & 0x1;
1662 }
1663
1664 static int __init dmar_free_unused_resources(void)
1665 {
1666 struct dmar_drhd_unit *dmaru, *dmaru_n;
1667
1668 /* DMAR units are in use */
1669 if (irq_remapping_enabled || intel_iommu_enabled)
1670 return 0;
1671
1672 if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1673 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1674
1675 down_write(&dmar_global_lock);
1676 list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1677 list_del(&dmaru->list);
1678 dmar_free_drhd(dmaru);
1679 }
1680 up_write(&dmar_global_lock);
1681
1682 return 0;
1683 }
1684
1685 late_initcall(dmar_free_unused_resources);
1686 IOMMU_INIT_POST(detect_intel_iommu);
Linux with added FPC-III support