?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Revert "tty: hvc: Fix data abort due to race in hvc_open"
[linux/fpc-iii.git] / drivers / iommu / iommu.c
blob03d6a26687bc45ec6aac98ae5181f1a70573c454
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 */
6
7 #define pr_fmt(fmt) "iommu: " fmt
8
9 #include <linux/device.h>
10 #include <linux/kernel.h>
11 #include <linux/bug.h>
12 #include <linux/types.h>
13 #include <linux/init.h>
14 #include <linux/export.h>
15 #include <linux/slab.h>
16 #include <linux/errno.h>
17 #include <linux/iommu.h>
18 #include <linux/idr.h>
19 #include <linux/notifier.h>
20 #include <linux/err.h>
21 #include <linux/pci.h>
22 #include <linux/bitops.h>
23 #include <linux/property.h>
24 #include <linux/fsl/mc.h>
25 #include <linux/module.h>
26 #include <trace/events/iommu.h>
27
28 static struct kset *iommu_group_kset;
29 static DEFINE_IDA(iommu_group_ida);
30
31 static unsigned int iommu_def_domain_type __read_mostly;
32 static bool iommu_dma_strict __read_mostly = true;
33 static u32 iommu_cmd_line __read_mostly;
34
35 struct iommu_group {
36 struct kobject kobj;
37 struct kobject *devices_kobj;
38 struct list_head devices;
39 struct mutex mutex;
40 struct blocking_notifier_head notifier;
41 void *iommu_data;
42 void (*iommu_data_release)(void *iommu_data);
43 char *name;
44 int id;
45 struct iommu_domain *default_domain;
46 struct iommu_domain *domain;
47 };
48
49 struct group_device {
50 struct list_head list;
51 struct device *dev;
52 char *name;
53 };
54
55 struct iommu_group_attribute {
56 struct attribute attr;
57 ssize_t (*show)(struct iommu_group *group, char *buf);
58 ssize_t (*store)(struct iommu_group *group,
59 const char *buf, size_t count);
60 };
61
62 static const char * const iommu_group_resv_type_string[] = {
63 [IOMMU_RESV_DIRECT] = "direct",
64 [IOMMU_RESV_DIRECT_RELAXABLE] = "direct-relaxable",
65 [IOMMU_RESV_RESERVED] = "reserved",
66 [IOMMU_RESV_MSI] = "msi",
67 [IOMMU_RESV_SW_MSI] = "msi",
68 };
69
70 #define IOMMU_CMD_LINE_DMA_API BIT(0)
71
72 static void iommu_set_cmd_line_dma_api(void)
73 {
74 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
75 }
76
77 static bool iommu_cmd_line_dma_api(void)
78 {
79 return !!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API);
80 }
81
82 #define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
83 struct iommu_group_attribute iommu_group_attr_##_name = \
84 __ATTR(_name, _mode, _show, _store)
85
86 #define to_iommu_group_attr(_attr) \
87 container_of(_attr, struct iommu_group_attribute, attr)
88 #define to_iommu_group(_kobj) \
89 container_of(_kobj, struct iommu_group, kobj)
90
91 static LIST_HEAD(iommu_device_list);
92 static DEFINE_SPINLOCK(iommu_device_lock);
93
94 /*
95 * Use a function instead of an array here because the domain-type is a
96 * bit-field, so an array would waste memory.
97 */
98 static const char *iommu_domain_type_str(unsigned int t)
99 {
100 switch (t) {
101 case IOMMU_DOMAIN_BLOCKED:
102 return "Blocked";
103 case IOMMU_DOMAIN_IDENTITY:
104 return "Passthrough";
105 case IOMMU_DOMAIN_UNMANAGED:
106 return "Unmanaged";
107 case IOMMU_DOMAIN_DMA:
108 return "Translated";
109 default:
110 return "Unknown";
111 }
112 }
113
114 static int __init iommu_subsys_init(void)
115 {
116 bool cmd_line = iommu_cmd_line_dma_api();
117
118 if (!cmd_line) {
119 if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH))
120 iommu_set_default_passthrough(false);
121 else
122 iommu_set_default_translated(false);
123
124 if (iommu_default_passthrough() && mem_encrypt_active()) {
125 pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n");
126 iommu_set_default_translated(false);
127 }
128 }
129
130 pr_info("Default domain type: %s %s\n",
131 iommu_domain_type_str(iommu_def_domain_type),
132 cmd_line ? "(set via kernel command line)" : "");
133
134 return 0;
135 }
136 subsys_initcall(iommu_subsys_init);
137
138 int iommu_device_register(struct iommu_device *iommu)
139 {
140 spin_lock(&iommu_device_lock);
141 list_add_tail(&iommu->list, &iommu_device_list);
142 spin_unlock(&iommu_device_lock);
143 return 0;
144 }
145 EXPORT_SYMBOL_GPL(iommu_device_register);
146
147 void iommu_device_unregister(struct iommu_device *iommu)
148 {
149 spin_lock(&iommu_device_lock);
150 list_del(&iommu->list);
151 spin_unlock(&iommu_device_lock);
152 }
153 EXPORT_SYMBOL_GPL(iommu_device_unregister);
154
155 static struct dev_iommu *dev_iommu_get(struct device *dev)
156 {
157 struct dev_iommu *param = dev->iommu;
158
159 if (param)
160 return param;
161
162 param = kzalloc(sizeof(*param), GFP_KERNEL);
163 if (!param)
164 return NULL;
165
166 mutex_init(&param->lock);
167 dev->iommu = param;
168 return param;
169 }
170
171 static void dev_iommu_free(struct device *dev)
172 {
173 iommu_fwspec_free(dev);
174 kfree(dev->iommu);
175 dev->iommu = NULL;
176 }
177
178 int iommu_probe_device(struct device *dev)
179 {
180 const struct iommu_ops *ops = dev->bus->iommu_ops;
181 int ret;
182
183 WARN_ON(dev->iommu_group);
184 if (!ops)
185 return -EINVAL;
186
187 if (!dev_iommu_get(dev))
188 return -ENOMEM;
189
190 if (!try_module_get(ops->owner)) {
191 ret = -EINVAL;
192 goto err_free_dev_param;
193 }
194
195 ret = ops->add_device(dev);
196 if (ret)
197 goto err_module_put;
198
199 return 0;
200
201 err_module_put:
202 module_put(ops->owner);
203 err_free_dev_param:
204 dev_iommu_free(dev);
205 return ret;
206 }
207
208 void iommu_release_device(struct device *dev)
209 {
210 const struct iommu_ops *ops = dev->bus->iommu_ops;
211
212 if (dev->iommu_group)
213 ops->remove_device(dev);
214
215 if (dev->iommu) {
216 module_put(ops->owner);
217 dev_iommu_free(dev);
218 }
219 }
220
221 static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
222 unsigned type);
223 static int __iommu_attach_device(struct iommu_domain *domain,
224 struct device *dev);
225 static int __iommu_attach_group(struct iommu_domain *domain,
226 struct iommu_group *group);
227 static void __iommu_detach_group(struct iommu_domain *domain,
228 struct iommu_group *group);
229
230 static int __init iommu_set_def_domain_type(char *str)
231 {
232 bool pt;
233 int ret;
234
235 ret = kstrtobool(str, &pt);
236 if (ret)
237 return ret;
238
239 if (pt)
240 iommu_set_default_passthrough(true);
241 else
242 iommu_set_default_translated(true);
243
244 return 0;
245 }
246 early_param("iommu.passthrough", iommu_set_def_domain_type);
247
248 static int __init iommu_dma_setup(char *str)
249 {
250 return kstrtobool(str, &iommu_dma_strict);
251 }
252 early_param("iommu.strict", iommu_dma_setup);
253
254 static ssize_t iommu_group_attr_show(struct kobject *kobj,
255 struct attribute *__attr, char *buf)
256 {
257 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
258 struct iommu_group *group = to_iommu_group(kobj);
259 ssize_t ret = -EIO;
260
261 if (attr->show)
262 ret = attr->show(group, buf);
263 return ret;
264 }
265
266 static ssize_t iommu_group_attr_store(struct kobject *kobj,
267 struct attribute *__attr,
268 const char *buf, size_t count)
269 {
270 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
271 struct iommu_group *group = to_iommu_group(kobj);
272 ssize_t ret = -EIO;
273
274 if (attr->store)
275 ret = attr->store(group, buf, count);
276 return ret;
277 }
278
279 static const struct sysfs_ops iommu_group_sysfs_ops = {
280 .show = iommu_group_attr_show,
281 .store = iommu_group_attr_store,
282 };
283
284 static int iommu_group_create_file(struct iommu_group *group,
285 struct iommu_group_attribute *attr)
286 {
287 return sysfs_create_file(&group->kobj, &attr->attr);
288 }
289
290 static void iommu_group_remove_file(struct iommu_group *group,
291 struct iommu_group_attribute *attr)
292 {
293 sysfs_remove_file(&group->kobj, &attr->attr);
294 }
295
296 static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf)
297 {
298 return sprintf(buf, "%s\n", group->name);
299 }
300
301 /**
302 * iommu_insert_resv_region - Insert a new region in the
303 * list of reserved regions.
304 * @new: new region to insert
305 * @regions: list of regions
306 *
307 * Elements are sorted by start address and overlapping segments
308 * of the same type are merged.
309 */
310 int iommu_insert_resv_region(struct iommu_resv_region *new,
311 struct list_head *regions)
312 {
313 struct iommu_resv_region *iter, *tmp, *nr, *top;
314 LIST_HEAD(stack);
315
316 nr = iommu_alloc_resv_region(new->start, new->length,
317 new->prot, new->type);
318 if (!nr)
319 return -ENOMEM;
320
321 /* First add the new element based on start address sorting */
322 list_for_each_entry(iter, regions, list) {
323 if (nr->start < iter->start ||
324 (nr->start == iter->start && nr->type <= iter->type))
325 break;
326 }
327 list_add_tail(&nr->list, &iter->list);
328
329 /* Merge overlapping segments of type nr->type in @regions, if any */
330 list_for_each_entry_safe(iter, tmp, regions, list) {
331 phys_addr_t top_end, iter_end = iter->start + iter->length - 1;
332
333 /* no merge needed on elements of different types than @new */
334 if (iter->type != new->type) {
335 list_move_tail(&iter->list, &stack);
336 continue;
337 }
338
339 /* look for the last stack element of same type as @iter */
340 list_for_each_entry_reverse(top, &stack, list)
341 if (top->type == iter->type)
342 goto check_overlap;
343
344 list_move_tail(&iter->list, &stack);
345 continue;
346
347 check_overlap:
348 top_end = top->start + top->length - 1;
349
350 if (iter->start > top_end + 1) {
351 list_move_tail(&iter->list, &stack);
352 } else {
353 top->length = max(top_end, iter_end) - top->start + 1;
354 list_del(&iter->list);
355 kfree(iter);
356 }
357 }
358 list_splice(&stack, regions);
359 return 0;
360 }
361
362 static int
363 iommu_insert_device_resv_regions(struct list_head *dev_resv_regions,
364 struct list_head *group_resv_regions)
365 {
366 struct iommu_resv_region *entry;
367 int ret = 0;
368
369 list_for_each_entry(entry, dev_resv_regions, list) {
370 ret = iommu_insert_resv_region(entry, group_resv_regions);
371 if (ret)
372 break;
373 }
374 return ret;
375 }
376
377 int iommu_get_group_resv_regions(struct iommu_group *group,
378 struct list_head *head)
379 {
380 struct group_device *device;
381 int ret = 0;
382
383 mutex_lock(&group->mutex);
384 list_for_each_entry(device, &group->devices, list) {
385 struct list_head dev_resv_regions;
386
387 INIT_LIST_HEAD(&dev_resv_regions);
388 iommu_get_resv_regions(device->dev, &dev_resv_regions);
389 ret = iommu_insert_device_resv_regions(&dev_resv_regions, head);
390 iommu_put_resv_regions(device->dev, &dev_resv_regions);
391 if (ret)
392 break;
393 }
394 mutex_unlock(&group->mutex);
395 return ret;
396 }
397 EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions);
398
399 static ssize_t iommu_group_show_resv_regions(struct iommu_group *group,
400 char *buf)
401 {
402 struct iommu_resv_region *region, *next;
403 struct list_head group_resv_regions;
404 char *str = buf;
405
406 INIT_LIST_HEAD(&group_resv_regions);
407 iommu_get_group_resv_regions(group, &group_resv_regions);
408
409 list_for_each_entry_safe(region, next, &group_resv_regions, list) {
410 str += sprintf(str, "0x%016llx 0x%016llx %s\n",
411 (long long int)region->start,
412 (long long int)(region->start +
413 region->length - 1),
414 iommu_group_resv_type_string[region->type]);
415 kfree(region);
416 }
417
418 return (str - buf);
419 }
420
421 static ssize_t iommu_group_show_type(struct iommu_group *group,
422 char *buf)
423 {
424 char *type = "unknown\n";
425
426 if (group->default_domain) {
427 switch (group->default_domain->type) {
428 case IOMMU_DOMAIN_BLOCKED:
429 type = "blocked\n";
430 break;
431 case IOMMU_DOMAIN_IDENTITY:
432 type = "identity\n";
433 break;
434 case IOMMU_DOMAIN_UNMANAGED:
435 type = "unmanaged\n";
436 break;
437 case IOMMU_DOMAIN_DMA:
438 type = "DMA\n";
439 break;
440 }
441 }
442 strcpy(buf, type);
443
444 return strlen(type);
445 }
446
447 static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL);
448
449 static IOMMU_GROUP_ATTR(reserved_regions, 0444,
450 iommu_group_show_resv_regions, NULL);
451
452 static IOMMU_GROUP_ATTR(type, 0444, iommu_group_show_type, NULL);
453
454 static void iommu_group_release(struct kobject *kobj)
455 {
456 struct iommu_group *group = to_iommu_group(kobj);
457
458 pr_debug("Releasing group %d\n", group->id);
459
460 if (group->iommu_data_release)
461 group->iommu_data_release(group->iommu_data);
462
463 ida_simple_remove(&iommu_group_ida, group->id);
464
465 if (group->default_domain)
466 iommu_domain_free(group->default_domain);
467
468 kfree(group->name);
469 kfree(group);
470 }
471
472 static struct kobj_type iommu_group_ktype = {
473 .sysfs_ops = &iommu_group_sysfs_ops,
474 .release = iommu_group_release,
475 };
476
477 /**
478 * iommu_group_alloc - Allocate a new group
479 *
480 * This function is called by an iommu driver to allocate a new iommu
481 * group. The iommu group represents the minimum granularity of the iommu.
482 * Upon successful return, the caller holds a reference to the supplied
483 * group in order to hold the group until devices are added. Use
484 * iommu_group_put() to release this extra reference count, allowing the
485 * group to be automatically reclaimed once it has no devices or external
486 * references.
487 */
488 struct iommu_group *iommu_group_alloc(void)
489 {
490 struct iommu_group *group;
491 int ret;
492
493 group = kzalloc(sizeof(*group), GFP_KERNEL);
494 if (!group)
495 return ERR_PTR(-ENOMEM);
496
497 group->kobj.kset = iommu_group_kset;
498 mutex_init(&group->mutex);
499 INIT_LIST_HEAD(&group->devices);
500 BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
501
502 ret = ida_simple_get(&iommu_group_ida, 0, 0, GFP_KERNEL);
503 if (ret < 0) {
504 kfree(group);
505 return ERR_PTR(ret);
506 }
507 group->id = ret;
508
509 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype,
510 NULL, "%d", group->id);
511 if (ret) {
512 ida_simple_remove(&iommu_group_ida, group->id);
513 kobject_put(&group->kobj);
514 return ERR_PTR(ret);
515 }
516
517 group->devices_kobj = kobject_create_and_add("devices", &group->kobj);
518 if (!group->devices_kobj) {
519 kobject_put(&group->kobj); /* triggers .release & free */
520 return ERR_PTR(-ENOMEM);
521 }
522
523 /*
524 * The devices_kobj holds a reference on the group kobject, so
525 * as long as that exists so will the group. We can therefore
526 * use the devices_kobj for reference counting.
527 */
528 kobject_put(&group->kobj);
529
530 ret = iommu_group_create_file(group,
531 &iommu_group_attr_reserved_regions);
532 if (ret)
533 return ERR_PTR(ret);
534
535 ret = iommu_group_create_file(group, &iommu_group_attr_type);
536 if (ret)
537 return ERR_PTR(ret);
538
539 pr_debug("Allocated group %d\n", group->id);
540
541 return group;
542 }
543 EXPORT_SYMBOL_GPL(iommu_group_alloc);
544
545 struct iommu_group *iommu_group_get_by_id(int id)
546 {
547 struct kobject *group_kobj;
548 struct iommu_group *group;
549 const char *name;
550
551 if (!iommu_group_kset)
552 return NULL;
553
554 name = kasprintf(GFP_KERNEL, "%d", id);
555 if (!name)
556 return NULL;
557
558 group_kobj = kset_find_obj(iommu_group_kset, name);
559 kfree(name);
560
561 if (!group_kobj)
562 return NULL;
563
564 group = container_of(group_kobj, struct iommu_group, kobj);
565 BUG_ON(group->id != id);
566
567 kobject_get(group->devices_kobj);
568 kobject_put(&group->kobj);
569
570 return group;
571 }
572 EXPORT_SYMBOL_GPL(iommu_group_get_by_id);
573
574 /**
575 * iommu_group_get_iommudata - retrieve iommu_data registered for a group
576 * @group: the group
577 *
578 * iommu drivers can store data in the group for use when doing iommu
579 * operations. This function provides a way to retrieve it. Caller
580 * should hold a group reference.
581 */
582 void *iommu_group_get_iommudata(struct iommu_group *group)
583 {
584 return group->iommu_data;
585 }
586 EXPORT_SYMBOL_GPL(iommu_group_get_iommudata);
587
588 /**
589 * iommu_group_set_iommudata - set iommu_data for a group
590 * @group: the group
591 * @iommu_data: new data
592 * @release: release function for iommu_data
593 *
594 * iommu drivers can store data in the group for use when doing iommu
595 * operations. This function provides a way to set the data after
596 * the group has been allocated. Caller should hold a group reference.
597 */
598 void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data,
599 void (*release)(void *iommu_data))
600 {
601 group->iommu_data = iommu_data;
602 group->iommu_data_release = release;
603 }
604 EXPORT_SYMBOL_GPL(iommu_group_set_iommudata);
605
606 /**
607 * iommu_group_set_name - set name for a group
608 * @group: the group
609 * @name: name
610 *
611 * Allow iommu driver to set a name for a group. When set it will
612 * appear in a name attribute file under the group in sysfs.
613 */
614 int iommu_group_set_name(struct iommu_group *group, const char *name)
615 {
616 int ret;
617
618 if (group->name) {
619 iommu_group_remove_file(group, &iommu_group_attr_name);
620 kfree(group->name);
621 group->name = NULL;
622 if (!name)
623 return 0;
624 }
625
626 group->name = kstrdup(name, GFP_KERNEL);
627 if (!group->name)
628 return -ENOMEM;
629
630 ret = iommu_group_create_file(group, &iommu_group_attr_name);
631 if (ret) {
632 kfree(group->name);
633 group->name = NULL;
634 return ret;
635 }
636
637 return 0;
638 }
639 EXPORT_SYMBOL_GPL(iommu_group_set_name);
640
641 static int iommu_group_create_direct_mappings(struct iommu_group *group,
642 struct device *dev)
643 {
644 struct iommu_domain *domain = group->default_domain;
645 struct iommu_resv_region *entry;
646 struct list_head mappings;
647 unsigned long pg_size;
648 int ret = 0;
649
650 if (!domain || domain->type != IOMMU_DOMAIN_DMA)
651 return 0;
652
653 BUG_ON(!domain->pgsize_bitmap);
654
655 pg_size = 1UL << __ffs(domain->pgsize_bitmap);
656 INIT_LIST_HEAD(&mappings);
657
658 iommu_get_resv_regions(dev, &mappings);
659
660 /* We need to consider overlapping regions for different devices */
661 list_for_each_entry(entry, &mappings, list) {
662 dma_addr_t start, end, addr;
663
664 if (domain->ops->apply_resv_region)
665 domain->ops->apply_resv_region(dev, domain, entry);
666
667 start = ALIGN(entry->start, pg_size);
668 end = ALIGN(entry->start + entry->length, pg_size);
669
670 if (entry->type != IOMMU_RESV_DIRECT &&
671 entry->type != IOMMU_RESV_DIRECT_RELAXABLE)
672 continue;
673
674 for (addr = start; addr < end; addr += pg_size) {
675 phys_addr_t phys_addr;
676
677 phys_addr = iommu_iova_to_phys(domain, addr);
678 if (phys_addr)
679 continue;
680
681 ret = iommu_map(domain, addr, addr, pg_size, entry->prot);
682 if (ret)
683 goto out;
684 }
685
686 }
687
688 iommu_flush_tlb_all(domain);
689
690 out:
691 iommu_put_resv_regions(dev, &mappings);
692
693 return ret;
694 }
695
696 static bool iommu_is_attach_deferred(struct iommu_domain *domain,
697 struct device *dev)
698 {
699 if (domain->ops->is_attach_deferred)
700 return domain->ops->is_attach_deferred(domain, dev);
701
702 return false;
703 }
704
705 /**
706 * iommu_group_add_device - add a device to an iommu group
707 * @group: the group into which to add the device (reference should be held)
708 * @dev: the device
709 *
710 * This function is called by an iommu driver to add a device into a
711 * group. Adding a device increments the group reference count.
712 */
713 int iommu_group_add_device(struct iommu_group *group, struct device *dev)
714 {
715 int ret, i = 0;
716 struct group_device *device;
717
718 device = kzalloc(sizeof(*device), GFP_KERNEL);
719 if (!device)
720 return -ENOMEM;
721
722 device->dev = dev;
723
724 ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group");
725 if (ret)
726 goto err_free_device;
727
728 device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj));
729 rename:
730 if (!device->name) {
731 ret = -ENOMEM;
732 goto err_remove_link;
733 }
734
735 ret = sysfs_create_link_nowarn(group->devices_kobj,
736 &dev->kobj, device->name);
737 if (ret) {
738 if (ret == -EEXIST && i >= 0) {
739 /*
740 * Account for the slim chance of collision
741 * and append an instance to the name.
742 */
743 kfree(device->name);
744 device->name = kasprintf(GFP_KERNEL, "%s.%d",
745 kobject_name(&dev->kobj), i++);
746 goto rename;
747 }
748 goto err_free_name;
749 }
750
751 kobject_get(group->devices_kobj);
752
753 dev->iommu_group = group;
754
755 iommu_group_create_direct_mappings(group, dev);
756
757 mutex_lock(&group->mutex);
758 list_add_tail(&device->list, &group->devices);
759 if (group->domain && !iommu_is_attach_deferred(group->domain, dev))
760 ret = __iommu_attach_device(group->domain, dev);
761 mutex_unlock(&group->mutex);
762 if (ret)
763 goto err_put_group;
764
765 /* Notify any listeners about change to group. */
766 blocking_notifier_call_chain(&group->notifier,
767 IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev);
768
769 trace_add_device_to_group(group->id, dev);
770
771 dev_info(dev, "Adding to iommu group %d\n", group->id);
772
773 return 0;
774
775 err_put_group:
776 mutex_lock(&group->mutex);
777 list_del(&device->list);
778 mutex_unlock(&group->mutex);
779 dev->iommu_group = NULL;
780 kobject_put(group->devices_kobj);
781 sysfs_remove_link(group->devices_kobj, device->name);
782 err_free_name:
783 kfree(device->name);
784 err_remove_link:
785 sysfs_remove_link(&dev->kobj, "iommu_group");
786 err_free_device:
787 kfree(device);
788 dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret);
789 return ret;
790 }
791 EXPORT_SYMBOL_GPL(iommu_group_add_device);
792
793 /**
794 * iommu_group_remove_device - remove a device from it's current group
795 * @dev: device to be removed
796 *
797 * This function is called by an iommu driver to remove the device from
798 * it's current group. This decrements the iommu group reference count.
799 */
800 void iommu_group_remove_device(struct device *dev)
801 {
802 struct iommu_group *group = dev->iommu_group;
803 struct group_device *tmp_device, *device = NULL;
804
805 dev_info(dev, "Removing from iommu group %d\n", group->id);
806
807 /* Pre-notify listeners that a device is being removed. */
808 blocking_notifier_call_chain(&group->notifier,
809 IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev);
810
811 mutex_lock(&group->mutex);
812 list_for_each_entry(tmp_device, &group->devices, list) {
813 if (tmp_device->dev == dev) {
814 device = tmp_device;
815 list_del(&device->list);
816 break;
817 }
818 }
819 mutex_unlock(&group->mutex);
820
821 if (!device)
822 return;
823
824 sysfs_remove_link(group->devices_kobj, device->name);
825 sysfs_remove_link(&dev->kobj, "iommu_group");
826
827 trace_remove_device_from_group(group->id, dev);
828
829 kfree(device->name);
830 kfree(device);
831 dev->iommu_group = NULL;
832 kobject_put(group->devices_kobj);
833 }
834 EXPORT_SYMBOL_GPL(iommu_group_remove_device);
835
836 static int iommu_group_device_count(struct iommu_group *group)
837 {
838 struct group_device *entry;
839 int ret = 0;
840
841 list_for_each_entry(entry, &group->devices, list)
842 ret++;
843
844 return ret;
845 }
846
847 /**
848 * iommu_group_for_each_dev - iterate over each device in the group
849 * @group: the group
850 * @data: caller opaque data to be passed to callback function
851 * @fn: caller supplied callback function
852 *
853 * This function is called by group users to iterate over group devices.
854 * Callers should hold a reference count to the group during callback.
855 * The group->mutex is held across callbacks, which will block calls to
856 * iommu_group_add/remove_device.
857 */
858 static int __iommu_group_for_each_dev(struct iommu_group *group, void *data,
859 int (*fn)(struct device *, void *))
860 {
861 struct group_device *device;
862 int ret = 0;
863
864 list_for_each_entry(device, &group->devices, list) {
865 ret = fn(device->dev, data);
866 if (ret)
867 break;
868 }
869 return ret;
870 }
871
872
873 int iommu_group_for_each_dev(struct iommu_group *group, void *data,
874 int (*fn)(struct device *, void *))
875 {
876 int ret;
877
878 mutex_lock(&group->mutex);
879 ret = __iommu_group_for_each_dev(group, data, fn);
880 mutex_unlock(&group->mutex);
881
882 return ret;
883 }
884 EXPORT_SYMBOL_GPL(iommu_group_for_each_dev);
885
886 /**
887 * iommu_group_get - Return the group for a device and increment reference
888 * @dev: get the group that this device belongs to
889 *
890 * This function is called by iommu drivers and users to get the group
891 * for the specified device. If found, the group is returned and the group
892 * reference in incremented, else NULL.
893 */
894 struct iommu_group *iommu_group_get(struct device *dev)
895 {
896 struct iommu_group *group = dev->iommu_group;
897
898 if (group)
899 kobject_get(group->devices_kobj);
900
901 return group;
902 }
903 EXPORT_SYMBOL_GPL(iommu_group_get);
904
905 /**
906 * iommu_group_ref_get - Increment reference on a group
907 * @group: the group to use, must not be NULL
908 *
909 * This function is called by iommu drivers to take additional references on an
910 * existing group. Returns the given group for convenience.
911 */
912 struct iommu_group *iommu_group_ref_get(struct iommu_group *group)
913 {
914 kobject_get(group->devices_kobj);
915 return group;
916 }
917 EXPORT_SYMBOL_GPL(iommu_group_ref_get);
918
919 /**
920 * iommu_group_put - Decrement group reference
921 * @group: the group to use
922 *
923 * This function is called by iommu drivers and users to release the
924 * iommu group. Once the reference count is zero, the group is released.
925 */
926 void iommu_group_put(struct iommu_group *group)
927 {
928 if (group)
929 kobject_put(group->devices_kobj);
930 }
931 EXPORT_SYMBOL_GPL(iommu_group_put);
932
933 /**
934 * iommu_group_register_notifier - Register a notifier for group changes
935 * @group: the group to watch
936 * @nb: notifier block to signal
937 *
938 * This function allows iommu group users to track changes in a group.
939 * See include/linux/iommu.h for actions sent via this notifier. Caller
940 * should hold a reference to the group throughout notifier registration.
941 */
942 int iommu_group_register_notifier(struct iommu_group *group,
943 struct notifier_block *nb)
944 {
945 return blocking_notifier_chain_register(&group->notifier, nb);
946 }
947 EXPORT_SYMBOL_GPL(iommu_group_register_notifier);
948
949 /**
950 * iommu_group_unregister_notifier - Unregister a notifier
951 * @group: the group to watch
952 * @nb: notifier block to signal
953 *
954 * Unregister a previously registered group notifier block.
955 */
956 int iommu_group_unregister_notifier(struct iommu_group *group,
957 struct notifier_block *nb)
958 {
959 return blocking_notifier_chain_unregister(&group->notifier, nb);
960 }
961 EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier);
962
963 /**
964 * iommu_register_device_fault_handler() - Register a device fault handler
965 * @dev: the device
966 * @handler: the fault handler
967 * @data: private data passed as argument to the handler
968 *
969 * When an IOMMU fault event is received, this handler gets called with the
970 * fault event and data as argument. The handler should return 0 on success. If
971 * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also
972 * complete the fault by calling iommu_page_response() with one of the following
973 * response code:
974 * - IOMMU_PAGE_RESP_SUCCESS: retry the translation
975 * - IOMMU_PAGE_RESP_INVALID: terminate the fault
976 * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting
977 * page faults if possible.
978 *
979 * Return 0 if the fault handler was installed successfully, or an error.
980 */
981 int iommu_register_device_fault_handler(struct device *dev,
982 iommu_dev_fault_handler_t handler,
983 void *data)
984 {
985 struct dev_iommu *param = dev->iommu;
986 int ret = 0;
987
988 if (!param)
989 return -EINVAL;
990
991 mutex_lock(&param->lock);
992 /* Only allow one fault handler registered for each device */
993 if (param->fault_param) {
994 ret = -EBUSY;
995 goto done_unlock;
996 }
997
998 get_device(dev);
999 param->fault_param = kzalloc(sizeof(*param->fault_param), GFP_KERNEL);
1000 if (!param->fault_param) {
1001 put_device(dev);
1002 ret = -ENOMEM;
1003 goto done_unlock;
1004 }
1005 param->fault_param->handler = handler;
1006 param->fault_param->data = data;
1007 mutex_init(&param->fault_param->lock);
1008 INIT_LIST_HEAD(&param->fault_param->faults);
1009
1010 done_unlock:
1011 mutex_unlock(&param->lock);
1012
1013 return ret;
1014 }
1015 EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler);
1016
1017 /**
1018 * iommu_unregister_device_fault_handler() - Unregister the device fault handler
1019 * @dev: the device
1020 *
1021 * Remove the device fault handler installed with
1022 * iommu_register_device_fault_handler().
1023 *
1024 * Return 0 on success, or an error.
1025 */
1026 int iommu_unregister_device_fault_handler(struct device *dev)
1027 {
1028 struct dev_iommu *param = dev->iommu;
1029 int ret = 0;
1030
1031 if (!param)
1032 return -EINVAL;
1033
1034 mutex_lock(&param->lock);
1035
1036 if (!param->fault_param)
1037 goto unlock;
1038
1039 /* we cannot unregister handler if there are pending faults */
1040 if (!list_empty(&param->fault_param->faults)) {
1041 ret = -EBUSY;
1042 goto unlock;
1043 }
1044
1045 kfree(param->fault_param);
1046 param->fault_param = NULL;
1047 put_device(dev);
1048 unlock:
1049 mutex_unlock(&param->lock);
1050
1051 return ret;
1052 }
1053 EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler);
1054
1055 /**
1056 * iommu_report_device_fault() - Report fault event to device driver
1057 * @dev: the device
1058 * @evt: fault event data
1059 *
1060 * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ
1061 * handler. When this function fails and the fault is recoverable, it is the
1062 * caller's responsibility to complete the fault.
1063 *
1064 * Return 0 on success, or an error.
1065 */
1066 int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt)
1067 {
1068 struct dev_iommu *param = dev->iommu;
1069 struct iommu_fault_event *evt_pending = NULL;
1070 struct iommu_fault_param *fparam;
1071 int ret = 0;
1072
1073 if (!param || !evt)
1074 return -EINVAL;
1075
1076 /* we only report device fault if there is a handler registered */
1077 mutex_lock(&param->lock);
1078 fparam = param->fault_param;
1079 if (!fparam || !fparam->handler) {
1080 ret = -EINVAL;
1081 goto done_unlock;
1082 }
1083
1084 if (evt->fault.type == IOMMU_FAULT_PAGE_REQ &&
1085 (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) {
1086 evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event),
1087 GFP_KERNEL);
1088 if (!evt_pending) {
1089 ret = -ENOMEM;
1090 goto done_unlock;
1091 }
1092 mutex_lock(&fparam->lock);
1093 list_add_tail(&evt_pending->list, &fparam->faults);
1094 mutex_unlock(&fparam->lock);
1095 }
1096
1097 ret = fparam->handler(&evt->fault, fparam->data);
1098 if (ret && evt_pending) {
1099 mutex_lock(&fparam->lock);
1100 list_del(&evt_pending->list);
1101 mutex_unlock(&fparam->lock);
1102 kfree(evt_pending);
1103 }
1104 done_unlock:
1105 mutex_unlock(&param->lock);
1106 return ret;
1107 }
1108 EXPORT_SYMBOL_GPL(iommu_report_device_fault);
1109
1110 int iommu_page_response(struct device *dev,
1111 struct iommu_page_response *msg)
1112 {
1113 bool pasid_valid;
1114 int ret = -EINVAL;
1115 struct iommu_fault_event *evt;
1116 struct iommu_fault_page_request *prm;
1117 struct dev_iommu *param = dev->iommu;
1118 struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1119
1120 if (!domain || !domain->ops->page_response)
1121 return -ENODEV;
1122
1123 if (!param || !param->fault_param)
1124 return -EINVAL;
1125
1126 if (msg->version != IOMMU_PAGE_RESP_VERSION_1 ||
1127 msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID)
1128 return -EINVAL;
1129
1130 /* Only send response if there is a fault report pending */
1131 mutex_lock(&param->fault_param->lock);
1132 if (list_empty(&param->fault_param->faults)) {
1133 dev_warn_ratelimited(dev, "no pending PRQ, drop response\n");
1134 goto done_unlock;
1135 }
1136 /*
1137 * Check if we have a matching page request pending to respond,
1138 * otherwise return -EINVAL
1139 */
1140 list_for_each_entry(evt, &param->fault_param->faults, list) {
1141 prm = &evt->fault.prm;
1142 pasid_valid = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
1143
1144 if ((pasid_valid && prm->pasid != msg->pasid) ||
1145 prm->grpid != msg->grpid)
1146 continue;
1147
1148 /* Sanitize the reply */
1149 msg->flags = pasid_valid ? IOMMU_PAGE_RESP_PASID_VALID : 0;
1150
1151 ret = domain->ops->page_response(dev, evt, msg);
1152 list_del(&evt->list);
1153 kfree(evt);
1154 break;
1155 }
1156
1157 done_unlock:
1158 mutex_unlock(&param->fault_param->lock);
1159 return ret;
1160 }
1161 EXPORT_SYMBOL_GPL(iommu_page_response);
1162
1163 /**
1164 * iommu_group_id - Return ID for a group
1165 * @group: the group to ID
1166 *
1167 * Return the unique ID for the group matching the sysfs group number.
1168 */
1169 int iommu_group_id(struct iommu_group *group)
1170 {
1171 return group->id;
1172 }
1173 EXPORT_SYMBOL_GPL(iommu_group_id);
1174
1175 static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
1176 unsigned long *devfns);
1177
1178 /*
1179 * To consider a PCI device isolated, we require ACS to support Source
1180 * Validation, Request Redirection, Completer Redirection, and Upstream
1181 * Forwarding. This effectively means that devices cannot spoof their
1182 * requester ID, requests and completions cannot be redirected, and all
1183 * transactions are forwarded upstream, even as it passes through a
1184 * bridge where the target device is downstream.
1185 */
1186 #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
1187
1188 /*
1189 * For multifunction devices which are not isolated from each other, find
1190 * all the other non-isolated functions and look for existing groups. For
1191 * each function, we also need to look for aliases to or from other devices
1192 * that may already have a group.
1193 */
1194 static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev,
1195 unsigned long *devfns)
1196 {
1197 struct pci_dev *tmp = NULL;
1198 struct iommu_group *group;
1199
1200 if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))
1201 return NULL;
1202
1203 for_each_pci_dev(tmp) {
1204 if (tmp == pdev || tmp->bus != pdev->bus ||
1205 PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
1206 pci_acs_enabled(tmp, REQ_ACS_FLAGS))
1207 continue;
1208
1209 group = get_pci_alias_group(tmp, devfns);
1210 if (group) {
1211 pci_dev_put(tmp);
1212 return group;
1213 }
1214 }
1215
1216 return NULL;
1217 }
1218
1219 /*
1220 * Look for aliases to or from the given device for existing groups. DMA
1221 * aliases are only supported on the same bus, therefore the search
1222 * space is quite small (especially since we're really only looking at pcie
1223 * device, and therefore only expect multiple slots on the root complex or
1224 * downstream switch ports). It's conceivable though that a pair of
1225 * multifunction devices could have aliases between them that would cause a
1226 * loop. To prevent this, we use a bitmap to track where we've been.
1227 */
1228 static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
1229 unsigned long *devfns)
1230 {
1231 struct pci_dev *tmp = NULL;
1232 struct iommu_group *group;
1233
1234 if (test_and_set_bit(pdev->devfn & 0xff, devfns))
1235 return NULL;
1236
1237 group = iommu_group_get(&pdev->dev);
1238 if (group)
1239 return group;
1240
1241 for_each_pci_dev(tmp) {
1242 if (tmp == pdev || tmp->bus != pdev->bus)
1243 continue;
1244
1245 /* We alias them or they alias us */
1246 if (pci_devs_are_dma_aliases(pdev, tmp)) {
1247 group = get_pci_alias_group(tmp, devfns);
1248 if (group) {
1249 pci_dev_put(tmp);
1250 return group;
1251 }
1252
1253 group = get_pci_function_alias_group(tmp, devfns);
1254 if (group) {
1255 pci_dev_put(tmp);
1256 return group;
1257 }
1258 }
1259 }
1260
1261 return NULL;
1262 }
1263
1264 struct group_for_pci_data {
1265 struct pci_dev *pdev;
1266 struct iommu_group *group;
1267 };
1268
1269 /*
1270 * DMA alias iterator callback, return the last seen device. Stop and return
1271 * the IOMMU group if we find one along the way.
1272 */
1273 static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque)
1274 {
1275 struct group_for_pci_data *data = opaque;
1276
1277 data->pdev = pdev;
1278 data->group = iommu_group_get(&pdev->dev);
1279
1280 return data->group != NULL;
1281 }
1282
1283 /*
1284 * Generic device_group call-back function. It just allocates one
1285 * iommu-group per device.
1286 */
1287 struct iommu_group *generic_device_group(struct device *dev)
1288 {
1289 return iommu_group_alloc();
1290 }
1291 EXPORT_SYMBOL_GPL(generic_device_group);
1292
1293 /*
1294 * Use standard PCI bus topology, isolation features, and DMA alias quirks
1295 * to find or create an IOMMU group for a device.
1296 */
1297 struct iommu_group *pci_device_group(struct device *dev)
1298 {
1299 struct pci_dev *pdev = to_pci_dev(dev);
1300 struct group_for_pci_data data;
1301 struct pci_bus *bus;
1302 struct iommu_group *group = NULL;
1303 u64 devfns[4] = { 0 };
1304
1305 if (WARN_ON(!dev_is_pci(dev)))
1306 return ERR_PTR(-EINVAL);
1307
1308 /*
1309 * Find the upstream DMA alias for the device. A device must not
1310 * be aliased due to topology in order to have its own IOMMU group.
1311 * If we find an alias along the way that already belongs to a
1312 * group, use it.
1313 */
1314 if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data))
1315 return data.group;
1316
1317 pdev = data.pdev;
1318
1319 /*
1320 * Continue upstream from the point of minimum IOMMU granularity
1321 * due to aliases to the point where devices are protected from
1322 * peer-to-peer DMA by PCI ACS. Again, if we find an existing
1323 * group, use it.
1324 */
1325 for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) {
1326 if (!bus->self)
1327 continue;
1328
1329 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
1330 break;
1331
1332 pdev = bus->self;
1333
1334 group = iommu_group_get(&pdev->dev);
1335 if (group)
1336 return group;
1337 }
1338
1339 /*
1340 * Look for existing groups on device aliases. If we alias another
1341 * device or another device aliases us, use the same group.
1342 */
1343 group = get_pci_alias_group(pdev, (unsigned long *)devfns);
1344 if (group)
1345 return group;
1346
1347 /*
1348 * Look for existing groups on non-isolated functions on the same
1349 * slot and aliases of those funcions, if any. No need to clear
1350 * the search bitmap, the tested devfns are still valid.
1351 */
1352 group = get_pci_function_alias_group(pdev, (unsigned long *)devfns);
1353 if (group)
1354 return group;
1355
1356 /* No shared group found, allocate new */
1357 return iommu_group_alloc();
1358 }
1359 EXPORT_SYMBOL_GPL(pci_device_group);
1360
1361 /* Get the IOMMU group for device on fsl-mc bus */
1362 struct iommu_group *fsl_mc_device_group(struct device *dev)
1363 {
1364 struct device *cont_dev = fsl_mc_cont_dev(dev);
1365 struct iommu_group *group;
1366
1367 group = iommu_group_get(cont_dev);
1368 if (!group)
1369 group = iommu_group_alloc();
1370 return group;
1371 }
1372 EXPORT_SYMBOL_GPL(fsl_mc_device_group);
1373
1374 /**
1375 * iommu_group_get_for_dev - Find or create the IOMMU group for a device
1376 * @dev: target device
1377 *
1378 * This function is intended to be called by IOMMU drivers and extended to
1379 * support common, bus-defined algorithms when determining or creating the
1380 * IOMMU group for a device. On success, the caller will hold a reference
1381 * to the returned IOMMU group, which will already include the provided
1382 * device. The reference should be released with iommu_group_put().
1383 */
1384 struct iommu_group *iommu_group_get_for_dev(struct device *dev)
1385 {
1386 const struct iommu_ops *ops = dev->bus->iommu_ops;
1387 struct iommu_group *group;
1388 int ret;
1389
1390 group = iommu_group_get(dev);
1391 if (group)
1392 return group;
1393
1394 if (!ops)
1395 return ERR_PTR(-EINVAL);
1396
1397 group = ops->device_group(dev);
1398 if (WARN_ON_ONCE(group == NULL))
1399 return ERR_PTR(-EINVAL);
1400
1401 if (IS_ERR(group))
1402 return group;
1403
1404 /*
1405 * Try to allocate a default domain - needs support from the
1406 * IOMMU driver.
1407 */
1408 if (!group->default_domain) {
1409 struct iommu_domain *dom;
1410
1411 dom = __iommu_domain_alloc(dev->bus, iommu_def_domain_type);
1412 if (!dom && iommu_def_domain_type != IOMMU_DOMAIN_DMA) {
1413 dom = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_DMA);
1414 if (dom) {
1415 dev_warn(dev,
1416 "failed to allocate default IOMMU domain of type %u; falling back to IOMMU_DOMAIN_DMA",
1417 iommu_def_domain_type);
1418 }
1419 }
1420
1421 group->default_domain = dom;
1422 if (!group->domain)
1423 group->domain = dom;
1424
1425 if (dom && !iommu_dma_strict) {
1426 int attr = 1;
1427 iommu_domain_set_attr(dom,
1428 DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE,
1429 &attr);
1430 }
1431 }
1432
1433 ret = iommu_group_add_device(group, dev);
1434 if (ret) {
1435 iommu_group_put(group);
1436 return ERR_PTR(ret);
1437 }
1438
1439 return group;
1440 }
1441 EXPORT_SYMBOL_GPL(iommu_group_get_for_dev);
1442
1443 struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)
1444 {
1445 return group->default_domain;
1446 }
1447
1448 static int add_iommu_group(struct device *dev, void *data)
1449 {
1450 int ret = iommu_probe_device(dev);
1451
1452 /*
1453 * We ignore -ENODEV errors for now, as they just mean that the
1454 * device is not translated by an IOMMU. We still care about
1455 * other errors and fail to initialize when they happen.
1456 */
1457 if (ret == -ENODEV)
1458 ret = 0;
1459
1460 return ret;
1461 }
1462
1463 static int remove_iommu_group(struct device *dev, void *data)
1464 {
1465 iommu_release_device(dev);
1466
1467 return 0;
1468 }
1469
1470 static int iommu_bus_notifier(struct notifier_block *nb,
1471 unsigned long action, void *data)
1472 {
1473 unsigned long group_action = 0;
1474 struct device *dev = data;
1475 struct iommu_group *group;
1476
1477 /*
1478 * ADD/DEL call into iommu driver ops if provided, which may
1479 * result in ADD/DEL notifiers to group->notifier
1480 */
1481 if (action == BUS_NOTIFY_ADD_DEVICE) {
1482 int ret;
1483
1484 ret = iommu_probe_device(dev);
1485 return (ret) ? NOTIFY_DONE : NOTIFY_OK;
1486 } else if (action == BUS_NOTIFY_REMOVED_DEVICE) {
1487 iommu_release_device(dev);
1488 return NOTIFY_OK;
1489 }
1490
1491 /*
1492 * Remaining BUS_NOTIFYs get filtered and republished to the
1493 * group, if anyone is listening
1494 */
1495 group = iommu_group_get(dev);
1496 if (!group)
1497 return 0;
1498
1499 switch (action) {
1500 case BUS_NOTIFY_BIND_DRIVER:
1501 group_action = IOMMU_GROUP_NOTIFY_BIND_DRIVER;
1502 break;
1503 case BUS_NOTIFY_BOUND_DRIVER:
1504 group_action = IOMMU_GROUP_NOTIFY_BOUND_DRIVER;
1505 break;
1506 case BUS_NOTIFY_UNBIND_DRIVER:
1507 group_action = IOMMU_GROUP_NOTIFY_UNBIND_DRIVER;
1508 break;
1509 case BUS_NOTIFY_UNBOUND_DRIVER:
1510 group_action = IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER;
1511 break;
1512 }
1513
1514 if (group_action)
1515 blocking_notifier_call_chain(&group->notifier,
1516 group_action, dev);
1517
1518 iommu_group_put(group);
1519 return 0;
1520 }
1521
1522 static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
1523 {
1524 int err;
1525 struct notifier_block *nb;
1526
1527 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
1528 if (!nb)
1529 return -ENOMEM;
1530
1531 nb->notifier_call = iommu_bus_notifier;
1532
1533 err = bus_register_notifier(bus, nb);
1534 if (err)
1535 goto out_free;
1536
1537 err = bus_for_each_dev(bus, NULL, NULL, add_iommu_group);
1538 if (err)
1539 goto out_err;
1540
1541
1542 return 0;
1543
1544 out_err:
1545 /* Clean up */
1546 bus_for_each_dev(bus, NULL, NULL, remove_iommu_group);
1547 bus_unregister_notifier(bus, nb);
1548
1549 out_free:
1550 kfree(nb);
1551
1552 return err;
1553 }
1554
1555 /**
1556 * bus_set_iommu - set iommu-callbacks for the bus
1557 * @bus: bus.
1558 * @ops: the callbacks provided by the iommu-driver
1559 *
1560 * This function is called by an iommu driver to set the iommu methods
1561 * used for a particular bus. Drivers for devices on that bus can use
1562 * the iommu-api after these ops are registered.
1563 * This special function is needed because IOMMUs are usually devices on
1564 * the bus itself, so the iommu drivers are not initialized when the bus
1565 * is set up. With this function the iommu-driver can set the iommu-ops
1566 * afterwards.
1567 */
1568 int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops)
1569 {
1570 int err;
1571
1572 if (ops == NULL) {
1573 bus->iommu_ops = NULL;
1574 return 0;
1575 }
1576
1577 if (bus->iommu_ops != NULL)
1578 return -EBUSY;
1579
1580 bus->iommu_ops = ops;
1581
1582 /* Do IOMMU specific setup for this bus-type */
1583 err = iommu_bus_init(bus, ops);
1584 if (err)
1585 bus->iommu_ops = NULL;
1586
1587 return err;
1588 }
1589 EXPORT_SYMBOL_GPL(bus_set_iommu);
1590
1591 bool iommu_present(struct bus_type *bus)
1592 {
1593 return bus->iommu_ops != NULL;
1594 }
1595 EXPORT_SYMBOL_GPL(iommu_present);
1596
1597 bool iommu_capable(struct bus_type *bus, enum iommu_cap cap)
1598 {
1599 if (!bus->iommu_ops || !bus->iommu_ops->capable)
1600 return false;
1601
1602 return bus->iommu_ops->capable(cap);
1603 }
1604 EXPORT_SYMBOL_GPL(iommu_capable);
1605
1606 /**
1607 * iommu_set_fault_handler() - set a fault handler for an iommu domain
1608 * @domain: iommu domain
1609 * @handler: fault handler
1610 * @token: user data, will be passed back to the fault handler
1611 *
1612 * This function should be used by IOMMU users which want to be notified
1613 * whenever an IOMMU fault happens.
1614 *
1615 * The fault handler itself should return 0 on success, and an appropriate
1616 * error code otherwise.
1617 */
1618 void iommu_set_fault_handler(struct iommu_domain *domain,
1619 iommu_fault_handler_t handler,
1620 void *token)
1621 {
1622 BUG_ON(!domain);
1623
1624 domain->handler = handler;
1625 domain->handler_token = token;
1626 }
1627 EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
1628
1629 static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
1630 unsigned type)
1631 {
1632 struct iommu_domain *domain;
1633
1634 if (bus == NULL || bus->iommu_ops == NULL)
1635 return NULL;
1636
1637 domain = bus->iommu_ops->domain_alloc(type);
1638 if (!domain)
1639 return NULL;
1640
1641 domain->ops = bus->iommu_ops;
1642 domain->type = type;
1643 /* Assume all sizes by default; the driver may override this later */
1644 domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap;
1645
1646 return domain;
1647 }
1648
1649 struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
1650 {
1651 return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED);
1652 }
1653 EXPORT_SYMBOL_GPL(iommu_domain_alloc);
1654
1655 void iommu_domain_free(struct iommu_domain *domain)
1656 {
1657 domain->ops->domain_free(domain);
1658 }
1659 EXPORT_SYMBOL_GPL(iommu_domain_free);
1660
1661 static int __iommu_attach_device(struct iommu_domain *domain,
1662 struct device *dev)
1663 {
1664 int ret;
1665
1666 if (unlikely(domain->ops->attach_dev == NULL))
1667 return -ENODEV;
1668
1669 ret = domain->ops->attach_dev(domain, dev);
1670 if (!ret)
1671 trace_attach_device_to_domain(dev);
1672 return ret;
1673 }
1674
1675 int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
1676 {
1677 struct iommu_group *group;
1678 int ret;
1679
1680 group = iommu_group_get(dev);
1681 if (!group)
1682 return -ENODEV;
1683
1684 /*
1685 * Lock the group to make sure the device-count doesn't
1686 * change while we are attaching
1687 */
1688 mutex_lock(&group->mutex);
1689 ret = -EINVAL;
1690 if (iommu_group_device_count(group) != 1)
1691 goto out_unlock;
1692
1693 ret = __iommu_attach_group(domain, group);
1694
1695 out_unlock:
1696 mutex_unlock(&group->mutex);
1697 iommu_group_put(group);
1698
1699 return ret;
1700 }
1701 EXPORT_SYMBOL_GPL(iommu_attach_device);
1702
1703 int iommu_cache_invalidate(struct iommu_domain *domain, struct device *dev,
1704 struct iommu_cache_invalidate_info *inv_info)
1705 {
1706 if (unlikely(!domain->ops->cache_invalidate))
1707 return -ENODEV;
1708
1709 return domain->ops->cache_invalidate(domain, dev, inv_info);
1710 }
1711 EXPORT_SYMBOL_GPL(iommu_cache_invalidate);
1712
1713 int iommu_sva_bind_gpasid(struct iommu_domain *domain,
1714 struct device *dev, struct iommu_gpasid_bind_data *data)
1715 {
1716 if (unlikely(!domain->ops->sva_bind_gpasid))
1717 return -ENODEV;
1718
1719 return domain->ops->sva_bind_gpasid(domain, dev, data);
1720 }
1721 EXPORT_SYMBOL_GPL(iommu_sva_bind_gpasid);
1722
1723 int iommu_sva_unbind_gpasid(struct iommu_domain *domain, struct device *dev,
1724 ioasid_t pasid)
1725 {
1726 if (unlikely(!domain->ops->sva_unbind_gpasid))
1727 return -ENODEV;
1728
1729 return domain->ops->sva_unbind_gpasid(dev, pasid);
1730 }
1731 EXPORT_SYMBOL_GPL(iommu_sva_unbind_gpasid);
1732
1733 static void __iommu_detach_device(struct iommu_domain *domain,
1734 struct device *dev)
1735 {
1736 if (iommu_is_attach_deferred(domain, dev))
1737 return;
1738
1739 if (unlikely(domain->ops->detach_dev == NULL))
1740 return;
1741
1742 domain->ops->detach_dev(domain, dev);
1743 trace_detach_device_from_domain(dev);
1744 }
1745
1746 void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
1747 {
1748 struct iommu_group *group;
1749
1750 group = iommu_group_get(dev);
1751 if (!group)
1752 return;
1753
1754 mutex_lock(&group->mutex);
1755 if (iommu_group_device_count(group) != 1) {
1756 WARN_ON(1);
1757 goto out_unlock;
1758 }
1759
1760 __iommu_detach_group(domain, group);
1761
1762 out_unlock:
1763 mutex_unlock(&group->mutex);
1764 iommu_group_put(group);
1765 }
1766 EXPORT_SYMBOL_GPL(iommu_detach_device);
1767
1768 struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
1769 {
1770 struct iommu_domain *domain;
1771 struct iommu_group *group;
1772
1773 group = iommu_group_get(dev);
1774 if (!group)
1775 return NULL;
1776
1777 domain = group->domain;
1778
1779 iommu_group_put(group);
1780
1781 return domain;
1782 }
1783 EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
1784
1785 /*
1786 * For IOMMU_DOMAIN_DMA implementations which already provide their own
1787 * guarantees that the group and its default domain are valid and correct.
1788 */
1789 struct iommu_domain *iommu_get_dma_domain(struct device *dev)
1790 {
1791 return dev->iommu_group->default_domain;
1792 }
1793
1794 /*
1795 * IOMMU groups are really the natural working unit of the IOMMU, but
1796 * the IOMMU API works on domains and devices. Bridge that gap by
1797 * iterating over the devices in a group. Ideally we'd have a single
1798 * device which represents the requestor ID of the group, but we also
1799 * allow IOMMU drivers to create policy defined minimum sets, where
1800 * the physical hardware may be able to distiguish members, but we
1801 * wish to group them at a higher level (ex. untrusted multi-function
1802 * PCI devices). Thus we attach each device.
1803 */
1804 static int iommu_group_do_attach_device(struct device *dev, void *data)
1805 {
1806 struct iommu_domain *domain = data;
1807
1808 return __iommu_attach_device(domain, dev);
1809 }
1810
1811 static int __iommu_attach_group(struct iommu_domain *domain,
1812 struct iommu_group *group)
1813 {
1814 int ret;
1815
1816 if (group->default_domain && group->domain != group->default_domain)
1817 return -EBUSY;
1818
1819 ret = __iommu_group_for_each_dev(group, domain,
1820 iommu_group_do_attach_device);
1821 if (ret == 0)
1822 group->domain = domain;
1823
1824 return ret;
1825 }
1826
1827 int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
1828 {
1829 int ret;
1830
1831 mutex_lock(&group->mutex);
1832 ret = __iommu_attach_group(domain, group);
1833 mutex_unlock(&group->mutex);
1834
1835 return ret;
1836 }
1837 EXPORT_SYMBOL_GPL(iommu_attach_group);
1838
1839 static int iommu_group_do_detach_device(struct device *dev, void *data)
1840 {
1841 struct iommu_domain *domain = data;
1842
1843 __iommu_detach_device(domain, dev);
1844
1845 return 0;
1846 }
1847
1848 static void __iommu_detach_group(struct iommu_domain *domain,
1849 struct iommu_group *group)
1850 {
1851 int ret;
1852
1853 if (!group->default_domain) {
1854 __iommu_group_for_each_dev(group, domain,
1855 iommu_group_do_detach_device);
1856 group->domain = NULL;
1857 return;
1858 }
1859
1860 if (group->domain == group->default_domain)
1861 return;
1862
1863 /* Detach by re-attaching to the default domain */
1864 ret = __iommu_group_for_each_dev(group, group->default_domain,
1865 iommu_group_do_attach_device);
1866 if (ret != 0)
1867 WARN_ON(1);
1868 else
1869 group->domain = group->default_domain;
1870 }
1871
1872 void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
1873 {
1874 mutex_lock(&group->mutex);
1875 __iommu_detach_group(domain, group);
1876 mutex_unlock(&group->mutex);
1877 }
1878 EXPORT_SYMBOL_GPL(iommu_detach_group);
1879
1880 phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
1881 {
1882 if (unlikely(domain->ops->iova_to_phys == NULL))
1883 return 0;
1884
1885 return domain->ops->iova_to_phys(domain, iova);
1886 }
1887 EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
1888
1889 static size_t iommu_pgsize(struct iommu_domain *domain,
1890 unsigned long addr_merge, size_t size)
1891 {
1892 unsigned int pgsize_idx;
1893 size_t pgsize;
1894
1895 /* Max page size that still fits into 'size' */
1896 pgsize_idx = __fls(size);
1897
1898 /* need to consider alignment requirements ? */
1899 if (likely(addr_merge)) {
1900 /* Max page size allowed by address */
1901 unsigned int align_pgsize_idx = __ffs(addr_merge);
1902 pgsize_idx = min(pgsize_idx, align_pgsize_idx);
1903 }
1904
1905 /* build a mask of acceptable page sizes */
1906 pgsize = (1UL << (pgsize_idx + 1)) - 1;
1907
1908 /* throw away page sizes not supported by the hardware */
1909 pgsize &= domain->pgsize_bitmap;
1910
1911 /* make sure we're still sane */
1912 BUG_ON(!pgsize);
1913
1914 /* pick the biggest page */
1915 pgsize_idx = __fls(pgsize);
1916 pgsize = 1UL << pgsize_idx;
1917
1918 return pgsize;
1919 }
1920
1921 int __iommu_map(struct iommu_domain *domain, unsigned long iova,
1922 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
1923 {
1924 const struct iommu_ops *ops = domain->ops;
1925 unsigned long orig_iova = iova;
1926 unsigned int min_pagesz;
1927 size_t orig_size = size;
1928 phys_addr_t orig_paddr = paddr;
1929 int ret = 0;
1930
1931 if (unlikely(ops->map == NULL ||
1932 domain->pgsize_bitmap == 0UL))
1933 return -ENODEV;
1934
1935 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
1936 return -EINVAL;
1937
1938 /* find out the minimum page size supported */
1939 min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
1940
1941 /*
1942 * both the virtual address and the physical one, as well as
1943 * the size of the mapping, must be aligned (at least) to the
1944 * size of the smallest page supported by the hardware
1945 */
1946 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
1947 pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
1948 iova, &paddr, size, min_pagesz);
1949 return -EINVAL;
1950 }
1951
1952 pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
1953
1954 while (size) {
1955 size_t pgsize = iommu_pgsize(domain, iova | paddr, size);
1956
1957 pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
1958 iova, &paddr, pgsize);
1959 ret = ops->map(domain, iova, paddr, pgsize, prot, gfp);
1960
1961 if (ret)
1962 break;
1963
1964 iova += pgsize;
1965 paddr += pgsize;
1966 size -= pgsize;
1967 }
1968
1969 if (ops->iotlb_sync_map)
1970 ops->iotlb_sync_map(domain);
1971
1972 /* unroll mapping in case something went wrong */
1973 if (ret)
1974 iommu_unmap(domain, orig_iova, orig_size - size);
1975 else
1976 trace_map(orig_iova, orig_paddr, orig_size);
1977
1978 return ret;
1979 }
1980
1981 int iommu_map(struct iommu_domain *domain, unsigned long iova,
1982 phys_addr_t paddr, size_t size, int prot)
1983 {
1984 might_sleep();
1985 return __iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL);
1986 }
1987 EXPORT_SYMBOL_GPL(iommu_map);
1988
1989 int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova,
1990 phys_addr_t paddr, size_t size, int prot)
1991 {
1992 return __iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC);
1993 }
1994 EXPORT_SYMBOL_GPL(iommu_map_atomic);
1995
1996 static size_t __iommu_unmap(struct iommu_domain *domain,
1997 unsigned long iova, size_t size,
1998 struct iommu_iotlb_gather *iotlb_gather)
1999 {
2000 const struct iommu_ops *ops = domain->ops;
2001 size_t unmapped_page, unmapped = 0;
2002 unsigned long orig_iova = iova;
2003 unsigned int min_pagesz;
2004
2005 if (unlikely(ops->unmap == NULL ||
2006 domain->pgsize_bitmap == 0UL))
2007 return 0;
2008
2009 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
2010 return 0;
2011
2012 /* find out the minimum page size supported */
2013 min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
2014
2015 /*
2016 * The virtual address, as well as the size of the mapping, must be
2017 * aligned (at least) to the size of the smallest page supported
2018 * by the hardware
2019 */
2020 if (!IS_ALIGNED(iova | size, min_pagesz)) {
2021 pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
2022 iova, size, min_pagesz);
2023 return 0;
2024 }
2025
2026 pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);
2027
2028 /*
2029 * Keep iterating until we either unmap 'size' bytes (or more)
2030 * or we hit an area that isn't mapped.
2031 */
2032 while (unmapped < size) {
2033 size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
2034
2035 unmapped_page = ops->unmap(domain, iova, pgsize, iotlb_gather);
2036 if (!unmapped_page)
2037 break;
2038
2039 pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
2040 iova, unmapped_page);
2041
2042 iova += unmapped_page;
2043 unmapped += unmapped_page;
2044 }
2045
2046 trace_unmap(orig_iova, size, unmapped);
2047 return unmapped;
2048 }
2049
2050 size_t iommu_unmap(struct iommu_domain *domain,
2051 unsigned long iova, size_t size)
2052 {
2053 struct iommu_iotlb_gather iotlb_gather;
2054 size_t ret;
2055
2056 iommu_iotlb_gather_init(&iotlb_gather);
2057 ret = __iommu_unmap(domain, iova, size, &iotlb_gather);
2058 iommu_tlb_sync(domain, &iotlb_gather);
2059
2060 return ret;
2061 }
2062 EXPORT_SYMBOL_GPL(iommu_unmap);
2063
2064 size_t iommu_unmap_fast(struct iommu_domain *domain,
2065 unsigned long iova, size_t size,
2066 struct iommu_iotlb_gather *iotlb_gather)
2067 {
2068 return __iommu_unmap(domain, iova, size, iotlb_gather);
2069 }
2070 EXPORT_SYMBOL_GPL(iommu_unmap_fast);
2071
2072 size_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
2073 struct scatterlist *sg, unsigned int nents, int prot,
2074 gfp_t gfp)
2075 {
2076 size_t len = 0, mapped = 0;
2077 phys_addr_t start;
2078 unsigned int i = 0;
2079 int ret;
2080
2081 while (i <= nents) {
2082 phys_addr_t s_phys = sg_phys(sg);
2083
2084 if (len && s_phys != start + len) {
2085 ret = __iommu_map(domain, iova + mapped, start,
2086 len, prot, gfp);
2087
2088 if (ret)
2089 goto out_err;
2090
2091 mapped += len;
2092 len = 0;
2093 }
2094
2095 if (len) {
2096 len += sg->length;
2097 } else {
2098 len = sg->length;
2099 start = s_phys;
2100 }
2101
2102 if (++i < nents)
2103 sg = sg_next(sg);
2104 }
2105
2106 return mapped;
2107
2108 out_err:
2109 /* undo mappings already done */
2110 iommu_unmap(domain, iova, mapped);
2111
2112 return 0;
2113
2114 }
2115
2116 size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
2117 struct scatterlist *sg, unsigned int nents, int prot)
2118 {
2119 might_sleep();
2120 return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_KERNEL);
2121 }
2122 EXPORT_SYMBOL_GPL(iommu_map_sg);
2123
2124 size_t iommu_map_sg_atomic(struct iommu_domain *domain, unsigned long iova,
2125 struct scatterlist *sg, unsigned int nents, int prot)
2126 {
2127 return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC);
2128 }
2129 EXPORT_SYMBOL_GPL(iommu_map_sg_atomic);
2130
2131 int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
2132 phys_addr_t paddr, u64 size, int prot)
2133 {
2134 if (unlikely(domain->ops->domain_window_enable == NULL))
2135 return -ENODEV;
2136
2137 return domain->ops->domain_window_enable(domain, wnd_nr, paddr, size,
2138 prot);
2139 }
2140 EXPORT_SYMBOL_GPL(iommu_domain_window_enable);
2141
2142 void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr)
2143 {
2144 if (unlikely(domain->ops->domain_window_disable == NULL))
2145 return;
2146
2147 return domain->ops->domain_window_disable(domain, wnd_nr);
2148 }
2149 EXPORT_SYMBOL_GPL(iommu_domain_window_disable);
2150
2151 /**
2152 * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework
2153 * @domain: the iommu domain where the fault has happened
2154 * @dev: the device where the fault has happened
2155 * @iova: the faulting address
2156 * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...)
2157 *
2158 * This function should be called by the low-level IOMMU implementations
2159 * whenever IOMMU faults happen, to allow high-level users, that are
2160 * interested in such events, to know about them.
2161 *
2162 * This event may be useful for several possible use cases:
2163 * - mere logging of the event
2164 * - dynamic TLB/PTE loading
2165 * - if restarting of the faulting device is required
2166 *
2167 * Returns 0 on success and an appropriate error code otherwise (if dynamic
2168 * PTE/TLB loading will one day be supported, implementations will be able
2169 * to tell whether it succeeded or not according to this return value).
2170 *
2171 * Specifically, -ENOSYS is returned if a fault handler isn't installed
2172 * (though fault handlers can also return -ENOSYS, in case they want to
2173 * elicit the default behavior of the IOMMU drivers).
2174 */
2175 int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
2176 unsigned long iova, int flags)
2177 {
2178 int ret = -ENOSYS;
2179
2180 /*
2181 * if upper layers showed interest and installed a fault handler,
2182 * invoke it.
2183 */
2184 if (domain->handler)
2185 ret = domain->handler(domain, dev, iova, flags,
2186 domain->handler_token);
2187
2188 trace_io_page_fault(dev, iova, flags);
2189 return ret;
2190 }
2191 EXPORT_SYMBOL_GPL(report_iommu_fault);
2192
2193 static int __init iommu_init(void)
2194 {
2195 iommu_group_kset = kset_create_and_add("iommu_groups",
2196 NULL, kernel_kobj);
2197 BUG_ON(!iommu_group_kset);
2198
2199 iommu_debugfs_setup();
2200
2201 return 0;
2202 }
2203 core_initcall(iommu_init);
2204
2205 int iommu_domain_get_attr(struct iommu_domain *domain,
2206 enum iommu_attr attr, void *data)
2207 {
2208 struct iommu_domain_geometry *geometry;
2209 bool *paging;
2210 int ret = 0;
2211
2212 switch (attr) {
2213 case DOMAIN_ATTR_GEOMETRY:
2214 geometry = data;
2215 *geometry = domain->geometry;
2216
2217 break;
2218 case DOMAIN_ATTR_PAGING:
2219 paging = data;
2220 *paging = (domain->pgsize_bitmap != 0UL);
2221 break;
2222 default:
2223 if (!domain->ops->domain_get_attr)
2224 return -EINVAL;
2225
2226 ret = domain->ops->domain_get_attr(domain, attr, data);
2227 }
2228
2229 return ret;
2230 }
2231 EXPORT_SYMBOL_GPL(iommu_domain_get_attr);
2232
2233 int iommu_domain_set_attr(struct iommu_domain *domain,
2234 enum iommu_attr attr, void *data)
2235 {
2236 int ret = 0;
2237
2238 switch (attr) {
2239 default:
2240 if (domain->ops->domain_set_attr == NULL)
2241 return -EINVAL;
2242
2243 ret = domain->ops->domain_set_attr(domain, attr, data);
2244 }
2245
2246 return ret;
2247 }
2248 EXPORT_SYMBOL_GPL(iommu_domain_set_attr);
2249
2250 void iommu_get_resv_regions(struct device *dev, struct list_head *list)
2251 {
2252 const struct iommu_ops *ops = dev->bus->iommu_ops;
2253
2254 if (ops && ops->get_resv_regions)
2255 ops->get_resv_regions(dev, list);
2256 }
2257
2258 void iommu_put_resv_regions(struct device *dev, struct list_head *list)
2259 {
2260 const struct iommu_ops *ops = dev->bus->iommu_ops;
2261
2262 if (ops && ops->put_resv_regions)
2263 ops->put_resv_regions(dev, list);
2264 }
2265
2266 /**
2267 * generic_iommu_put_resv_regions - Reserved region driver helper
2268 * @dev: device for which to free reserved regions
2269 * @list: reserved region list for device
2270 *
2271 * IOMMU drivers can use this to implement their .put_resv_regions() callback
2272 * for simple reservations. Memory allocated for each reserved region will be
2273 * freed. If an IOMMU driver allocates additional resources per region, it is
2274 * going to have to implement a custom callback.
2275 */
2276 void generic_iommu_put_resv_regions(struct device *dev, struct list_head *list)
2277 {
2278 struct iommu_resv_region *entry, *next;
2279
2280 list_for_each_entry_safe(entry, next, list, list)
2281 kfree(entry);
2282 }
2283 EXPORT_SYMBOL(generic_iommu_put_resv_regions);
2284
2285 struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
2286 size_t length, int prot,
2287 enum iommu_resv_type type)
2288 {
2289 struct iommu_resv_region *region;
2290
2291 region = kzalloc(sizeof(*region), GFP_KERNEL);
2292 if (!region)
2293 return NULL;
2294
2295 INIT_LIST_HEAD(&region->list);
2296 region->start = start;
2297 region->length = length;
2298 region->prot = prot;
2299 region->type = type;
2300 return region;
2301 }
2302 EXPORT_SYMBOL_GPL(iommu_alloc_resv_region);
2303
2304 static int
2305 request_default_domain_for_dev(struct device *dev, unsigned long type)
2306 {
2307 struct iommu_domain *domain;
2308 struct iommu_group *group;
2309 int ret;
2310
2311 /* Device must already be in a group before calling this function */
2312 group = iommu_group_get(dev);
2313 if (!group)
2314 return -EINVAL;
2315
2316 mutex_lock(&group->mutex);
2317
2318 ret = 0;
2319 if (group->default_domain && group->default_domain->type == type)
2320 goto out;
2321
2322 /* Don't change mappings of existing devices */
2323 ret = -EBUSY;
2324 if (iommu_group_device_count(group) != 1)
2325 goto out;
2326
2327 ret = -ENOMEM;
2328 domain = __iommu_domain_alloc(dev->bus, type);
2329 if (!domain)
2330 goto out;
2331
2332 /* Attach the device to the domain */
2333 ret = __iommu_attach_group(domain, group);
2334 if (ret) {
2335 iommu_domain_free(domain);
2336 goto out;
2337 }
2338
2339 /* Make the domain the default for this group */
2340 if (group->default_domain)
2341 iommu_domain_free(group->default_domain);
2342 group->default_domain = domain;
2343
2344 iommu_group_create_direct_mappings(group, dev);
2345
2346 dev_info(dev, "Using iommu %s mapping\n",
2347 type == IOMMU_DOMAIN_DMA ? "dma" : "direct");
2348
2349 ret = 0;
2350 out:
2351 mutex_unlock(&group->mutex);
2352 iommu_group_put(group);
2353
2354 return ret;
2355 }
2356
2357 /* Request that a device is direct mapped by the IOMMU */
2358 int iommu_request_dm_for_dev(struct device *dev)
2359 {
2360 return request_default_domain_for_dev(dev, IOMMU_DOMAIN_IDENTITY);
2361 }
2362
2363 /* Request that a device can't be direct mapped by the IOMMU */
2364 int iommu_request_dma_domain_for_dev(struct device *dev)
2365 {
2366 return request_default_domain_for_dev(dev, IOMMU_DOMAIN_DMA);
2367 }
2368
2369 void iommu_set_default_passthrough(bool cmd_line)
2370 {
2371 if (cmd_line)
2372 iommu_set_cmd_line_dma_api();
2373
2374 iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
2375 }
2376
2377 void iommu_set_default_translated(bool cmd_line)
2378 {
2379 if (cmd_line)
2380 iommu_set_cmd_line_dma_api();
2381
2382 iommu_def_domain_type = IOMMU_DOMAIN_DMA;
2383 }
2384
2385 bool iommu_default_passthrough(void)
2386 {
2387 return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY;
2388 }
2389 EXPORT_SYMBOL_GPL(iommu_default_passthrough);
2390
2391 const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode)
2392 {
2393 const struct iommu_ops *ops = NULL;
2394 struct iommu_device *iommu;
2395
2396 spin_lock(&iommu_device_lock);
2397 list_for_each_entry(iommu, &iommu_device_list, list)
2398 if (iommu->fwnode == fwnode) {
2399 ops = iommu->ops;
2400 break;
2401 }
2402 spin_unlock(&iommu_device_lock);
2403 return ops;
2404 }
2405
2406 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
2407 const struct iommu_ops *ops)
2408 {
2409 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2410
2411 if (fwspec)
2412 return ops == fwspec->ops ? 0 : -EINVAL;
2413
2414 if (!dev_iommu_get(dev))
2415 return -ENOMEM;
2416
2417 /* Preallocate for the overwhelmingly common case of 1 ID */
2418 fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL);
2419 if (!fwspec)
2420 return -ENOMEM;
2421
2422 of_node_get(to_of_node(iommu_fwnode));
2423 fwspec->iommu_fwnode = iommu_fwnode;
2424 fwspec->ops = ops;
2425 dev_iommu_fwspec_set(dev, fwspec);
2426 return 0;
2427 }
2428 EXPORT_SYMBOL_GPL(iommu_fwspec_init);
2429
2430 void iommu_fwspec_free(struct device *dev)
2431 {
2432 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2433
2434 if (fwspec) {
2435 fwnode_handle_put(fwspec->iommu_fwnode);
2436 kfree(fwspec);
2437 dev_iommu_fwspec_set(dev, NULL);
2438 }
2439 }
2440 EXPORT_SYMBOL_GPL(iommu_fwspec_free);
2441
2442 int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids)
2443 {
2444 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2445 int i, new_num;
2446
2447 if (!fwspec)
2448 return -EINVAL;
2449
2450 new_num = fwspec->num_ids + num_ids;
2451 if (new_num > 1) {
2452 fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num),
2453 GFP_KERNEL);
2454 if (!fwspec)
2455 return -ENOMEM;
2456
2457 dev_iommu_fwspec_set(dev, fwspec);
2458 }
2459
2460 for (i = 0; i < num_ids; i++)
2461 fwspec->ids[fwspec->num_ids + i] = ids[i];
2462
2463 fwspec->num_ids = new_num;
2464 return 0;
2465 }
2466 EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids);
2467
2468 /*
2469 * Per device IOMMU features.
2470 */
2471 bool iommu_dev_has_feature(struct device *dev, enum iommu_dev_features feat)
2472 {
2473 const struct iommu_ops *ops = dev->bus->iommu_ops;
2474
2475 if (ops && ops->dev_has_feat)
2476 return ops->dev_has_feat(dev, feat);
2477
2478 return false;
2479 }
2480 EXPORT_SYMBOL_GPL(iommu_dev_has_feature);
2481
2482 int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat)
2483 {
2484 const struct iommu_ops *ops = dev->bus->iommu_ops;
2485
2486 if (ops && ops->dev_enable_feat)
2487 return ops->dev_enable_feat(dev, feat);
2488
2489 return -ENODEV;
2490 }
2491 EXPORT_SYMBOL_GPL(iommu_dev_enable_feature);
2492
2493 /*
2494 * The device drivers should do the necessary cleanups before calling this.
2495 * For example, before disabling the aux-domain feature, the device driver
2496 * should detach all aux-domains. Otherwise, this will return -EBUSY.
2497 */
2498 int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
2499 {
2500 const struct iommu_ops *ops = dev->bus->iommu_ops;
2501
2502 if (ops && ops->dev_disable_feat)
2503 return ops->dev_disable_feat(dev, feat);
2504
2505 return -EBUSY;
2506 }
2507 EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);
2508
2509 bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat)
2510 {
2511 const struct iommu_ops *ops = dev->bus->iommu_ops;
2512
2513 if (ops && ops->dev_feat_enabled)
2514 return ops->dev_feat_enabled(dev, feat);
2515
2516 return false;
2517 }
2518 EXPORT_SYMBOL_GPL(iommu_dev_feature_enabled);
2519
2520 /*
2521 * Aux-domain specific attach/detach.
2522 *
2523 * Only works if iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX) returns
2524 * true. Also, as long as domains are attached to a device through this
2525 * interface, any tries to call iommu_attach_device() should fail
2526 * (iommu_detach_device() can't fail, so we fail when trying to re-attach).
2527 * This should make us safe against a device being attached to a guest as a
2528 * whole while there are still pasid users on it (aux and sva).
2529 */
2530 int iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev)
2531 {
2532 int ret = -ENODEV;
2533
2534 if (domain->ops->aux_attach_dev)
2535 ret = domain->ops->aux_attach_dev(domain, dev);
2536
2537 if (!ret)
2538 trace_attach_device_to_domain(dev);
2539
2540 return ret;
2541 }
2542 EXPORT_SYMBOL_GPL(iommu_aux_attach_device);
2543
2544 void iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev)
2545 {
2546 if (domain->ops->aux_detach_dev) {
2547 domain->ops->aux_detach_dev(domain, dev);
2548 trace_detach_device_from_domain(dev);
2549 }
2550 }
2551 EXPORT_SYMBOL_GPL(iommu_aux_detach_device);
2552
2553 int iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev)
2554 {
2555 int ret = -ENODEV;
2556
2557 if (domain->ops->aux_get_pasid)
2558 ret = domain->ops->aux_get_pasid(domain, dev);
2559
2560 return ret;
2561 }
2562 EXPORT_SYMBOL_GPL(iommu_aux_get_pasid);
2563
2564 /**
2565 * iommu_sva_bind_device() - Bind a process address space to a device
2566 * @dev: the device
2567 * @mm: the mm to bind, caller must hold a reference to it
2568 *
2569 * Create a bond between device and address space, allowing the device to access
2570 * the mm using the returned PASID. If a bond already exists between @device and
2571 * @mm, it is returned and an additional reference is taken. Caller must call
2572 * iommu_sva_unbind_device() to release each reference.
2573 *
2574 * iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to
2575 * initialize the required SVA features.
2576 *
2577 * On error, returns an ERR_PTR value.
2578 */
2579 struct iommu_sva *
2580 iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata)
2581 {
2582 struct iommu_group *group;
2583 struct iommu_sva *handle = ERR_PTR(-EINVAL);
2584 const struct iommu_ops *ops = dev->bus->iommu_ops;
2585
2586 if (!ops || !ops->sva_bind)
2587 return ERR_PTR(-ENODEV);
2588
2589 group = iommu_group_get(dev);
2590 if (!group)
2591 return ERR_PTR(-ENODEV);
2592
2593 /* Ensure device count and domain don't change while we're binding */
2594 mutex_lock(&group->mutex);
2595
2596 /*
2597 * To keep things simple, SVA currently doesn't support IOMMU groups
2598 * with more than one device. Existing SVA-capable systems are not
2599 * affected by the problems that required IOMMU groups (lack of ACS
2600 * isolation, device ID aliasing and other hardware issues).
2601 */
2602 if (iommu_group_device_count(group) != 1)
2603 goto out_unlock;
2604
2605 handle = ops->sva_bind(dev, mm, drvdata);
2606
2607 out_unlock:
2608 mutex_unlock(&group->mutex);
2609 iommu_group_put(group);
2610
2611 return handle;
2612 }
2613 EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
2614
2615 /**
2616 * iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device
2617 * @handle: the handle returned by iommu_sva_bind_device()
2618 *
2619 * Put reference to a bond between device and address space. The device should
2620 * not be issuing any more transaction for this PASID. All outstanding page
2621 * requests for this PASID must have been flushed to the IOMMU.
2622 *
2623 * Returns 0 on success, or an error value
2624 */
2625 void iommu_sva_unbind_device(struct iommu_sva *handle)
2626 {
2627 struct iommu_group *group;
2628 struct device *dev = handle->dev;
2629 const struct iommu_ops *ops = dev->bus->iommu_ops;
2630
2631 if (!ops || !ops->sva_unbind)
2632 return;
2633
2634 group = iommu_group_get(dev);
2635 if (!group)
2636 return;
2637
2638 mutex_lock(&group->mutex);
2639 ops->sva_unbind(handle);
2640 mutex_unlock(&group->mutex);
2641
2642 iommu_group_put(group);
2643 }
2644 EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
2645
2646 int iommu_sva_set_ops(struct iommu_sva *handle,
2647 const struct iommu_sva_ops *sva_ops)
2648 {
2649 if (handle->ops && handle->ops != sva_ops)
2650 return -EEXIST;
2651
2652 handle->ops = sva_ops;
2653 return 0;
2654 }
2655 EXPORT_SYMBOL_GPL(iommu_sva_set_ops);
2656
2657 int iommu_sva_get_pasid(struct iommu_sva *handle)
2658 {
2659 const struct iommu_ops *ops = handle->dev->bus->iommu_ops;
2660
2661 if (!ops || !ops->sva_get_pasid)
2662 return IOMMU_PASID_INVALID;
2663
2664 return ops->sva_get_pasid(handle);
2665 }
2666 EXPORT_SYMBOL_GPL(iommu_sva_get_pasid);
Linux with added FPC-III support