?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
fuse: flush requests on umount
[linux/fpc-iii.git] / drivers / iommu / amd_iommu.c
blob505a9adac2d51f560617ae36c7a2f53a58f906b6
1 /*
2 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20 #include <linux/ratelimit.h>
21 #include <linux/pci.h>
22 #include <linux/pci-ats.h>
23 #include <linux/bitmap.h>
24 #include <linux/slab.h>
25 #include <linux/debugfs.h>
26 #include <linux/scatterlist.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/iommu-helper.h>
29 #include <linux/iommu.h>
30 #include <linux/delay.h>
31 #include <linux/amd-iommu.h>
32 #include <linux/notifier.h>
33 #include <linux/export.h>
34 #include <linux/irq.h>
35 #include <linux/msi.h>
36 #include <asm/irq_remapping.h>
37 #include <asm/io_apic.h>
38 #include <asm/apic.h>
39 #include <asm/hw_irq.h>
40 #include <asm/msidef.h>
41 #include <asm/proto.h>
42 #include <asm/iommu.h>
43 #include <asm/gart.h>
44 #include <asm/dma.h>
45
46 #include "amd_iommu_proto.h"
47 #include "amd_iommu_types.h"
48 #include "irq_remapping.h"
49
50 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
51
52 #define LOOP_TIMEOUT 100000
53
54 /*
55 * This bitmap is used to advertise the page sizes our hardware support
56 * to the IOMMU core, which will then use this information to split
57 * physically contiguous memory regions it is mapping into page sizes
58 * that we support.
59 *
60 * 512GB Pages are not supported due to a hardware bug
61 */
62 #define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38))
63
64 static DEFINE_RWLOCK(amd_iommu_devtable_lock);
65
66 /* A list of preallocated protection domains */
67 static LIST_HEAD(iommu_pd_list);
68 static DEFINE_SPINLOCK(iommu_pd_list_lock);
69
70 /* List of all available dev_data structures */
71 static LIST_HEAD(dev_data_list);
72 static DEFINE_SPINLOCK(dev_data_list_lock);
73
74 LIST_HEAD(ioapic_map);
75 LIST_HEAD(hpet_map);
76
77 /*
78 * Domain for untranslated devices - only allocated
79 * if iommu=pt passed on kernel cmd line.
80 */
81 static struct protection_domain *pt_domain;
82
83 static const struct iommu_ops amd_iommu_ops;
84
85 static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
86 int amd_iommu_max_glx_val = -1;
87
88 static struct dma_map_ops amd_iommu_dma_ops;
89
90 /*
91 * This struct contains device specific data for the IOMMU
92 */
93 struct iommu_dev_data {
94 struct list_head list; /* For domain->dev_list */
95 struct list_head dev_data_list; /* For global dev_data_list */
96 struct list_head alias_list; /* Link alias-groups together */
97 struct iommu_dev_data *alias_data;/* The alias dev_data */
98 struct protection_domain *domain; /* Domain the device is bound to */
99 u16 devid; /* PCI Device ID */
100 bool iommu_v2; /* Device can make use of IOMMUv2 */
101 bool passthrough; /* Default for device is pt_domain */
102 struct {
103 bool enabled;
104 int qdep;
105 } ats; /* ATS state */
106 bool pri_tlp; /* PASID TLB required for
107 PPR completions */
108 u32 errata; /* Bitmap for errata to apply */
109 };
110
111 /*
112 * general struct to manage commands send to an IOMMU
113 */
114 struct iommu_cmd {
115 u32 data[4];
116 };
117
118 struct kmem_cache *amd_iommu_irq_cache;
119
120 static void update_domain(struct protection_domain *domain);
121 static int __init alloc_passthrough_domain(void);
122
123 /****************************************************************************
124 *
125 * Helper functions
126 *
127 ****************************************************************************/
128
129 static struct iommu_dev_data *alloc_dev_data(u16 devid)
130 {
131 struct iommu_dev_data *dev_data;
132 unsigned long flags;
133
134 dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
135 if (!dev_data)
136 return NULL;
137
138 INIT_LIST_HEAD(&dev_data->alias_list);
139
140 dev_data->devid = devid;
141
142 spin_lock_irqsave(&dev_data_list_lock, flags);
143 list_add_tail(&dev_data->dev_data_list, &dev_data_list);
144 spin_unlock_irqrestore(&dev_data_list_lock, flags);
145
146 return dev_data;
147 }
148
149 static void free_dev_data(struct iommu_dev_data *dev_data)
150 {
151 unsigned long flags;
152
153 spin_lock_irqsave(&dev_data_list_lock, flags);
154 list_del(&dev_data->dev_data_list);
155 spin_unlock_irqrestore(&dev_data_list_lock, flags);
156
157 kfree(dev_data);
158 }
159
160 static struct iommu_dev_data *search_dev_data(u16 devid)
161 {
162 struct iommu_dev_data *dev_data;
163 unsigned long flags;
164
165 spin_lock_irqsave(&dev_data_list_lock, flags);
166 list_for_each_entry(dev_data, &dev_data_list, dev_data_list) {
167 if (dev_data->devid == devid)
168 goto out_unlock;
169 }
170
171 dev_data = NULL;
172
173 out_unlock:
174 spin_unlock_irqrestore(&dev_data_list_lock, flags);
175
176 return dev_data;
177 }
178
179 static struct iommu_dev_data *find_dev_data(u16 devid)
180 {
181 struct iommu_dev_data *dev_data;
182
183 dev_data = search_dev_data(devid);
184
185 if (dev_data == NULL)
186 dev_data = alloc_dev_data(devid);
187
188 return dev_data;
189 }
190
191 static inline u16 get_device_id(struct device *dev)
192 {
193 struct pci_dev *pdev = to_pci_dev(dev);
194
195 return PCI_DEVID(pdev->bus->number, pdev->devfn);
196 }
197
198 static struct iommu_dev_data *get_dev_data(struct device *dev)
199 {
200 return dev->archdata.iommu;
201 }
202
203 static bool pci_iommuv2_capable(struct pci_dev *pdev)
204 {
205 static const int caps[] = {
206 PCI_EXT_CAP_ID_ATS,
207 PCI_EXT_CAP_ID_PRI,
208 PCI_EXT_CAP_ID_PASID,
209 };
210 int i, pos;
211
212 for (i = 0; i < 3; ++i) {
213 pos = pci_find_ext_capability(pdev, caps[i]);
214 if (pos == 0)
215 return false;
216 }
217
218 return true;
219 }
220
221 static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
222 {
223 struct iommu_dev_data *dev_data;
224
225 dev_data = get_dev_data(&pdev->dev);
226
227 return dev_data->errata & (1 << erratum) ? true : false;
228 }
229
230 /*
231 * In this function the list of preallocated protection domains is traversed to
232 * find the domain for a specific device
233 */
234 static struct dma_ops_domain *find_protection_domain(u16 devid)
235 {
236 struct dma_ops_domain *entry, *ret = NULL;
237 unsigned long flags;
238 u16 alias = amd_iommu_alias_table[devid];
239
240 if (list_empty(&iommu_pd_list))
241 return NULL;
242
243 spin_lock_irqsave(&iommu_pd_list_lock, flags);
244
245 list_for_each_entry(entry, &iommu_pd_list, list) {
246 if (entry->target_dev == devid ||
247 entry->target_dev == alias) {
248 ret = entry;
249 break;
250 }
251 }
252
253 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
254
255 return ret;
256 }
257
258 /*
259 * This function checks if the driver got a valid device from the caller to
260 * avoid dereferencing invalid pointers.
261 */
262 static bool check_device(struct device *dev)
263 {
264 u16 devid;
265
266 if (!dev || !dev->dma_mask)
267 return false;
268
269 /* No PCI device */
270 if (!dev_is_pci(dev))
271 return false;
272
273 devid = get_device_id(dev);
274
275 /* Out of our scope? */
276 if (devid > amd_iommu_last_bdf)
277 return false;
278
279 if (amd_iommu_rlookup_table[devid] == NULL)
280 return false;
281
282 return true;
283 }
284
285 static void init_iommu_group(struct device *dev)
286 {
287 struct iommu_group *group;
288
289 group = iommu_group_get_for_dev(dev);
290 if (!IS_ERR(group))
291 iommu_group_put(group);
292 }
293
294 static int __last_alias(struct pci_dev *pdev, u16 alias, void *data)
295 {
296 *(u16 *)data = alias;
297 return 0;
298 }
299
300 static u16 get_alias(struct device *dev)
301 {
302 struct pci_dev *pdev = to_pci_dev(dev);
303 u16 devid, ivrs_alias, pci_alias;
304
305 devid = get_device_id(dev);
306 ivrs_alias = amd_iommu_alias_table[devid];
307 pci_for_each_dma_alias(pdev, __last_alias, &pci_alias);
308
309 if (ivrs_alias == pci_alias)
310 return ivrs_alias;
311
312 /*
313 * DMA alias showdown
314 *
315 * The IVRS is fairly reliable in telling us about aliases, but it
316 * can't know about every screwy device. If we don't have an IVRS
317 * reported alias, use the PCI reported alias. In that case we may
318 * still need to initialize the rlookup and dev_table entries if the
319 * alias is to a non-existent device.
320 */
321 if (ivrs_alias == devid) {
322 if (!amd_iommu_rlookup_table[pci_alias]) {
323 amd_iommu_rlookup_table[pci_alias] =
324 amd_iommu_rlookup_table[devid];
325 memcpy(amd_iommu_dev_table[pci_alias].data,
326 amd_iommu_dev_table[devid].data,
327 sizeof(amd_iommu_dev_table[pci_alias].data));
328 }
329
330 return pci_alias;
331 }
332
333 pr_info("AMD-Vi: Using IVRS reported alias %02x:%02x.%d "
334 "for device %s[%04x:%04x], kernel reported alias "
335 "%02x:%02x.%d\n", PCI_BUS_NUM(ivrs_alias), PCI_SLOT(ivrs_alias),
336 PCI_FUNC(ivrs_alias), dev_name(dev), pdev->vendor, pdev->device,
337 PCI_BUS_NUM(pci_alias), PCI_SLOT(pci_alias),
338 PCI_FUNC(pci_alias));
339
340 /*
341 * If we don't have a PCI DMA alias and the IVRS alias is on the same
342 * bus, then the IVRS table may know about a quirk that we don't.
343 */
344 if (pci_alias == devid &&
345 PCI_BUS_NUM(ivrs_alias) == pdev->bus->number) {
346 pdev->dev_flags |= PCI_DEV_FLAGS_DMA_ALIAS_DEVFN;
347 pdev->dma_alias_devfn = ivrs_alias & 0xff;
348 pr_info("AMD-Vi: Added PCI DMA alias %02x.%d for %s\n",
349 PCI_SLOT(ivrs_alias), PCI_FUNC(ivrs_alias),
350 dev_name(dev));
351 }
352
353 return ivrs_alias;
354 }
355
356 static int iommu_init_device(struct device *dev)
357 {
358 struct pci_dev *pdev = to_pci_dev(dev);
359 struct iommu_dev_data *dev_data;
360 u16 alias;
361
362 if (dev->archdata.iommu)
363 return 0;
364
365 dev_data = find_dev_data(get_device_id(dev));
366 if (!dev_data)
367 return -ENOMEM;
368
369 alias = get_alias(dev);
370
371 if (alias != dev_data->devid) {
372 struct iommu_dev_data *alias_data;
373
374 alias_data = find_dev_data(alias);
375 if (alias_data == NULL) {
376 pr_err("AMD-Vi: Warning: Unhandled device %s\n",
377 dev_name(dev));
378 free_dev_data(dev_data);
379 return -ENOTSUPP;
380 }
381 dev_data->alias_data = alias_data;
382
383 /* Add device to the alias_list */
384 list_add(&dev_data->alias_list, &alias_data->alias_list);
385 }
386
387 if (pci_iommuv2_capable(pdev)) {
388 struct amd_iommu *iommu;
389
390 iommu = amd_iommu_rlookup_table[dev_data->devid];
391 dev_data->iommu_v2 = iommu->is_iommu_v2;
392 }
393
394 dev->archdata.iommu = dev_data;
395
396 iommu_device_link(amd_iommu_rlookup_table[dev_data->devid]->iommu_dev,
397 dev);
398
399 return 0;
400 }
401
402 static void iommu_ignore_device(struct device *dev)
403 {
404 u16 devid, alias;
405
406 devid = get_device_id(dev);
407 alias = amd_iommu_alias_table[devid];
408
409 memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
410 memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));
411
412 amd_iommu_rlookup_table[devid] = NULL;
413 amd_iommu_rlookup_table[alias] = NULL;
414 }
415
416 static void iommu_uninit_device(struct device *dev)
417 {
418 struct iommu_dev_data *dev_data = search_dev_data(get_device_id(dev));
419
420 if (!dev_data)
421 return;
422
423 iommu_device_unlink(amd_iommu_rlookup_table[dev_data->devid]->iommu_dev,
424 dev);
425
426 iommu_group_remove_device(dev);
427
428 /* Unlink from alias, it may change if another device is re-plugged */
429 dev_data->alias_data = NULL;
430
431 /*
432 * We keep dev_data around for unplugged devices and reuse it when the
433 * device is re-plugged - not doing so would introduce a ton of races.
434 */
435 }
436
437 void __init amd_iommu_uninit_devices(void)
438 {
439 struct iommu_dev_data *dev_data, *n;
440 struct pci_dev *pdev = NULL;
441
442 for_each_pci_dev(pdev) {
443
444 if (!check_device(&pdev->dev))
445 continue;
446
447 iommu_uninit_device(&pdev->dev);
448 }
449
450 /* Free all of our dev_data structures */
451 list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)
452 free_dev_data(dev_data);
453 }
454
455 int __init amd_iommu_init_devices(void)
456 {
457 struct pci_dev *pdev = NULL;
458 int ret = 0;
459
460 for_each_pci_dev(pdev) {
461
462 if (!check_device(&pdev->dev))
463 continue;
464
465 ret = iommu_init_device(&pdev->dev);
466 if (ret == -ENOTSUPP)
467 iommu_ignore_device(&pdev->dev);
468 else if (ret)
469 goto out_free;
470 }
471
472 /*
473 * Initialize IOMMU groups only after iommu_init_device() has
474 * had a chance to populate any IVRS defined aliases.
475 */
476 for_each_pci_dev(pdev) {
477 if (check_device(&pdev->dev))
478 init_iommu_group(&pdev->dev);
479 }
480
481 return 0;
482
483 out_free:
484
485 amd_iommu_uninit_devices();
486
487 return ret;
488 }
489 #ifdef CONFIG_AMD_IOMMU_STATS
490
491 /*
492 * Initialization code for statistics collection
493 */
494
495 DECLARE_STATS_COUNTER(compl_wait);
496 DECLARE_STATS_COUNTER(cnt_map_single);
497 DECLARE_STATS_COUNTER(cnt_unmap_single);
498 DECLARE_STATS_COUNTER(cnt_map_sg);
499 DECLARE_STATS_COUNTER(cnt_unmap_sg);
500 DECLARE_STATS_COUNTER(cnt_alloc_coherent);
501 DECLARE_STATS_COUNTER(cnt_free_coherent);
502 DECLARE_STATS_COUNTER(cross_page);
503 DECLARE_STATS_COUNTER(domain_flush_single);
504 DECLARE_STATS_COUNTER(domain_flush_all);
505 DECLARE_STATS_COUNTER(alloced_io_mem);
506 DECLARE_STATS_COUNTER(total_map_requests);
507 DECLARE_STATS_COUNTER(complete_ppr);
508 DECLARE_STATS_COUNTER(invalidate_iotlb);
509 DECLARE_STATS_COUNTER(invalidate_iotlb_all);
510 DECLARE_STATS_COUNTER(pri_requests);
511
512 static struct dentry *stats_dir;
513 static struct dentry *de_fflush;
514
515 static void amd_iommu_stats_add(struct __iommu_counter *cnt)
516 {
517 if (stats_dir == NULL)
518 return;
519
520 cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
521 &cnt->value);
522 }
523
524 static void amd_iommu_stats_init(void)
525 {
526 stats_dir = debugfs_create_dir("amd-iommu", NULL);
527 if (stats_dir == NULL)
528 return;
529
530 de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir,
531 &amd_iommu_unmap_flush);
532
533 amd_iommu_stats_add(&compl_wait);
534 amd_iommu_stats_add(&cnt_map_single);
535 amd_iommu_stats_add(&cnt_unmap_single);
536 amd_iommu_stats_add(&cnt_map_sg);
537 amd_iommu_stats_add(&cnt_unmap_sg);
538 amd_iommu_stats_add(&cnt_alloc_coherent);
539 amd_iommu_stats_add(&cnt_free_coherent);
540 amd_iommu_stats_add(&cross_page);
541 amd_iommu_stats_add(&domain_flush_single);
542 amd_iommu_stats_add(&domain_flush_all);
543 amd_iommu_stats_add(&alloced_io_mem);
544 amd_iommu_stats_add(&total_map_requests);
545 amd_iommu_stats_add(&complete_ppr);
546 amd_iommu_stats_add(&invalidate_iotlb);
547 amd_iommu_stats_add(&invalidate_iotlb_all);
548 amd_iommu_stats_add(&pri_requests);
549 }
550
551 #endif
552
553 /****************************************************************************
554 *
555 * Interrupt handling functions
556 *
557 ****************************************************************************/
558
559 static void dump_dte_entry(u16 devid)
560 {
561 int i;
562
563 for (i = 0; i < 4; ++i)
564 pr_err("AMD-Vi: DTE[%d]: %016llx\n", i,
565 amd_iommu_dev_table[devid].data[i]);
566 }
567
568 static void dump_command(unsigned long phys_addr)
569 {
570 struct iommu_cmd *cmd = phys_to_virt(phys_addr);
571 int i;
572
573 for (i = 0; i < 4; ++i)
574 pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
575 }
576
577 static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
578 {
579 int type, devid, domid, flags;
580 volatile u32 *event = __evt;
581 int count = 0;
582 u64 address;
583
584 retry:
585 type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
586 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
587 domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
588 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
589 address = (u64)(((u64)event[3]) << 32) | event[2];
590
591 if (type == 0) {
592 /* Did we hit the erratum? */
593 if (++count == LOOP_TIMEOUT) {
594 pr_err("AMD-Vi: No event written to event log\n");
595 return;
596 }
597 udelay(1);
598 goto retry;
599 }
600
601 printk(KERN_ERR "AMD-Vi: Event logged [");
602
603 switch (type) {
604 case EVENT_TYPE_ILL_DEV:
605 printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
606 "address=0x%016llx flags=0x%04x]\n",
607 PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
608 address, flags);
609 dump_dte_entry(devid);
610 break;
611 case EVENT_TYPE_IO_FAULT:
612 printk("IO_PAGE_FAULT device=%02x:%02x.%x "
613 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
614 PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
615 domid, address, flags);
616 break;
617 case EVENT_TYPE_DEV_TAB_ERR:
618 printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
619 "address=0x%016llx flags=0x%04x]\n",
620 PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
621 address, flags);
622 break;
623 case EVENT_TYPE_PAGE_TAB_ERR:
624 printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
625 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
626 PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
627 domid, address, flags);
628 break;
629 case EVENT_TYPE_ILL_CMD:
630 printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
631 dump_command(address);
632 break;
633 case EVENT_TYPE_CMD_HARD_ERR:
634 printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
635 "flags=0x%04x]\n", address, flags);
636 break;
637 case EVENT_TYPE_IOTLB_INV_TO:
638 printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
639 "address=0x%016llx]\n",
640 PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
641 address);
642 break;
643 case EVENT_TYPE_INV_DEV_REQ:
644 printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
645 "address=0x%016llx flags=0x%04x]\n",
646 PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
647 address, flags);
648 break;
649 default:
650 printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
651 }
652
653 memset(__evt, 0, 4 * sizeof(u32));
654 }
655
656 static void iommu_poll_events(struct amd_iommu *iommu)
657 {
658 u32 head, tail;
659
660 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
661 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
662
663 while (head != tail) {
664 iommu_print_event(iommu, iommu->evt_buf + head);
665 head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
666 }
667
668 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
669 }
670
671 static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw)
672 {
673 struct amd_iommu_fault fault;
674
675 INC_STATS_COUNTER(pri_requests);
676
677 if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
678 pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
679 return;
680 }
681
682 fault.address = raw[1];
683 fault.pasid = PPR_PASID(raw[0]);
684 fault.device_id = PPR_DEVID(raw[0]);
685 fault.tag = PPR_TAG(raw[0]);
686 fault.flags = PPR_FLAGS(raw[0]);
687
688 atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
689 }
690
691 static void iommu_poll_ppr_log(struct amd_iommu *iommu)
692 {
693 u32 head, tail;
694
695 if (iommu->ppr_log == NULL)
696 return;
697
698 head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
699 tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
700
701 while (head != tail) {
702 volatile u64 *raw;
703 u64 entry[2];
704 int i;
705
706 raw = (u64 *)(iommu->ppr_log + head);
707
708 /*
709 * Hardware bug: Interrupt may arrive before the entry is
710 * written to memory. If this happens we need to wait for the
711 * entry to arrive.
712 */
713 for (i = 0; i < LOOP_TIMEOUT; ++i) {
714 if (PPR_REQ_TYPE(raw[0]) != 0)
715 break;
716 udelay(1);
717 }
718
719 /* Avoid memcpy function-call overhead */
720 entry[0] = raw[0];
721 entry[1] = raw[1];
722
723 /*
724 * To detect the hardware bug we need to clear the entry
725 * back to zero.
726 */
727 raw[0] = raw[1] = 0UL;
728
729 /* Update head pointer of hardware ring-buffer */
730 head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
731 writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
732
733 /* Handle PPR entry */
734 iommu_handle_ppr_entry(iommu, entry);
735
736 /* Refresh ring-buffer information */
737 head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
738 tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
739 }
740 }
741
742 irqreturn_t amd_iommu_int_thread(int irq, void *data)
743 {
744 struct amd_iommu *iommu = (struct amd_iommu *) data;
745 u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
746
747 while (status & (MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK)) {
748 /* Enable EVT and PPR interrupts again */
749 writel((MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK),
750 iommu->mmio_base + MMIO_STATUS_OFFSET);
751
752 if (status & MMIO_STATUS_EVT_INT_MASK) {
753 pr_devel("AMD-Vi: Processing IOMMU Event Log\n");
754 iommu_poll_events(iommu);
755 }
756
757 if (status & MMIO_STATUS_PPR_INT_MASK) {
758 pr_devel("AMD-Vi: Processing IOMMU PPR Log\n");
759 iommu_poll_ppr_log(iommu);
760 }
761
762 /*
763 * Hardware bug: ERBT1312
764 * When re-enabling interrupt (by writing 1
765 * to clear the bit), the hardware might also try to set
766 * the interrupt bit in the event status register.
767 * In this scenario, the bit will be set, and disable
768 * subsequent interrupts.
769 *
770 * Workaround: The IOMMU driver should read back the
771 * status register and check if the interrupt bits are cleared.
772 * If not, driver will need to go through the interrupt handler
773 * again and re-clear the bits
774 */
775 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
776 }
777 return IRQ_HANDLED;
778 }
779
780 irqreturn_t amd_iommu_int_handler(int irq, void *data)
781 {
782 return IRQ_WAKE_THREAD;
783 }
784
785 /****************************************************************************
786 *
787 * IOMMU command queuing functions
788 *
789 ****************************************************************************/
790
791 static int wait_on_sem(volatile u64 *sem)
792 {
793 int i = 0;
794
795 while (*sem == 0 && i < LOOP_TIMEOUT) {
796 udelay(1);
797 i += 1;
798 }
799
800 if (i == LOOP_TIMEOUT) {
801 pr_alert("AMD-Vi: Completion-Wait loop timed out\n");
802 return -EIO;
803 }
804
805 return 0;
806 }
807
808 static void copy_cmd_to_buffer(struct amd_iommu *iommu,
809 struct iommu_cmd *cmd,
810 u32 tail)
811 {
812 u8 *target;
813
814 target = iommu->cmd_buf + tail;
815 tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
816
817 /* Copy command to buffer */
818 memcpy(target, cmd, sizeof(*cmd));
819
820 /* Tell the IOMMU about it */
821 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
822 }
823
824 static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
825 {
826 WARN_ON(address & 0x7ULL);
827
828 memset(cmd, 0, sizeof(*cmd));
829 cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
830 cmd->data[1] = upper_32_bits(__pa(address));
831 cmd->data[2] = 1;
832 CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
833 }
834
835 static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
836 {
837 memset(cmd, 0, sizeof(*cmd));
838 cmd->data[0] = devid;
839 CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
840 }
841
842 static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
843 size_t size, u16 domid, int pde)
844 {
845 u64 pages;
846 int s;
847
848 pages = iommu_num_pages(address, size, PAGE_SIZE);
849 s = 0;
850
851 if (pages > 1) {
852 /*
853 * If we have to flush more than one page, flush all
854 * TLB entries for this domain
855 */
856 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
857 s = 1;
858 }
859
860 address &= PAGE_MASK;
861
862 memset(cmd, 0, sizeof(*cmd));
863 cmd->data[1] |= domid;
864 cmd->data[2] = lower_32_bits(address);
865 cmd->data[3] = upper_32_bits(address);
866 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
867 if (s) /* size bit - we flush more than one 4kb page */
868 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
869 if (pde) /* PDE bit - we want to flush everything, not only the PTEs */
870 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
871 }
872
873 static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
874 u64 address, size_t size)
875 {
876 u64 pages;
877 int s;
878
879 pages = iommu_num_pages(address, size, PAGE_SIZE);
880 s = 0;
881
882 if (pages > 1) {
883 /*
884 * If we have to flush more than one page, flush all
885 * TLB entries for this domain
886 */
887 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
888 s = 1;
889 }
890
891 address &= PAGE_MASK;
892
893 memset(cmd, 0, sizeof(*cmd));
894 cmd->data[0] = devid;
895 cmd->data[0] |= (qdep & 0xff) << 24;
896 cmd->data[1] = devid;
897 cmd->data[2] = lower_32_bits(address);
898 cmd->data[3] = upper_32_bits(address);
899 CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
900 if (s)
901 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
902 }
903
904 static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
905 u64 address, bool size)
906 {
907 memset(cmd, 0, sizeof(*cmd));
908
909 address &= ~(0xfffULL);
910
911 cmd->data[0] = pasid;
912 cmd->data[1] = domid;
913 cmd->data[2] = lower_32_bits(address);
914 cmd->data[3] = upper_32_bits(address);
915 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
916 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
917 if (size)
918 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
919 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
920 }
921
922 static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
923 int qdep, u64 address, bool size)
924 {
925 memset(cmd, 0, sizeof(*cmd));
926
927 address &= ~(0xfffULL);
928
929 cmd->data[0] = devid;
930 cmd->data[0] |= ((pasid >> 8) & 0xff) << 16;
931 cmd->data[0] |= (qdep & 0xff) << 24;
932 cmd->data[1] = devid;
933 cmd->data[1] |= (pasid & 0xff) << 16;
934 cmd->data[2] = lower_32_bits(address);
935 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
936 cmd->data[3] = upper_32_bits(address);
937 if (size)
938 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
939 CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
940 }
941
942 static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
943 int status, int tag, bool gn)
944 {
945 memset(cmd, 0, sizeof(*cmd));
946
947 cmd->data[0] = devid;
948 if (gn) {
949 cmd->data[1] = pasid;
950 cmd->data[2] = CMD_INV_IOMMU_PAGES_GN_MASK;
951 }
952 cmd->data[3] = tag & 0x1ff;
953 cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;
954
955 CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
956 }
957
958 static void build_inv_all(struct iommu_cmd *cmd)
959 {
960 memset(cmd, 0, sizeof(*cmd));
961 CMD_SET_TYPE(cmd, CMD_INV_ALL);
962 }
963
964 static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)
965 {
966 memset(cmd, 0, sizeof(*cmd));
967 cmd->data[0] = devid;
968 CMD_SET_TYPE(cmd, CMD_INV_IRT);
969 }
970
971 /*
972 * Writes the command to the IOMMUs command buffer and informs the
973 * hardware about the new command.
974 */
975 static int iommu_queue_command_sync(struct amd_iommu *iommu,
976 struct iommu_cmd *cmd,
977 bool sync)
978 {
979 u32 left, tail, head, next_tail;
980 unsigned long flags;
981
982 WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);
983
984 again:
985 spin_lock_irqsave(&iommu->lock, flags);
986
987 head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
988 tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
989 next_tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
990 left = (head - next_tail) % iommu->cmd_buf_size;
991
992 if (left <= 2) {
993 struct iommu_cmd sync_cmd;
994 volatile u64 sem = 0;
995 int ret;
996
997 build_completion_wait(&sync_cmd, (u64)&sem);
998 copy_cmd_to_buffer(iommu, &sync_cmd, tail);
999
1000 spin_unlock_irqrestore(&iommu->lock, flags);
1001
1002 if ((ret = wait_on_sem(&sem)) != 0)
1003 return ret;
1004
1005 goto again;
1006 }
1007
1008 copy_cmd_to_buffer(iommu, cmd, tail);
1009
1010 /* We need to sync now to make sure all commands are processed */
1011 iommu->need_sync = sync;
1012
1013 spin_unlock_irqrestore(&iommu->lock, flags);
1014
1015 return 0;
1016 }
1017
1018 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
1019 {
1020 return iommu_queue_command_sync(iommu, cmd, true);
1021 }
1022
1023 /*
1024 * This function queues a completion wait command into the command
1025 * buffer of an IOMMU
1026 */
1027 static int iommu_completion_wait(struct amd_iommu *iommu)
1028 {
1029 struct iommu_cmd cmd;
1030 volatile u64 sem = 0;
1031 int ret;
1032
1033 if (!iommu->need_sync)
1034 return 0;
1035
1036 build_completion_wait(&cmd, (u64)&sem);
1037
1038 ret = iommu_queue_command_sync(iommu, &cmd, false);
1039 if (ret)
1040 return ret;
1041
1042 return wait_on_sem(&sem);
1043 }
1044
1045 static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
1046 {
1047 struct iommu_cmd cmd;
1048
1049 build_inv_dte(&cmd, devid);
1050
1051 return iommu_queue_command(iommu, &cmd);
1052 }
1053
1054 static void iommu_flush_dte_all(struct amd_iommu *iommu)
1055 {
1056 u32 devid;
1057
1058 for (devid = 0; devid <= 0xffff; ++devid)
1059 iommu_flush_dte(iommu, devid);
1060
1061 iommu_completion_wait(iommu);
1062 }
1063
1064 /*
1065 * This function uses heavy locking and may disable irqs for some time. But
1066 * this is no issue because it is only called during resume.
1067 */
1068 static void iommu_flush_tlb_all(struct amd_iommu *iommu)
1069 {
1070 u32 dom_id;
1071
1072 for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {
1073 struct iommu_cmd cmd;
1074 build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
1075 dom_id, 1);
1076 iommu_queue_command(iommu, &cmd);
1077 }
1078
1079 iommu_completion_wait(iommu);
1080 }
1081
1082 static void iommu_flush_all(struct amd_iommu *iommu)
1083 {
1084 struct iommu_cmd cmd;
1085
1086 build_inv_all(&cmd);
1087
1088 iommu_queue_command(iommu, &cmd);
1089 iommu_completion_wait(iommu);
1090 }
1091
1092 static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)
1093 {
1094 struct iommu_cmd cmd;
1095
1096 build_inv_irt(&cmd, devid);
1097
1098 iommu_queue_command(iommu, &cmd);
1099 }
1100
1101 static void iommu_flush_irt_all(struct amd_iommu *iommu)
1102 {
1103 u32 devid;
1104
1105 for (devid = 0; devid <= MAX_DEV_TABLE_ENTRIES; devid++)
1106 iommu_flush_irt(iommu, devid);
1107
1108 iommu_completion_wait(iommu);
1109 }
1110
1111 void iommu_flush_all_caches(struct amd_iommu *iommu)
1112 {
1113 if (iommu_feature(iommu, FEATURE_IA)) {
1114 iommu_flush_all(iommu);
1115 } else {
1116 iommu_flush_dte_all(iommu);
1117 iommu_flush_irt_all(iommu);
1118 iommu_flush_tlb_all(iommu);
1119 }
1120 }
1121
1122 /*
1123 * Command send function for flushing on-device TLB
1124 */
1125 static int device_flush_iotlb(struct iommu_dev_data *dev_data,
1126 u64 address, size_t size)
1127 {
1128 struct amd_iommu *iommu;
1129 struct iommu_cmd cmd;
1130 int qdep;
1131
1132 qdep = dev_data->ats.qdep;
1133 iommu = amd_iommu_rlookup_table[dev_data->devid];
1134
1135 build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);
1136
1137 return iommu_queue_command(iommu, &cmd);
1138 }
1139
1140 /*
1141 * Command send function for invalidating a device table entry
1142 */
1143 static int device_flush_dte(struct iommu_dev_data *dev_data)
1144 {
1145 struct amd_iommu *iommu;
1146 int ret;
1147
1148 iommu = amd_iommu_rlookup_table[dev_data->devid];
1149
1150 ret = iommu_flush_dte(iommu, dev_data->devid);
1151 if (ret)
1152 return ret;
1153
1154 if (dev_data->ats.enabled)
1155 ret = device_flush_iotlb(dev_data, 0, ~0UL);
1156
1157 return ret;
1158 }
1159
1160 /*
1161 * TLB invalidation function which is called from the mapping functions.
1162 * It invalidates a single PTE if the range to flush is within a single
1163 * page. Otherwise it flushes the whole TLB of the IOMMU.
1164 */
1165 static void __domain_flush_pages(struct protection_domain *domain,
1166 u64 address, size_t size, int pde)
1167 {
1168 struct iommu_dev_data *dev_data;
1169 struct iommu_cmd cmd;
1170 int ret = 0, i;
1171
1172 build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
1173
1174 for (i = 0; i < amd_iommus_present; ++i) {
1175 if (!domain->dev_iommu[i])
1176 continue;
1177
1178 /*
1179 * Devices of this domain are behind this IOMMU
1180 * We need a TLB flush
1181 */
1182 ret |= iommu_queue_command(amd_iommus[i], &cmd);
1183 }
1184
1185 list_for_each_entry(dev_data, &domain->dev_list, list) {
1186
1187 if (!dev_data->ats.enabled)
1188 continue;
1189
1190 ret |= device_flush_iotlb(dev_data, address, size);
1191 }
1192
1193 WARN_ON(ret);
1194 }
1195
1196 static void domain_flush_pages(struct protection_domain *domain,
1197 u64 address, size_t size)
1198 {
1199 __domain_flush_pages(domain, address, size, 0);
1200 }
1201
1202 /* Flush the whole IO/TLB for a given protection domain */
1203 static void domain_flush_tlb(struct protection_domain *domain)
1204 {
1205 __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
1206 }
1207
1208 /* Flush the whole IO/TLB for a given protection domain - including PDE */
1209 static void domain_flush_tlb_pde(struct protection_domain *domain)
1210 {
1211 __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
1212 }
1213
1214 static void domain_flush_complete(struct protection_domain *domain)
1215 {
1216 int i;
1217
1218 for (i = 0; i < amd_iommus_present; ++i) {
1219 if (!domain->dev_iommu[i])
1220 continue;
1221
1222 /*
1223 * Devices of this domain are behind this IOMMU
1224 * We need to wait for completion of all commands.
1225 */
1226 iommu_completion_wait(amd_iommus[i]);
1227 }
1228 }
1229
1230
1231 /*
1232 * This function flushes the DTEs for all devices in domain
1233 */
1234 static void domain_flush_devices(struct protection_domain *domain)
1235 {
1236 struct iommu_dev_data *dev_data;
1237
1238 list_for_each_entry(dev_data, &domain->dev_list, list)
1239 device_flush_dte(dev_data);
1240 }
1241
1242 /****************************************************************************
1243 *
1244 * The functions below are used the create the page table mappings for
1245 * unity mapped regions.
1246 *
1247 ****************************************************************************/
1248
1249 /*
1250 * This function is used to add another level to an IO page table. Adding
1251 * another level increases the size of the address space by 9 bits to a size up
1252 * to 64 bits.
1253 */
1254 static bool increase_address_space(struct protection_domain *domain,
1255 gfp_t gfp)
1256 {
1257 u64 *pte;
1258
1259 if (domain->mode == PAGE_MODE_6_LEVEL)
1260 /* address space already 64 bit large */
1261 return false;
1262
1263 pte = (void *)get_zeroed_page(gfp);
1264 if (!pte)
1265 return false;
1266
1267 *pte = PM_LEVEL_PDE(domain->mode,
1268 virt_to_phys(domain->pt_root));
1269 domain->pt_root = pte;
1270 domain->mode += 1;
1271 domain->updated = true;
1272
1273 return true;
1274 }
1275
1276 static u64 *alloc_pte(struct protection_domain *domain,
1277 unsigned long address,
1278 unsigned long page_size,
1279 u64 **pte_page,
1280 gfp_t gfp)
1281 {
1282 int level, end_lvl;
1283 u64 *pte, *page;
1284
1285 BUG_ON(!is_power_of_2(page_size));
1286
1287 while (address > PM_LEVEL_SIZE(domain->mode))
1288 increase_address_space(domain, gfp);
1289
1290 level = domain->mode - 1;
1291 pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
1292 address = PAGE_SIZE_ALIGN(address, page_size);
1293 end_lvl = PAGE_SIZE_LEVEL(page_size);
1294
1295 while (level > end_lvl) {
1296 if (!IOMMU_PTE_PRESENT(*pte)) {
1297 page = (u64 *)get_zeroed_page(gfp);
1298 if (!page)
1299 return NULL;
1300 *pte = PM_LEVEL_PDE(level, virt_to_phys(page));
1301 }
1302
1303 /* No level skipping support yet */
1304 if (PM_PTE_LEVEL(*pte) != level)
1305 return NULL;
1306
1307 level -= 1;
1308
1309 pte = IOMMU_PTE_PAGE(*pte);
1310
1311 if (pte_page && level == end_lvl)
1312 *pte_page = pte;
1313
1314 pte = &pte[PM_LEVEL_INDEX(level, address)];
1315 }
1316
1317 return pte;
1318 }
1319
1320 /*
1321 * This function checks if there is a PTE for a given dma address. If
1322 * there is one, it returns the pointer to it.
1323 */
1324 static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)
1325 {
1326 int level;
1327 u64 *pte;
1328
1329 if (address > PM_LEVEL_SIZE(domain->mode))
1330 return NULL;
1331
1332 level = domain->mode - 1;
1333 pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
1334
1335 while (level > 0) {
1336
1337 /* Not Present */
1338 if (!IOMMU_PTE_PRESENT(*pte))
1339 return NULL;
1340
1341 /* Large PTE */
1342 if (PM_PTE_LEVEL(*pte) == 0x07) {
1343 unsigned long pte_mask, __pte;
1344
1345 /*
1346 * If we have a series of large PTEs, make
1347 * sure to return a pointer to the first one.
1348 */
1349 pte_mask = PTE_PAGE_SIZE(*pte);
1350 pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
1351 __pte = ((unsigned long)pte) & pte_mask;
1352
1353 return (u64 *)__pte;
1354 }
1355
1356 /* No level skipping support yet */
1357 if (PM_PTE_LEVEL(*pte) != level)
1358 return NULL;
1359
1360 level -= 1;
1361
1362 /* Walk to the next level */
1363 pte = IOMMU_PTE_PAGE(*pte);
1364 pte = &pte[PM_LEVEL_INDEX(level, address)];
1365 }
1366
1367 return pte;
1368 }
1369
1370 /*
1371 * Generic mapping functions. It maps a physical address into a DMA
1372 * address space. It allocates the page table pages if necessary.
1373 * In the future it can be extended to a generic mapping function
1374 * supporting all features of AMD IOMMU page tables like level skipping
1375 * and full 64 bit address spaces.
1376 */
1377 static int iommu_map_page(struct protection_domain *dom,
1378 unsigned long bus_addr,
1379 unsigned long phys_addr,
1380 int prot,
1381 unsigned long page_size)
1382 {
1383 u64 __pte, *pte;
1384 int i, count;
1385
1386 if (!(prot & IOMMU_PROT_MASK))
1387 return -EINVAL;
1388
1389 bus_addr = PAGE_ALIGN(bus_addr);
1390 phys_addr = PAGE_ALIGN(phys_addr);
1391 count = PAGE_SIZE_PTE_COUNT(page_size);
1392 pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
1393
1394 if (!pte)
1395 return -ENOMEM;
1396
1397 for (i = 0; i < count; ++i)
1398 if (IOMMU_PTE_PRESENT(pte[i]))
1399 return -EBUSY;
1400
1401 if (page_size > PAGE_SIZE) {
1402 __pte = PAGE_SIZE_PTE(phys_addr, page_size);
1403 __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
1404 } else
1405 __pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;
1406
1407 if (prot & IOMMU_PROT_IR)
1408 __pte |= IOMMU_PTE_IR;
1409 if (prot & IOMMU_PROT_IW)
1410 __pte |= IOMMU_PTE_IW;
1411
1412 for (i = 0; i < count; ++i)
1413 pte[i] = __pte;
1414
1415 update_domain(dom);
1416
1417 return 0;
1418 }
1419
1420 static unsigned long iommu_unmap_page(struct protection_domain *dom,
1421 unsigned long bus_addr,
1422 unsigned long page_size)
1423 {
1424 unsigned long long unmap_size, unmapped;
1425 u64 *pte;
1426
1427 BUG_ON(!is_power_of_2(page_size));
1428
1429 unmapped = 0;
1430
1431 while (unmapped < page_size) {
1432
1433 pte = fetch_pte(dom, bus_addr);
1434
1435 if (!pte) {
1436 /*
1437 * No PTE for this address
1438 * move forward in 4kb steps
1439 */
1440 unmap_size = PAGE_SIZE;
1441 } else if (PM_PTE_LEVEL(*pte) == 0) {
1442 /* 4kb PTE found for this address */
1443 unmap_size = PAGE_SIZE;
1444 *pte = 0ULL;
1445 } else {
1446 int count, i;
1447
1448 /* Large PTE found which maps this address */
1449 unmap_size = PTE_PAGE_SIZE(*pte);
1450
1451 /* Only unmap from the first pte in the page */
1452 if ((unmap_size - 1) & bus_addr)
1453 break;
1454 count = PAGE_SIZE_PTE_COUNT(unmap_size);
1455 for (i = 0; i < count; i++)
1456 pte[i] = 0ULL;
1457 }
1458
1459 bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size;
1460 unmapped += unmap_size;
1461 }
1462
1463 BUG_ON(unmapped && !is_power_of_2(unmapped));
1464
1465 return unmapped;
1466 }
1467
1468 /*
1469 * This function checks if a specific unity mapping entry is needed for
1470 * this specific IOMMU.
1471 */
1472 static int iommu_for_unity_map(struct amd_iommu *iommu,
1473 struct unity_map_entry *entry)
1474 {
1475 u16 bdf, i;
1476
1477 for (i = entry->devid_start; i <= entry->devid_end; ++i) {
1478 bdf = amd_iommu_alias_table[i];
1479 if (amd_iommu_rlookup_table[bdf] == iommu)
1480 return 1;
1481 }
1482
1483 return 0;
1484 }
1485
1486 /*
1487 * This function actually applies the mapping to the page table of the
1488 * dma_ops domain.
1489 */
1490 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
1491 struct unity_map_entry *e)
1492 {
1493 u64 addr;
1494 int ret;
1495
1496 for (addr = e->address_start; addr < e->address_end;
1497 addr += PAGE_SIZE) {
1498 ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
1499 PAGE_SIZE);
1500 if (ret)
1501 return ret;
1502 /*
1503 * if unity mapping is in aperture range mark the page
1504 * as allocated in the aperture
1505 */
1506 if (addr < dma_dom->aperture_size)
1507 __set_bit(addr >> PAGE_SHIFT,
1508 dma_dom->aperture[0]->bitmap);
1509 }
1510
1511 return 0;
1512 }
1513
1514 /*
1515 * Init the unity mappings for a specific IOMMU in the system
1516 *
1517 * Basically iterates over all unity mapping entries and applies them to
1518 * the default domain DMA of that IOMMU if necessary.
1519 */
1520 static int iommu_init_unity_mappings(struct amd_iommu *iommu)
1521 {
1522 struct unity_map_entry *entry;
1523 int ret;
1524
1525 list_for_each_entry(entry, &amd_iommu_unity_map, list) {
1526 if (!iommu_for_unity_map(iommu, entry))
1527 continue;
1528 ret = dma_ops_unity_map(iommu->default_dom, entry);
1529 if (ret)
1530 return ret;
1531 }
1532
1533 return 0;
1534 }
1535
1536 /*
1537 * Inits the unity mappings required for a specific device
1538 */
1539 static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
1540 u16 devid)
1541 {
1542 struct unity_map_entry *e;
1543 int ret;
1544
1545 list_for_each_entry(e, &amd_iommu_unity_map, list) {
1546 if (!(devid >= e->devid_start && devid <= e->devid_end))
1547 continue;
1548 ret = dma_ops_unity_map(dma_dom, e);
1549 if (ret)
1550 return ret;
1551 }
1552
1553 return 0;
1554 }
1555
1556 /****************************************************************************
1557 *
1558 * The next functions belong to the address allocator for the dma_ops
1559 * interface functions. They work like the allocators in the other IOMMU
1560 * drivers. Its basically a bitmap which marks the allocated pages in
1561 * the aperture. Maybe it could be enhanced in the future to a more
1562 * efficient allocator.
1563 *
1564 ****************************************************************************/
1565
1566 /*
1567 * The address allocator core functions.
1568 *
1569 * called with domain->lock held
1570 */
1571
1572 /*
1573 * Used to reserve address ranges in the aperture (e.g. for exclusion
1574 * ranges.
1575 */
1576 static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
1577 unsigned long start_page,
1578 unsigned int pages)
1579 {
1580 unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
1581
1582 if (start_page + pages > last_page)
1583 pages = last_page - start_page;
1584
1585 for (i = start_page; i < start_page + pages; ++i) {
1586 int index = i / APERTURE_RANGE_PAGES;
1587 int page = i % APERTURE_RANGE_PAGES;
1588 __set_bit(page, dom->aperture[index]->bitmap);
1589 }
1590 }
1591
1592 /*
1593 * This function is used to add a new aperture range to an existing
1594 * aperture in case of dma_ops domain allocation or address allocation
1595 * failure.
1596 */
1597 static int alloc_new_range(struct dma_ops_domain *dma_dom,
1598 bool populate, gfp_t gfp)
1599 {
1600 int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
1601 struct amd_iommu *iommu;
1602 unsigned long i, old_size;
1603
1604 #ifdef CONFIG_IOMMU_STRESS
1605 populate = false;
1606 #endif
1607
1608 if (index >= APERTURE_MAX_RANGES)
1609 return -ENOMEM;
1610
1611 dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
1612 if (!dma_dom->aperture[index])
1613 return -ENOMEM;
1614
1615 dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
1616 if (!dma_dom->aperture[index]->bitmap)
1617 goto out_free;
1618
1619 dma_dom->aperture[index]->offset = dma_dom->aperture_size;
1620
1621 if (populate) {
1622 unsigned long address = dma_dom->aperture_size;
1623 int i, num_ptes = APERTURE_RANGE_PAGES / 512;
1624 u64 *pte, *pte_page;
1625
1626 for (i = 0; i < num_ptes; ++i) {
1627 pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
1628 &pte_page, gfp);
1629 if (!pte)
1630 goto out_free;
1631
1632 dma_dom->aperture[index]->pte_pages[i] = pte_page;
1633
1634 address += APERTURE_RANGE_SIZE / 64;
1635 }
1636 }
1637
1638 old_size = dma_dom->aperture_size;
1639 dma_dom->aperture_size += APERTURE_RANGE_SIZE;
1640
1641 /* Reserve address range used for MSI messages */
1642 if (old_size < MSI_ADDR_BASE_LO &&
1643 dma_dom->aperture_size > MSI_ADDR_BASE_LO) {
1644 unsigned long spage;
1645 int pages;
1646
1647 pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE);
1648 spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT;
1649
1650 dma_ops_reserve_addresses(dma_dom, spage, pages);
1651 }
1652
1653 /* Initialize the exclusion range if necessary */
1654 for_each_iommu(iommu) {
1655 if (iommu->exclusion_start &&
1656 iommu->exclusion_start >= dma_dom->aperture[index]->offset
1657 && iommu->exclusion_start < dma_dom->aperture_size) {
1658 unsigned long startpage;
1659 int pages = iommu_num_pages(iommu->exclusion_start,
1660 iommu->exclusion_length,
1661 PAGE_SIZE);
1662 startpage = iommu->exclusion_start >> PAGE_SHIFT;
1663 dma_ops_reserve_addresses(dma_dom, startpage, pages);
1664 }
1665 }
1666
1667 /*
1668 * Check for areas already mapped as present in the new aperture
1669 * range and mark those pages as reserved in the allocator. Such
1670 * mappings may already exist as a result of requested unity
1671 * mappings for devices.
1672 */
1673 for (i = dma_dom->aperture[index]->offset;
1674 i < dma_dom->aperture_size;
1675 i += PAGE_SIZE) {
1676 u64 *pte = fetch_pte(&dma_dom->domain, i);
1677 if (!pte || !IOMMU_PTE_PRESENT(*pte))
1678 continue;
1679
1680 dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, 1);
1681 }
1682
1683 update_domain(&dma_dom->domain);
1684
1685 return 0;
1686
1687 out_free:
1688 update_domain(&dma_dom->domain);
1689
1690 free_page((unsigned long)dma_dom->aperture[index]->bitmap);
1691
1692 kfree(dma_dom->aperture[index]);
1693 dma_dom->aperture[index] = NULL;
1694
1695 return -ENOMEM;
1696 }
1697
1698 static unsigned long dma_ops_area_alloc(struct device *dev,
1699 struct dma_ops_domain *dom,
1700 unsigned int pages,
1701 unsigned long align_mask,
1702 u64 dma_mask,
1703 unsigned long start)
1704 {
1705 unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
1706 int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
1707 int i = start >> APERTURE_RANGE_SHIFT;
1708 unsigned long boundary_size;
1709 unsigned long address = -1;
1710 unsigned long limit;
1711
1712 next_bit >>= PAGE_SHIFT;
1713
1714 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
1715 PAGE_SIZE) >> PAGE_SHIFT;
1716
1717 for (;i < max_index; ++i) {
1718 unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
1719
1720 if (dom->aperture[i]->offset >= dma_mask)
1721 break;
1722
1723 limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
1724 dma_mask >> PAGE_SHIFT);
1725
1726 address = iommu_area_alloc(dom->aperture[i]->bitmap,
1727 limit, next_bit, pages, 0,
1728 boundary_size, align_mask);
1729 if (address != -1) {
1730 address = dom->aperture[i]->offset +
1731 (address << PAGE_SHIFT);
1732 dom->next_address = address + (pages << PAGE_SHIFT);
1733 break;
1734 }
1735
1736 next_bit = 0;
1737 }
1738
1739 return address;
1740 }
1741
1742 static unsigned long dma_ops_alloc_addresses(struct device *dev,
1743 struct dma_ops_domain *dom,
1744 unsigned int pages,
1745 unsigned long align_mask,
1746 u64 dma_mask)
1747 {
1748 unsigned long address;
1749
1750 #ifdef CONFIG_IOMMU_STRESS
1751 dom->next_address = 0;
1752 dom->need_flush = true;
1753 #endif
1754
1755 address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1756 dma_mask, dom->next_address);
1757
1758 if (address == -1) {
1759 dom->next_address = 0;
1760 address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1761 dma_mask, 0);
1762 dom->need_flush = true;
1763 }
1764
1765 if (unlikely(address == -1))
1766 address = DMA_ERROR_CODE;
1767
1768 WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
1769
1770 return address;
1771 }
1772
1773 /*
1774 * The address free function.
1775 *
1776 * called with domain->lock held
1777 */
1778 static void dma_ops_free_addresses(struct dma_ops_domain *dom,
1779 unsigned long address,
1780 unsigned int pages)
1781 {
1782 unsigned i = address >> APERTURE_RANGE_SHIFT;
1783 struct aperture_range *range = dom->aperture[i];
1784
1785 BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
1786
1787 #ifdef CONFIG_IOMMU_STRESS
1788 if (i < 4)
1789 return;
1790 #endif
1791
1792 if (address >= dom->next_address)
1793 dom->need_flush = true;
1794
1795 address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
1796
1797 bitmap_clear(range->bitmap, address, pages);
1798
1799 }
1800
1801 /****************************************************************************
1802 *
1803 * The next functions belong to the domain allocation. A domain is
1804 * allocated for every IOMMU as the default domain. If device isolation
1805 * is enabled, every device get its own domain. The most important thing
1806 * about domains is the page table mapping the DMA address space they
1807 * contain.
1808 *
1809 ****************************************************************************/
1810
1811 /*
1812 * This function adds a protection domain to the global protection domain list
1813 */
1814 static void add_domain_to_list(struct protection_domain *domain)
1815 {
1816 unsigned long flags;
1817
1818 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1819 list_add(&domain->list, &amd_iommu_pd_list);
1820 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1821 }
1822
1823 /*
1824 * This function removes a protection domain to the global
1825 * protection domain list
1826 */
1827 static void del_domain_from_list(struct protection_domain *domain)
1828 {
1829 unsigned long flags;
1830
1831 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1832 list_del(&domain->list);
1833 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1834 }
1835
1836 static u16 domain_id_alloc(void)
1837 {
1838 unsigned long flags;
1839 int id;
1840
1841 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1842 id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
1843 BUG_ON(id == 0);
1844 if (id > 0 && id < MAX_DOMAIN_ID)
1845 __set_bit(id, amd_iommu_pd_alloc_bitmap);
1846 else
1847 id = 0;
1848 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1849
1850 return id;
1851 }
1852
1853 static void domain_id_free(int id)
1854 {
1855 unsigned long flags;
1856
1857 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1858 if (id > 0 && id < MAX_DOMAIN_ID)
1859 __clear_bit(id, amd_iommu_pd_alloc_bitmap);
1860 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1861 }
1862
1863 #define DEFINE_FREE_PT_FN(LVL, FN) \
1864 static void free_pt_##LVL (unsigned long __pt) \
1865 { \
1866 unsigned long p; \
1867 u64 *pt; \
1868 int i; \
1869 \
1870 pt = (u64 *)__pt; \
1871 \
1872 for (i = 0; i < 512; ++i) { \
1873 if (!IOMMU_PTE_PRESENT(pt[i])) \
1874 continue; \
1875 \
1876 p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \
1877 FN(p); \
1878 } \
1879 free_page((unsigned long)pt); \
1880 }
1881
1882 DEFINE_FREE_PT_FN(l2, free_page)
1883 DEFINE_FREE_PT_FN(l3, free_pt_l2)
1884 DEFINE_FREE_PT_FN(l4, free_pt_l3)
1885 DEFINE_FREE_PT_FN(l5, free_pt_l4)
1886 DEFINE_FREE_PT_FN(l6, free_pt_l5)
1887
1888 static void free_pagetable(struct protection_domain *domain)
1889 {
1890 unsigned long root = (unsigned long)domain->pt_root;
1891
1892 switch (domain->mode) {
1893 case PAGE_MODE_NONE:
1894 break;
1895 case PAGE_MODE_1_LEVEL:
1896 free_page(root);
1897 break;
1898 case PAGE_MODE_2_LEVEL:
1899 free_pt_l2(root);
1900 break;
1901 case PAGE_MODE_3_LEVEL:
1902 free_pt_l3(root);
1903 break;
1904 case PAGE_MODE_4_LEVEL:
1905 free_pt_l4(root);
1906 break;
1907 case PAGE_MODE_5_LEVEL:
1908 free_pt_l5(root);
1909 break;
1910 case PAGE_MODE_6_LEVEL:
1911 free_pt_l6(root);
1912 break;
1913 default:
1914 BUG();
1915 }
1916 }
1917
1918 static void free_gcr3_tbl_level1(u64 *tbl)
1919 {
1920 u64 *ptr;
1921 int i;
1922
1923 for (i = 0; i < 512; ++i) {
1924 if (!(tbl[i] & GCR3_VALID))
1925 continue;
1926
1927 ptr = __va(tbl[i] & PAGE_MASK);
1928
1929 free_page((unsigned long)ptr);
1930 }
1931 }
1932
1933 static void free_gcr3_tbl_level2(u64 *tbl)
1934 {
1935 u64 *ptr;
1936 int i;
1937
1938 for (i = 0; i < 512; ++i) {
1939 if (!(tbl[i] & GCR3_VALID))
1940 continue;
1941
1942 ptr = __va(tbl[i] & PAGE_MASK);
1943
1944 free_gcr3_tbl_level1(ptr);
1945 }
1946 }
1947
1948 static void free_gcr3_table(struct protection_domain *domain)
1949 {
1950 if (domain->glx == 2)
1951 free_gcr3_tbl_level2(domain->gcr3_tbl);
1952 else if (domain->glx == 1)
1953 free_gcr3_tbl_level1(domain->gcr3_tbl);
1954 else if (domain->glx != 0)
1955 BUG();
1956
1957 free_page((unsigned long)domain->gcr3_tbl);
1958 }
1959
1960 /*
1961 * Free a domain, only used if something went wrong in the
1962 * allocation path and we need to free an already allocated page table
1963 */
1964 static void dma_ops_domain_free(struct dma_ops_domain *dom)
1965 {
1966 int i;
1967
1968 if (!dom)
1969 return;
1970
1971 del_domain_from_list(&dom->domain);
1972
1973 free_pagetable(&dom->domain);
1974
1975 for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
1976 if (!dom->aperture[i])
1977 continue;
1978 free_page((unsigned long)dom->aperture[i]->bitmap);
1979 kfree(dom->aperture[i]);
1980 }
1981
1982 kfree(dom);
1983 }
1984
1985 /*
1986 * Allocates a new protection domain usable for the dma_ops functions.
1987 * It also initializes the page table and the address allocator data
1988 * structures required for the dma_ops interface
1989 */
1990 static struct dma_ops_domain *dma_ops_domain_alloc(void)
1991 {
1992 struct dma_ops_domain *dma_dom;
1993
1994 dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
1995 if (!dma_dom)
1996 return NULL;
1997
1998 spin_lock_init(&dma_dom->domain.lock);
1999
2000 dma_dom->domain.id = domain_id_alloc();
2001 if (dma_dom->domain.id == 0)
2002 goto free_dma_dom;
2003 INIT_LIST_HEAD(&dma_dom->domain.dev_list);
2004 dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
2005 dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
2006 dma_dom->domain.flags = PD_DMA_OPS_MASK;
2007 dma_dom->domain.priv = dma_dom;
2008 if (!dma_dom->domain.pt_root)
2009 goto free_dma_dom;
2010
2011 dma_dom->need_flush = false;
2012 dma_dom->target_dev = 0xffff;
2013
2014 add_domain_to_list(&dma_dom->domain);
2015
2016 if (alloc_new_range(dma_dom, true, GFP_KERNEL))
2017 goto free_dma_dom;
2018
2019 /*
2020 * mark the first page as allocated so we never return 0 as
2021 * a valid dma-address. So we can use 0 as error value
2022 */
2023 dma_dom->aperture[0]->bitmap[0] = 1;
2024 dma_dom->next_address = 0;
2025
2026
2027 return dma_dom;
2028
2029 free_dma_dom:
2030 dma_ops_domain_free(dma_dom);
2031
2032 return NULL;
2033 }
2034
2035 /*
2036 * little helper function to check whether a given protection domain is a
2037 * dma_ops domain
2038 */
2039 static bool dma_ops_domain(struct protection_domain *domain)
2040 {
2041 return domain->flags & PD_DMA_OPS_MASK;
2042 }
2043
2044 static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
2045 {
2046 u64 pte_root = 0;
2047 u64 flags = 0;
2048
2049 if (domain->mode != PAGE_MODE_NONE)
2050 pte_root = virt_to_phys(domain->pt_root);
2051
2052 pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
2053 << DEV_ENTRY_MODE_SHIFT;
2054 pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
2055
2056 flags = amd_iommu_dev_table[devid].data[1];
2057
2058 if (ats)
2059 flags |= DTE_FLAG_IOTLB;
2060
2061 if (domain->flags & PD_IOMMUV2_MASK) {
2062 u64 gcr3 = __pa(domain->gcr3_tbl);
2063 u64 glx = domain->glx;
2064 u64 tmp;
2065
2066 pte_root |= DTE_FLAG_GV;
2067 pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;
2068
2069 /* First mask out possible old values for GCR3 table */
2070 tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
2071 flags &= ~tmp;
2072
2073 tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
2074 flags &= ~tmp;
2075
2076 /* Encode GCR3 table into DTE */
2077 tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
2078 pte_root |= tmp;
2079
2080 tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
2081 flags |= tmp;
2082
2083 tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
2084 flags |= tmp;
2085 }
2086
2087 flags &= ~(0xffffUL);
2088 flags |= domain->id;
2089
2090 amd_iommu_dev_table[devid].data[1] = flags;
2091 amd_iommu_dev_table[devid].data[0] = pte_root;
2092 }
2093
2094 static void clear_dte_entry(u16 devid)
2095 {
2096 /* remove entry from the device table seen by the hardware */
2097 amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
2098 amd_iommu_dev_table[devid].data[1] = 0;
2099
2100 amd_iommu_apply_erratum_63(devid);
2101 }
2102
2103 static void do_attach(struct iommu_dev_data *dev_data,
2104 struct protection_domain *domain)
2105 {
2106 struct amd_iommu *iommu;
2107 bool ats;
2108
2109 iommu = amd_iommu_rlookup_table[dev_data->devid];
2110 ats = dev_data->ats.enabled;
2111
2112 /* Update data structures */
2113 dev_data->domain = domain;
2114 list_add(&dev_data->list, &domain->dev_list);
2115 set_dte_entry(dev_data->devid, domain, ats);
2116
2117 /* Do reference counting */
2118 domain->dev_iommu[iommu->index] += 1;
2119 domain->dev_cnt += 1;
2120
2121 /* Flush the DTE entry */
2122 device_flush_dte(dev_data);
2123 }
2124
2125 static void do_detach(struct iommu_dev_data *dev_data)
2126 {
2127 struct amd_iommu *iommu;
2128
2129 iommu = amd_iommu_rlookup_table[dev_data->devid];
2130
2131 /* decrease reference counters */
2132 dev_data->domain->dev_iommu[iommu->index] -= 1;
2133 dev_data->domain->dev_cnt -= 1;
2134
2135 /* Update data structures */
2136 dev_data->domain = NULL;
2137 list_del(&dev_data->list);
2138 clear_dte_entry(dev_data->devid);
2139
2140 /* Flush the DTE entry */
2141 device_flush_dte(dev_data);
2142 }
2143
2144 /*
2145 * If a device is not yet associated with a domain, this function does
2146 * assigns it visible for the hardware
2147 */
2148 static int __attach_device(struct iommu_dev_data *dev_data,
2149 struct protection_domain *domain)
2150 {
2151 struct iommu_dev_data *head, *entry;
2152 int ret;
2153
2154 /* lock domain */
2155 spin_lock(&domain->lock);
2156
2157 head = dev_data;
2158
2159 if (head->alias_data != NULL)
2160 head = head->alias_data;
2161
2162 /* Now we have the root of the alias group, if any */
2163
2164 ret = -EBUSY;
2165 if (head->domain != NULL)
2166 goto out_unlock;
2167
2168 /* Attach alias group root */
2169 do_attach(head, domain);
2170
2171 /* Attach other devices in the alias group */
2172 list_for_each_entry(entry, &head->alias_list, alias_list)
2173 do_attach(entry, domain);
2174
2175 ret = 0;
2176
2177 out_unlock:
2178
2179 /* ready */
2180 spin_unlock(&domain->lock);
2181
2182 return ret;
2183 }
2184
2185
2186 static void pdev_iommuv2_disable(struct pci_dev *pdev)
2187 {
2188 pci_disable_ats(pdev);
2189 pci_disable_pri(pdev);
2190 pci_disable_pasid(pdev);
2191 }
2192
2193 /* FIXME: Change generic reset-function to do the same */
2194 static int pri_reset_while_enabled(struct pci_dev *pdev)
2195 {
2196 u16 control;
2197 int pos;
2198
2199 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2200 if (!pos)
2201 return -EINVAL;
2202
2203 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
2204 control |= PCI_PRI_CTRL_RESET;
2205 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
2206
2207 return 0;
2208 }
2209
2210 static int pdev_iommuv2_enable(struct pci_dev *pdev)
2211 {
2212 bool reset_enable;
2213 int reqs, ret;
2214
2215 /* FIXME: Hardcode number of outstanding requests for now */
2216 reqs = 32;
2217 if (pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE))
2218 reqs = 1;
2219 reset_enable = pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_ENABLE_RESET);
2220
2221 /* Only allow access to user-accessible pages */
2222 ret = pci_enable_pasid(pdev, 0);
2223 if (ret)
2224 goto out_err;
2225
2226 /* First reset the PRI state of the device */
2227 ret = pci_reset_pri(pdev);
2228 if (ret)
2229 goto out_err;
2230
2231 /* Enable PRI */
2232 ret = pci_enable_pri(pdev, reqs);
2233 if (ret)
2234 goto out_err;
2235
2236 if (reset_enable) {
2237 ret = pri_reset_while_enabled(pdev);
2238 if (ret)
2239 goto out_err;
2240 }
2241
2242 ret = pci_enable_ats(pdev, PAGE_SHIFT);
2243 if (ret)
2244 goto out_err;
2245
2246 return 0;
2247
2248 out_err:
2249 pci_disable_pri(pdev);
2250 pci_disable_pasid(pdev);
2251
2252 return ret;
2253 }
2254
2255 /* FIXME: Move this to PCI code */
2256 #define PCI_PRI_TLP_OFF (1 << 15)
2257
2258 static bool pci_pri_tlp_required(struct pci_dev *pdev)
2259 {
2260 u16 status;
2261 int pos;
2262
2263 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2264 if (!pos)
2265 return false;
2266
2267 pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
2268
2269 return (status & PCI_PRI_TLP_OFF) ? true : false;
2270 }
2271
2272 /*
2273 * If a device is not yet associated with a domain, this function
2274 * assigns it visible for the hardware
2275 */
2276 static int attach_device(struct device *dev,
2277 struct protection_domain *domain)
2278 {
2279 struct pci_dev *pdev = to_pci_dev(dev);
2280 struct iommu_dev_data *dev_data;
2281 unsigned long flags;
2282 int ret;
2283
2284 dev_data = get_dev_data(dev);
2285
2286 if (domain->flags & PD_IOMMUV2_MASK) {
2287 if (!dev_data->iommu_v2 || !dev_data->passthrough)
2288 return -EINVAL;
2289
2290 if (pdev_iommuv2_enable(pdev) != 0)
2291 return -EINVAL;
2292
2293 dev_data->ats.enabled = true;
2294 dev_data->ats.qdep = pci_ats_queue_depth(pdev);
2295 dev_data->pri_tlp = pci_pri_tlp_required(pdev);
2296 } else if (amd_iommu_iotlb_sup &&
2297 pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
2298 dev_data->ats.enabled = true;
2299 dev_data->ats.qdep = pci_ats_queue_depth(pdev);
2300 }
2301
2302 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2303 ret = __attach_device(dev_data, domain);
2304 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2305
2306 /*
2307 * We might boot into a crash-kernel here. The crashed kernel
2308 * left the caches in the IOMMU dirty. So we have to flush
2309 * here to evict all dirty stuff.
2310 */
2311 domain_flush_tlb_pde(domain);
2312
2313 return ret;
2314 }
2315
2316 /*
2317 * Removes a device from a protection domain (unlocked)
2318 */
2319 static void __detach_device(struct iommu_dev_data *dev_data)
2320 {
2321 struct iommu_dev_data *head, *entry;
2322 struct protection_domain *domain;
2323 unsigned long flags;
2324
2325 BUG_ON(!dev_data->domain);
2326
2327 domain = dev_data->domain;
2328
2329 spin_lock_irqsave(&domain->lock, flags);
2330
2331 head = dev_data;
2332 if (head->alias_data != NULL)
2333 head = head->alias_data;
2334
2335 list_for_each_entry(entry, &head->alias_list, alias_list)
2336 do_detach(entry);
2337
2338 do_detach(head);
2339
2340 spin_unlock_irqrestore(&domain->lock, flags);
2341
2342 /*
2343 * If we run in passthrough mode the device must be assigned to the
2344 * passthrough domain if it is detached from any other domain.
2345 * Make sure we can deassign from the pt_domain itself.
2346 */
2347 if (dev_data->passthrough &&
2348 (dev_data->domain == NULL && domain != pt_domain))
2349 __attach_device(dev_data, pt_domain);
2350 }
2351
2352 /*
2353 * Removes a device from a protection domain (with devtable_lock held)
2354 */
2355 static void detach_device(struct device *dev)
2356 {
2357 struct protection_domain *domain;
2358 struct iommu_dev_data *dev_data;
2359 unsigned long flags;
2360
2361 dev_data = get_dev_data(dev);
2362 domain = dev_data->domain;
2363
2364 /* lock device table */
2365 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2366 __detach_device(dev_data);
2367 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2368
2369 if (domain->flags & PD_IOMMUV2_MASK)
2370 pdev_iommuv2_disable(to_pci_dev(dev));
2371 else if (dev_data->ats.enabled)
2372 pci_disable_ats(to_pci_dev(dev));
2373
2374 dev_data->ats.enabled = false;
2375 }
2376
2377 /*
2378 * Find out the protection domain structure for a given PCI device. This
2379 * will give us the pointer to the page table root for example.
2380 */
2381 static struct protection_domain *domain_for_device(struct device *dev)
2382 {
2383 struct iommu_dev_data *dev_data;
2384 struct protection_domain *dom = NULL;
2385 unsigned long flags;
2386
2387 dev_data = get_dev_data(dev);
2388
2389 if (dev_data->domain)
2390 return dev_data->domain;
2391
2392 if (dev_data->alias_data != NULL) {
2393 struct iommu_dev_data *alias_data = dev_data->alias_data;
2394
2395 read_lock_irqsave(&amd_iommu_devtable_lock, flags);
2396 if (alias_data->domain != NULL) {
2397 __attach_device(dev_data, alias_data->domain);
2398 dom = alias_data->domain;
2399 }
2400 read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2401 }
2402
2403 return dom;
2404 }
2405
2406 static int device_change_notifier(struct notifier_block *nb,
2407 unsigned long action, void *data)
2408 {
2409 struct dma_ops_domain *dma_domain;
2410 struct protection_domain *domain;
2411 struct iommu_dev_data *dev_data;
2412 struct device *dev = data;
2413 struct amd_iommu *iommu;
2414 unsigned long flags;
2415 u16 devid;
2416
2417 if (!check_device(dev))
2418 return 0;
2419
2420 devid = get_device_id(dev);
2421 iommu = amd_iommu_rlookup_table[devid];
2422 dev_data = get_dev_data(dev);
2423
2424 switch (action) {
2425 case BUS_NOTIFY_UNBOUND_DRIVER:
2426
2427 domain = domain_for_device(dev);
2428
2429 if (!domain)
2430 goto out;
2431 if (dev_data->passthrough)
2432 break;
2433 detach_device(dev);
2434 break;
2435 case BUS_NOTIFY_ADD_DEVICE:
2436
2437 iommu_init_device(dev);
2438 init_iommu_group(dev);
2439
2440 /*
2441 * dev_data is still NULL and
2442 * got initialized in iommu_init_device
2443 */
2444 dev_data = get_dev_data(dev);
2445
2446 if (iommu_pass_through || dev_data->iommu_v2) {
2447 dev_data->passthrough = true;
2448 attach_device(dev, pt_domain);
2449 break;
2450 }
2451
2452 domain = domain_for_device(dev);
2453
2454 /* allocate a protection domain if a device is added */
2455 dma_domain = find_protection_domain(devid);
2456 if (!dma_domain) {
2457 dma_domain = dma_ops_domain_alloc();
2458 if (!dma_domain)
2459 goto out;
2460 dma_domain->target_dev = devid;
2461
2462 spin_lock_irqsave(&iommu_pd_list_lock, flags);
2463 list_add_tail(&dma_domain->list, &iommu_pd_list);
2464 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
2465 }
2466
2467 dev->archdata.dma_ops = &amd_iommu_dma_ops;
2468
2469 break;
2470 case BUS_NOTIFY_DEL_DEVICE:
2471
2472 iommu_uninit_device(dev);
2473
2474 default:
2475 goto out;
2476 }
2477
2478 iommu_completion_wait(iommu);
2479
2480 out:
2481 return 0;
2482 }
2483
2484 static struct notifier_block device_nb = {
2485 .notifier_call = device_change_notifier,
2486 };
2487
2488 void amd_iommu_init_notifier(void)
2489 {
2490 bus_register_notifier(&pci_bus_type, &device_nb);
2491 }
2492
2493 /*****************************************************************************
2494 *
2495 * The next functions belong to the dma_ops mapping/unmapping code.
2496 *
2497 *****************************************************************************/
2498
2499 /*
2500 * In the dma_ops path we only have the struct device. This function
2501 * finds the corresponding IOMMU, the protection domain and the
2502 * requestor id for a given device.
2503 * If the device is not yet associated with a domain this is also done
2504 * in this function.
2505 */
2506 static struct protection_domain *get_domain(struct device *dev)
2507 {
2508 struct protection_domain *domain;
2509 struct dma_ops_domain *dma_dom;
2510 u16 devid = get_device_id(dev);
2511
2512 if (!check_device(dev))
2513 return ERR_PTR(-EINVAL);
2514
2515 domain = domain_for_device(dev);
2516 if (domain != NULL && !dma_ops_domain(domain))
2517 return ERR_PTR(-EBUSY);
2518
2519 if (domain != NULL)
2520 return domain;
2521
2522 /* Device not bound yet - bind it */
2523 dma_dom = find_protection_domain(devid);
2524 if (!dma_dom)
2525 dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
2526 attach_device(dev, &dma_dom->domain);
2527 DUMP_printk("Using protection domain %d for device %s\n",
2528 dma_dom->domain.id, dev_name(dev));
2529
2530 return &dma_dom->domain;
2531 }
2532
2533 static void update_device_table(struct protection_domain *domain)
2534 {
2535 struct iommu_dev_data *dev_data;
2536
2537 list_for_each_entry(dev_data, &domain->dev_list, list)
2538 set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
2539 }
2540
2541 static void update_domain(struct protection_domain *domain)
2542 {
2543 if (!domain->updated)
2544 return;
2545
2546 update_device_table(domain);
2547
2548 domain_flush_devices(domain);
2549 domain_flush_tlb_pde(domain);
2550
2551 domain->updated = false;
2552 }
2553
2554 /*
2555 * This function fetches the PTE for a given address in the aperture
2556 */
2557 static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
2558 unsigned long address)
2559 {
2560 struct aperture_range *aperture;
2561 u64 *pte, *pte_page;
2562
2563 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
2564 if (!aperture)
2565 return NULL;
2566
2567 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2568 if (!pte) {
2569 pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
2570 GFP_ATOMIC);
2571 aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
2572 } else
2573 pte += PM_LEVEL_INDEX(0, address);
2574
2575 update_domain(&dom->domain);
2576
2577 return pte;
2578 }
2579
2580 /*
2581 * This is the generic map function. It maps one 4kb page at paddr to
2582 * the given address in the DMA address space for the domain.
2583 */
2584 static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
2585 unsigned long address,
2586 phys_addr_t paddr,
2587 int direction)
2588 {
2589 u64 *pte, __pte;
2590
2591 WARN_ON(address > dom->aperture_size);
2592
2593 paddr &= PAGE_MASK;
2594
2595 pte = dma_ops_get_pte(dom, address);
2596 if (!pte)
2597 return DMA_ERROR_CODE;
2598
2599 __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
2600
2601 if (direction == DMA_TO_DEVICE)
2602 __pte |= IOMMU_PTE_IR;
2603 else if (direction == DMA_FROM_DEVICE)
2604 __pte |= IOMMU_PTE_IW;
2605 else if (direction == DMA_BIDIRECTIONAL)
2606 __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
2607
2608 WARN_ON(*pte);
2609
2610 *pte = __pte;
2611
2612 return (dma_addr_t)address;
2613 }
2614
2615 /*
2616 * The generic unmapping function for on page in the DMA address space.
2617 */
2618 static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
2619 unsigned long address)
2620 {
2621 struct aperture_range *aperture;
2622 u64 *pte;
2623
2624 if (address >= dom->aperture_size)
2625 return;
2626
2627 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
2628 if (!aperture)
2629 return;
2630
2631 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2632 if (!pte)
2633 return;
2634
2635 pte += PM_LEVEL_INDEX(0, address);
2636
2637 WARN_ON(!*pte);
2638
2639 *pte = 0ULL;
2640 }
2641
2642 /*
2643 * This function contains common code for mapping of a physically
2644 * contiguous memory region into DMA address space. It is used by all
2645 * mapping functions provided with this IOMMU driver.
2646 * Must be called with the domain lock held.
2647 */
2648 static dma_addr_t __map_single(struct device *dev,
2649 struct dma_ops_domain *dma_dom,
2650 phys_addr_t paddr,
2651 size_t size,
2652 int dir,
2653 bool align,
2654 u64 dma_mask)
2655 {
2656 dma_addr_t offset = paddr & ~PAGE_MASK;
2657 dma_addr_t address, start, ret;
2658 unsigned int pages;
2659 unsigned long align_mask = 0;
2660 int i;
2661
2662 pages = iommu_num_pages(paddr, size, PAGE_SIZE);
2663 paddr &= PAGE_MASK;
2664
2665 INC_STATS_COUNTER(total_map_requests);
2666
2667 if (pages > 1)
2668 INC_STATS_COUNTER(cross_page);
2669
2670 if (align)
2671 align_mask = (1UL << get_order(size)) - 1;
2672
2673 retry:
2674 address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
2675 dma_mask);
2676 if (unlikely(address == DMA_ERROR_CODE)) {
2677 /*
2678 * setting next_address here will let the address
2679 * allocator only scan the new allocated range in the
2680 * first run. This is a small optimization.
2681 */
2682 dma_dom->next_address = dma_dom->aperture_size;
2683
2684 if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
2685 goto out;
2686
2687 /*
2688 * aperture was successfully enlarged by 128 MB, try
2689 * allocation again
2690 */
2691 goto retry;
2692 }
2693
2694 start = address;
2695 for (i = 0; i < pages; ++i) {
2696 ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
2697 if (ret == DMA_ERROR_CODE)
2698 goto out_unmap;
2699
2700 paddr += PAGE_SIZE;
2701 start += PAGE_SIZE;
2702 }
2703 address += offset;
2704
2705 ADD_STATS_COUNTER(alloced_io_mem, size);
2706
2707 if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
2708 domain_flush_tlb(&dma_dom->domain);
2709 dma_dom->need_flush = false;
2710 } else if (unlikely(amd_iommu_np_cache))
2711 domain_flush_pages(&dma_dom->domain, address, size);
2712
2713 out:
2714 return address;
2715
2716 out_unmap:
2717
2718 for (--i; i >= 0; --i) {
2719 start -= PAGE_SIZE;
2720 dma_ops_domain_unmap(dma_dom, start);
2721 }
2722
2723 dma_ops_free_addresses(dma_dom, address, pages);
2724
2725 return DMA_ERROR_CODE;
2726 }
2727
2728 /*
2729 * Does the reverse of the __map_single function. Must be called with
2730 * the domain lock held too
2731 */
2732 static void __unmap_single(struct dma_ops_domain *dma_dom,
2733 dma_addr_t dma_addr,
2734 size_t size,
2735 int dir)
2736 {
2737 dma_addr_t flush_addr;
2738 dma_addr_t i, start;
2739 unsigned int pages;
2740
2741 if ((dma_addr == DMA_ERROR_CODE) ||
2742 (dma_addr + size > dma_dom->aperture_size))
2743 return;
2744
2745 flush_addr = dma_addr;
2746 pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
2747 dma_addr &= PAGE_MASK;
2748 start = dma_addr;
2749
2750 for (i = 0; i < pages; ++i) {
2751 dma_ops_domain_unmap(dma_dom, start);
2752 start += PAGE_SIZE;
2753 }
2754
2755 SUB_STATS_COUNTER(alloced_io_mem, size);
2756
2757 dma_ops_free_addresses(dma_dom, dma_addr, pages);
2758
2759 if (amd_iommu_unmap_flush || dma_dom->need_flush) {
2760 domain_flush_pages(&dma_dom->domain, flush_addr, size);
2761 dma_dom->need_flush = false;
2762 }
2763 }
2764
2765 /*
2766 * The exported map_single function for dma_ops.
2767 */
2768 static dma_addr_t map_page(struct device *dev, struct page *page,
2769 unsigned long offset, size_t size,
2770 enum dma_data_direction dir,
2771 struct dma_attrs *attrs)
2772 {
2773 unsigned long flags;
2774 struct protection_domain *domain;
2775 dma_addr_t addr;
2776 u64 dma_mask;
2777 phys_addr_t paddr = page_to_phys(page) + offset;
2778
2779 INC_STATS_COUNTER(cnt_map_single);
2780
2781 domain = get_domain(dev);
2782 if (PTR_ERR(domain) == -EINVAL)
2783 return (dma_addr_t)paddr;
2784 else if (IS_ERR(domain))
2785 return DMA_ERROR_CODE;
2786
2787 dma_mask = *dev->dma_mask;
2788
2789 spin_lock_irqsave(&domain->lock, flags);
2790
2791 addr = __map_single(dev, domain->priv, paddr, size, dir, false,
2792 dma_mask);
2793 if (addr == DMA_ERROR_CODE)
2794 goto out;
2795
2796 domain_flush_complete(domain);
2797
2798 out:
2799 spin_unlock_irqrestore(&domain->lock, flags);
2800
2801 return addr;
2802 }
2803
2804 /*
2805 * The exported unmap_single function for dma_ops.
2806 */
2807 static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
2808 enum dma_data_direction dir, struct dma_attrs *attrs)
2809 {
2810 unsigned long flags;
2811 struct protection_domain *domain;
2812
2813 INC_STATS_COUNTER(cnt_unmap_single);
2814
2815 domain = get_domain(dev);
2816 if (IS_ERR(domain))
2817 return;
2818
2819 spin_lock_irqsave(&domain->lock, flags);
2820
2821 __unmap_single(domain->priv, dma_addr, size, dir);
2822
2823 domain_flush_complete(domain);
2824
2825 spin_unlock_irqrestore(&domain->lock, flags);
2826 }
2827
2828 /*
2829 * The exported map_sg function for dma_ops (handles scatter-gather
2830 * lists).
2831 */
2832 static int map_sg(struct device *dev, struct scatterlist *sglist,
2833 int nelems, enum dma_data_direction dir,
2834 struct dma_attrs *attrs)
2835 {
2836 unsigned long flags;
2837 struct protection_domain *domain;
2838 int i;
2839 struct scatterlist *s;
2840 phys_addr_t paddr;
2841 int mapped_elems = 0;
2842 u64 dma_mask;
2843
2844 INC_STATS_COUNTER(cnt_map_sg);
2845
2846 domain = get_domain(dev);
2847 if (IS_ERR(domain))
2848 return 0;
2849
2850 dma_mask = *dev->dma_mask;
2851
2852 spin_lock_irqsave(&domain->lock, flags);
2853
2854 for_each_sg(sglist, s, nelems, i) {
2855 paddr = sg_phys(s);
2856
2857 s->dma_address = __map_single(dev, domain->priv,
2858 paddr, s->length, dir, false,
2859 dma_mask);
2860
2861 if (s->dma_address) {
2862 s->dma_length = s->length;
2863 mapped_elems++;
2864 } else
2865 goto unmap;
2866 }
2867
2868 domain_flush_complete(domain);
2869
2870 out:
2871 spin_unlock_irqrestore(&domain->lock, flags);
2872
2873 return mapped_elems;
2874 unmap:
2875 for_each_sg(sglist, s, mapped_elems, i) {
2876 if (s->dma_address)
2877 __unmap_single(domain->priv, s->dma_address,
2878 s->dma_length, dir);
2879 s->dma_address = s->dma_length = 0;
2880 }
2881
2882 mapped_elems = 0;
2883
2884 goto out;
2885 }
2886
2887 /*
2888 * The exported map_sg function for dma_ops (handles scatter-gather
2889 * lists).
2890 */
2891 static void unmap_sg(struct device *dev, struct scatterlist *sglist,
2892 int nelems, enum dma_data_direction dir,
2893 struct dma_attrs *attrs)
2894 {
2895 unsigned long flags;
2896 struct protection_domain *domain;
2897 struct scatterlist *s;
2898 int i;
2899
2900 INC_STATS_COUNTER(cnt_unmap_sg);
2901
2902 domain = get_domain(dev);
2903 if (IS_ERR(domain))
2904 return;
2905
2906 spin_lock_irqsave(&domain->lock, flags);
2907
2908 for_each_sg(sglist, s, nelems, i) {
2909 __unmap_single(domain->priv, s->dma_address,
2910 s->dma_length, dir);
2911 s->dma_address = s->dma_length = 0;
2912 }
2913
2914 domain_flush_complete(domain);
2915
2916 spin_unlock_irqrestore(&domain->lock, flags);
2917 }
2918
2919 /*
2920 * The exported alloc_coherent function for dma_ops.
2921 */
2922 static void *alloc_coherent(struct device *dev, size_t size,
2923 dma_addr_t *dma_addr, gfp_t flag,
2924 struct dma_attrs *attrs)
2925 {
2926 unsigned long flags;
2927 void *virt_addr;
2928 struct protection_domain *domain;
2929 phys_addr_t paddr;
2930 u64 dma_mask = dev->coherent_dma_mask;
2931
2932 INC_STATS_COUNTER(cnt_alloc_coherent);
2933
2934 domain = get_domain(dev);
2935 if (PTR_ERR(domain) == -EINVAL) {
2936 virt_addr = (void *)__get_free_pages(flag, get_order(size));
2937 *dma_addr = __pa(virt_addr);
2938 return virt_addr;
2939 } else if (IS_ERR(domain))
2940 return NULL;
2941
2942 dma_mask = dev->coherent_dma_mask;
2943 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
2944 flag |= __GFP_ZERO;
2945
2946 virt_addr = (void *)__get_free_pages(flag, get_order(size));
2947 if (!virt_addr)
2948 return NULL;
2949
2950 paddr = virt_to_phys(virt_addr);
2951
2952 if (!dma_mask)
2953 dma_mask = *dev->dma_mask;
2954
2955 spin_lock_irqsave(&domain->lock, flags);
2956
2957 *dma_addr = __map_single(dev, domain->priv, paddr,
2958 size, DMA_BIDIRECTIONAL, true, dma_mask);
2959
2960 if (*dma_addr == DMA_ERROR_CODE) {
2961 spin_unlock_irqrestore(&domain->lock, flags);
2962 goto out_free;
2963 }
2964
2965 domain_flush_complete(domain);
2966
2967 spin_unlock_irqrestore(&domain->lock, flags);
2968
2969 return virt_addr;
2970
2971 out_free:
2972
2973 free_pages((unsigned long)virt_addr, get_order(size));
2974
2975 return NULL;
2976 }
2977
2978 /*
2979 * The exported free_coherent function for dma_ops.
2980 */
2981 static void free_coherent(struct device *dev, size_t size,
2982 void *virt_addr, dma_addr_t dma_addr,
2983 struct dma_attrs *attrs)
2984 {
2985 unsigned long flags;
2986 struct protection_domain *domain;
2987
2988 INC_STATS_COUNTER(cnt_free_coherent);
2989
2990 domain = get_domain(dev);
2991 if (IS_ERR(domain))
2992 goto free_mem;
2993
2994 spin_lock_irqsave(&domain->lock, flags);
2995
2996 __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
2997
2998 domain_flush_complete(domain);
2999
3000 spin_unlock_irqrestore(&domain->lock, flags);
3001
3002 free_mem:
3003 free_pages((unsigned long)virt_addr, get_order(size));
3004 }
3005
3006 /*
3007 * This function is called by the DMA layer to find out if we can handle a
3008 * particular device. It is part of the dma_ops.
3009 */
3010 static int amd_iommu_dma_supported(struct device *dev, u64 mask)
3011 {
3012 return check_device(dev);
3013 }
3014
3015 /*
3016 * The function for pre-allocating protection domains.
3017 *
3018 * If the driver core informs the DMA layer if a driver grabs a device
3019 * we don't need to preallocate the protection domains anymore.
3020 * For now we have to.
3021 */
3022 static void __init prealloc_protection_domains(void)
3023 {
3024 struct iommu_dev_data *dev_data;
3025 struct dma_ops_domain *dma_dom;
3026 struct pci_dev *dev = NULL;
3027 u16 devid;
3028
3029 for_each_pci_dev(dev) {
3030
3031 /* Do we handle this device? */
3032 if (!check_device(&dev->dev))
3033 continue;
3034
3035 dev_data = get_dev_data(&dev->dev);
3036 if (!amd_iommu_force_isolation && dev_data->iommu_v2) {
3037 /* Make sure passthrough domain is allocated */
3038 alloc_passthrough_domain();
3039 dev_data->passthrough = true;
3040 attach_device(&dev->dev, pt_domain);
3041 pr_info("AMD-Vi: Using passthrough domain for device %s\n",
3042 dev_name(&dev->dev));
3043 }
3044
3045 /* Is there already any domain for it? */
3046 if (domain_for_device(&dev->dev))
3047 continue;
3048
3049 devid = get_device_id(&dev->dev);
3050
3051 dma_dom = dma_ops_domain_alloc();
3052 if (!dma_dom)
3053 continue;
3054 init_unity_mappings_for_device(dma_dom, devid);
3055 dma_dom->target_dev = devid;
3056
3057 attach_device(&dev->dev, &dma_dom->domain);
3058
3059 list_add_tail(&dma_dom->list, &iommu_pd_list);
3060 }
3061 }
3062
3063 static struct dma_map_ops amd_iommu_dma_ops = {
3064 .alloc = alloc_coherent,
3065 .free = free_coherent,
3066 .map_page = map_page,
3067 .unmap_page = unmap_page,
3068 .map_sg = map_sg,
3069 .unmap_sg = unmap_sg,
3070 .dma_supported = amd_iommu_dma_supported,
3071 };
3072
3073 static unsigned device_dma_ops_init(void)
3074 {
3075 struct iommu_dev_data *dev_data;
3076 struct pci_dev *pdev = NULL;
3077 unsigned unhandled = 0;
3078
3079 for_each_pci_dev(pdev) {
3080 if (!check_device(&pdev->dev)) {
3081
3082 iommu_ignore_device(&pdev->dev);
3083
3084 unhandled += 1;
3085 continue;
3086 }
3087
3088 dev_data = get_dev_data(&pdev->dev);
3089
3090 if (!dev_data->passthrough)
3091 pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
3092 else
3093 pdev->dev.archdata.dma_ops = &nommu_dma_ops;
3094 }
3095
3096 return unhandled;
3097 }
3098
3099 /*
3100 * The function which clues the AMD IOMMU driver into dma_ops.
3101 */
3102
3103 void __init amd_iommu_init_api(void)
3104 {
3105 bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
3106 }
3107
3108 int __init amd_iommu_init_dma_ops(void)
3109 {
3110 struct amd_iommu *iommu;
3111 int ret, unhandled;
3112
3113 /*
3114 * first allocate a default protection domain for every IOMMU we
3115 * found in the system. Devices not assigned to any other
3116 * protection domain will be assigned to the default one.
3117 */
3118 for_each_iommu(iommu) {
3119 iommu->default_dom = dma_ops_domain_alloc();
3120 if (iommu->default_dom == NULL)
3121 return -ENOMEM;
3122 iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
3123 ret = iommu_init_unity_mappings(iommu);
3124 if (ret)
3125 goto free_domains;
3126 }
3127
3128 /*
3129 * Pre-allocate the protection domains for each device.
3130 */
3131 prealloc_protection_domains();
3132
3133 iommu_detected = 1;
3134 swiotlb = 0;
3135
3136 /* Make the driver finally visible to the drivers */
3137 unhandled = device_dma_ops_init();
3138 if (unhandled && max_pfn > MAX_DMA32_PFN) {
3139 /* There are unhandled devices - initialize swiotlb for them */
3140 swiotlb = 1;
3141 }
3142
3143 amd_iommu_stats_init();
3144
3145 if (amd_iommu_unmap_flush)
3146 pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n");
3147 else
3148 pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");
3149
3150 return 0;
3151
3152 free_domains:
3153
3154 for_each_iommu(iommu) {
3155 dma_ops_domain_free(iommu->default_dom);
3156 }
3157
3158 return ret;
3159 }
3160
3161 /*****************************************************************************
3162 *
3163 * The following functions belong to the exported interface of AMD IOMMU
3164 *
3165 * This interface allows access to lower level functions of the IOMMU
3166 * like protection domain handling and assignement of devices to domains
3167 * which is not possible with the dma_ops interface.
3168 *
3169 *****************************************************************************/
3170
3171 static void cleanup_domain(struct protection_domain *domain)
3172 {
3173 struct iommu_dev_data *entry;
3174 unsigned long flags;
3175
3176 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
3177
3178 while (!list_empty(&domain->dev_list)) {
3179 entry = list_first_entry(&domain->dev_list,
3180 struct iommu_dev_data, list);
3181 __detach_device(entry);
3182 }
3183
3184 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
3185 }
3186
3187 static void protection_domain_free(struct protection_domain *domain)
3188 {
3189 if (!domain)
3190 return;
3191
3192 del_domain_from_list(domain);
3193
3194 if (domain->id)
3195 domain_id_free(domain->id);
3196
3197 kfree(domain);
3198 }
3199
3200 static struct protection_domain *protection_domain_alloc(void)
3201 {
3202 struct protection_domain *domain;
3203
3204 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
3205 if (!domain)
3206 return NULL;
3207
3208 spin_lock_init(&domain->lock);
3209 mutex_init(&domain->api_lock);
3210 domain->id = domain_id_alloc();
3211 if (!domain->id)
3212 goto out_err;
3213 INIT_LIST_HEAD(&domain->dev_list);
3214
3215 add_domain_to_list(domain);
3216
3217 return domain;
3218
3219 out_err:
3220 kfree(domain);
3221
3222 return NULL;
3223 }
3224
3225 static int __init alloc_passthrough_domain(void)
3226 {
3227 if (pt_domain != NULL)
3228 return 0;
3229
3230 /* allocate passthrough domain */
3231 pt_domain = protection_domain_alloc();
3232 if (!pt_domain)
3233 return -ENOMEM;
3234
3235 pt_domain->mode = PAGE_MODE_NONE;
3236
3237 return 0;
3238 }
3239 static int amd_iommu_domain_init(struct iommu_domain *dom)
3240 {
3241 struct protection_domain *domain;
3242
3243 domain = protection_domain_alloc();
3244 if (!domain)
3245 goto out_free;
3246
3247 domain->mode = PAGE_MODE_3_LEVEL;
3248 domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
3249 if (!domain->pt_root)
3250 goto out_free;
3251
3252 domain->iommu_domain = dom;
3253
3254 dom->priv = domain;
3255
3256 dom->geometry.aperture_start = 0;
3257 dom->geometry.aperture_end = ~0ULL;
3258 dom->geometry.force_aperture = true;
3259
3260 return 0;
3261
3262 out_free:
3263 protection_domain_free(domain);
3264
3265 return -ENOMEM;
3266 }
3267
3268 static void amd_iommu_domain_destroy(struct iommu_domain *dom)
3269 {
3270 struct protection_domain *domain = dom->priv;
3271
3272 if (!domain)
3273 return;
3274
3275 if (domain->dev_cnt > 0)
3276 cleanup_domain(domain);
3277
3278 BUG_ON(domain->dev_cnt != 0);
3279
3280 if (domain->mode != PAGE_MODE_NONE)
3281 free_pagetable(domain);
3282
3283 if (domain->flags & PD_IOMMUV2_MASK)
3284 free_gcr3_table(domain);
3285
3286 protection_domain_free(domain);
3287
3288 dom->priv = NULL;
3289 }
3290
3291 static void amd_iommu_detach_device(struct iommu_domain *dom,
3292 struct device *dev)
3293 {
3294 struct iommu_dev_data *dev_data = dev->archdata.iommu;
3295 struct amd_iommu *iommu;
3296 u16 devid;
3297
3298 if (!check_device(dev))
3299 return;
3300
3301 devid = get_device_id(dev);
3302
3303 if (dev_data->domain != NULL)
3304 detach_device(dev);
3305
3306 iommu = amd_iommu_rlookup_table[devid];
3307 if (!iommu)
3308 return;
3309
3310 iommu_completion_wait(iommu);
3311 }
3312
3313 static int amd_iommu_attach_device(struct iommu_domain *dom,
3314 struct device *dev)
3315 {
3316 struct protection_domain *domain = dom->priv;
3317 struct iommu_dev_data *dev_data;
3318 struct amd_iommu *iommu;
3319 int ret;
3320
3321 if (!check_device(dev))
3322 return -EINVAL;
3323
3324 dev_data = dev->archdata.iommu;
3325
3326 iommu = amd_iommu_rlookup_table[dev_data->devid];
3327 if (!iommu)
3328 return -EINVAL;
3329
3330 if (dev_data->domain)
3331 detach_device(dev);
3332
3333 ret = attach_device(dev, domain);
3334
3335 iommu_completion_wait(iommu);
3336
3337 return ret;
3338 }
3339
3340 static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
3341 phys_addr_t paddr, size_t page_size, int iommu_prot)
3342 {
3343 struct protection_domain *domain = dom->priv;
3344 int prot = 0;
3345 int ret;
3346
3347 if (domain->mode == PAGE_MODE_NONE)
3348 return -EINVAL;
3349
3350 if (iommu_prot & IOMMU_READ)
3351 prot |= IOMMU_PROT_IR;
3352 if (iommu_prot & IOMMU_WRITE)
3353 prot |= IOMMU_PROT_IW;
3354
3355 mutex_lock(&domain->api_lock);
3356 ret = iommu_map_page(domain, iova, paddr, prot, page_size);
3357 mutex_unlock(&domain->api_lock);
3358
3359 return ret;
3360 }
3361
3362 static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
3363 size_t page_size)
3364 {
3365 struct protection_domain *domain = dom->priv;
3366 size_t unmap_size;
3367
3368 if (domain->mode == PAGE_MODE_NONE)
3369 return -EINVAL;
3370
3371 mutex_lock(&domain->api_lock);
3372 unmap_size = iommu_unmap_page(domain, iova, page_size);
3373 mutex_unlock(&domain->api_lock);
3374
3375 domain_flush_tlb_pde(domain);
3376
3377 return unmap_size;
3378 }
3379
3380 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
3381 dma_addr_t iova)
3382 {
3383 struct protection_domain *domain = dom->priv;
3384 unsigned long offset_mask;
3385 phys_addr_t paddr;
3386 u64 *pte, __pte;
3387
3388 if (domain->mode == PAGE_MODE_NONE)
3389 return iova;
3390
3391 pte = fetch_pte(domain, iova);
3392
3393 if (!pte || !IOMMU_PTE_PRESENT(*pte))
3394 return 0;
3395
3396 if (PM_PTE_LEVEL(*pte) == 0)
3397 offset_mask = PAGE_SIZE - 1;
3398 else
3399 offset_mask = PTE_PAGE_SIZE(*pte) - 1;
3400
3401 __pte = *pte & PM_ADDR_MASK;
3402 paddr = (__pte & ~offset_mask) | (iova & offset_mask);
3403
3404 return paddr;
3405 }
3406
3407 static bool amd_iommu_capable(enum iommu_cap cap)
3408 {
3409 switch (cap) {
3410 case IOMMU_CAP_CACHE_COHERENCY:
3411 return true;
3412 case IOMMU_CAP_INTR_REMAP:
3413 return (irq_remapping_enabled == 1);
3414 }
3415
3416 return false;
3417 }
3418
3419 static const struct iommu_ops amd_iommu_ops = {
3420 .capable = amd_iommu_capable,
3421 .domain_init = amd_iommu_domain_init,
3422 .domain_destroy = amd_iommu_domain_destroy,
3423 .attach_dev = amd_iommu_attach_device,
3424 .detach_dev = amd_iommu_detach_device,
3425 .map = amd_iommu_map,
3426 .unmap = amd_iommu_unmap,
3427 .iova_to_phys = amd_iommu_iova_to_phys,
3428 .pgsize_bitmap = AMD_IOMMU_PGSIZES,
3429 };
3430
3431 /*****************************************************************************
3432 *
3433 * The next functions do a basic initialization of IOMMU for pass through
3434 * mode
3435 *
3436 * In passthrough mode the IOMMU is initialized and enabled but not used for
3437 * DMA-API translation.
3438 *
3439 *****************************************************************************/
3440
3441 int __init amd_iommu_init_passthrough(void)
3442 {
3443 struct iommu_dev_data *dev_data;
3444 struct pci_dev *dev = NULL;
3445 int ret;
3446
3447 ret = alloc_passthrough_domain();
3448 if (ret)
3449 return ret;
3450
3451 for_each_pci_dev(dev) {
3452 if (!check_device(&dev->dev))
3453 continue;
3454
3455 dev_data = get_dev_data(&dev->dev);
3456 dev_data->passthrough = true;
3457
3458 attach_device(&dev->dev, pt_domain);
3459 }
3460
3461 amd_iommu_stats_init();
3462
3463 pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
3464
3465 return 0;
3466 }
3467
3468 /* IOMMUv2 specific functions */
3469 int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
3470 {
3471 return atomic_notifier_chain_register(&ppr_notifier, nb);
3472 }
3473 EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);
3474
3475 int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)
3476 {
3477 return atomic_notifier_chain_unregister(&ppr_notifier, nb);
3478 }
3479 EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
3480
3481 void amd_iommu_domain_direct_map(struct iommu_domain *dom)
3482 {
3483 struct protection_domain *domain = dom->priv;
3484 unsigned long flags;
3485
3486 spin_lock_irqsave(&domain->lock, flags);
3487
3488 /* Update data structure */
3489 domain->mode = PAGE_MODE_NONE;
3490 domain->updated = true;
3491
3492 /* Make changes visible to IOMMUs */
3493 update_domain(domain);
3494
3495 /* Page-table is not visible to IOMMU anymore, so free it */
3496 free_pagetable(domain);
3497
3498 spin_unlock_irqrestore(&domain->lock, flags);
3499 }
3500 EXPORT_SYMBOL(amd_iommu_domain_direct_map);
3501
3502 int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
3503 {
3504 struct protection_domain *domain = dom->priv;
3505 unsigned long flags;
3506 int levels, ret;
3507
3508 if (pasids <= 0 || pasids > (PASID_MASK + 1))
3509 return -EINVAL;
3510
3511 /* Number of GCR3 table levels required */
3512 for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
3513 levels += 1;
3514
3515 if (levels > amd_iommu_max_glx_val)
3516 return -EINVAL;
3517
3518 spin_lock_irqsave(&domain->lock, flags);
3519
3520 /*
3521 * Save us all sanity checks whether devices already in the
3522 * domain support IOMMUv2. Just force that the domain has no
3523 * devices attached when it is switched into IOMMUv2 mode.
3524 */
3525 ret = -EBUSY;
3526 if (domain->dev_cnt > 0 || domain->flags & PD_IOMMUV2_MASK)
3527 goto out;
3528
3529 ret = -ENOMEM;
3530 domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
3531 if (domain->gcr3_tbl == NULL)
3532 goto out;
3533
3534 domain->glx = levels;
3535 domain->flags |= PD_IOMMUV2_MASK;
3536 domain->updated = true;
3537
3538 update_domain(domain);
3539
3540 ret = 0;
3541
3542 out:
3543 spin_unlock_irqrestore(&domain->lock, flags);
3544
3545 return ret;
3546 }
3547 EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
3548
3549 static int __flush_pasid(struct protection_domain *domain, int pasid,
3550 u64 address, bool size)
3551 {
3552 struct iommu_dev_data *dev_data;
3553 struct iommu_cmd cmd;
3554 int i, ret;
3555
3556 if (!(domain->flags & PD_IOMMUV2_MASK))
3557 return -EINVAL;
3558
3559 build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);
3560
3561 /*
3562 * IOMMU TLB needs to be flushed before Device TLB to
3563 * prevent device TLB refill from IOMMU TLB
3564 */
3565 for (i = 0; i < amd_iommus_present; ++i) {
3566 if (domain->dev_iommu[i] == 0)
3567 continue;
3568
3569 ret = iommu_queue_command(amd_iommus[i], &cmd);
3570 if (ret != 0)
3571 goto out;
3572 }
3573
3574 /* Wait until IOMMU TLB flushes are complete */
3575 domain_flush_complete(domain);
3576
3577 /* Now flush device TLBs */
3578 list_for_each_entry(dev_data, &domain->dev_list, list) {
3579 struct amd_iommu *iommu;
3580 int qdep;
3581
3582 BUG_ON(!dev_data->ats.enabled);
3583
3584 qdep = dev_data->ats.qdep;
3585 iommu = amd_iommu_rlookup_table[dev_data->devid];
3586
3587 build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,
3588 qdep, address, size);
3589
3590 ret = iommu_queue_command(iommu, &cmd);
3591 if (ret != 0)
3592 goto out;
3593 }
3594
3595 /* Wait until all device TLBs are flushed */
3596 domain_flush_complete(domain);
3597
3598 ret = 0;
3599
3600 out:
3601
3602 return ret;
3603 }
3604
3605 static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
3606 u64 address)
3607 {
3608 INC_STATS_COUNTER(invalidate_iotlb);
3609
3610 return __flush_pasid(domain, pasid, address, false);
3611 }
3612
3613 int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
3614 u64 address)
3615 {
3616 struct protection_domain *domain = dom->priv;
3617 unsigned long flags;
3618 int ret;
3619
3620 spin_lock_irqsave(&domain->lock, flags);
3621 ret = __amd_iommu_flush_page(domain, pasid, address);
3622 spin_unlock_irqrestore(&domain->lock, flags);
3623
3624 return ret;
3625 }
3626 EXPORT_SYMBOL(amd_iommu_flush_page);
3627
3628 static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
3629 {
3630 INC_STATS_COUNTER(invalidate_iotlb_all);
3631
3632 return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
3633 true);
3634 }
3635
3636 int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)
3637 {
3638 struct protection_domain *domain = dom->priv;
3639 unsigned long flags;
3640 int ret;
3641
3642 spin_lock_irqsave(&domain->lock, flags);
3643 ret = __amd_iommu_flush_tlb(domain, pasid);
3644 spin_unlock_irqrestore(&domain->lock, flags);
3645
3646 return ret;
3647 }
3648 EXPORT_SYMBOL(amd_iommu_flush_tlb);
3649
3650 static u64 *__get_gcr3_pte(u64 *root, int level, int pasid, bool alloc)
3651 {
3652 int index;
3653 u64 *pte;
3654
3655 while (true) {
3656
3657 index = (pasid >> (9 * level)) & 0x1ff;
3658 pte = &root[index];
3659
3660 if (level == 0)
3661 break;
3662
3663 if (!(*pte & GCR3_VALID)) {
3664 if (!alloc)
3665 return NULL;
3666
3667 root = (void *)get_zeroed_page(GFP_ATOMIC);
3668 if (root == NULL)
3669 return NULL;
3670
3671 *pte = __pa(root) | GCR3_VALID;
3672 }
3673
3674 root = __va(*pte & PAGE_MASK);
3675
3676 level -= 1;
3677 }
3678
3679 return pte;
3680 }
3681
3682 static int __set_gcr3(struct protection_domain *domain, int pasid,
3683 unsigned long cr3)
3684 {
3685 u64 *pte;
3686
3687 if (domain->mode != PAGE_MODE_NONE)
3688 return -EINVAL;
3689
3690 pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);
3691 if (pte == NULL)
3692 return -ENOMEM;
3693
3694 *pte = (cr3 & PAGE_MASK) | GCR3_VALID;
3695
3696 return __amd_iommu_flush_tlb(domain, pasid);
3697 }
3698
3699 static int __clear_gcr3(struct protection_domain *domain, int pasid)
3700 {
3701 u64 *pte;
3702
3703 if (domain->mode != PAGE_MODE_NONE)
3704 return -EINVAL;
3705
3706 pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);
3707 if (pte == NULL)
3708 return 0;
3709
3710 *pte = 0;
3711
3712 return __amd_iommu_flush_tlb(domain, pasid);
3713 }
3714
3715 int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
3716 unsigned long cr3)
3717 {
3718 struct protection_domain *domain = dom->priv;
3719 unsigned long flags;
3720 int ret;
3721
3722 spin_lock_irqsave(&domain->lock, flags);
3723 ret = __set_gcr3(domain, pasid, cr3);
3724 spin_unlock_irqrestore(&domain->lock, flags);
3725
3726 return ret;
3727 }
3728 EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);
3729
3730 int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)
3731 {
3732 struct protection_domain *domain = dom->priv;
3733 unsigned long flags;
3734 int ret;
3735
3736 spin_lock_irqsave(&domain->lock, flags);
3737 ret = __clear_gcr3(domain, pasid);
3738 spin_unlock_irqrestore(&domain->lock, flags);
3739
3740 return ret;
3741 }
3742 EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);
3743
3744 int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
3745 int status, int tag)
3746 {
3747 struct iommu_dev_data *dev_data;
3748 struct amd_iommu *iommu;
3749 struct iommu_cmd cmd;
3750
3751 INC_STATS_COUNTER(complete_ppr);
3752
3753 dev_data = get_dev_data(&pdev->dev);
3754 iommu = amd_iommu_rlookup_table[dev_data->devid];
3755
3756 build_complete_ppr(&cmd, dev_data->devid, pasid, status,
3757 tag, dev_data->pri_tlp);
3758
3759 return iommu_queue_command(iommu, &cmd);
3760 }
3761 EXPORT_SYMBOL(amd_iommu_complete_ppr);
3762
3763 struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev)
3764 {
3765 struct protection_domain *domain;
3766
3767 domain = get_domain(&pdev->dev);
3768 if (IS_ERR(domain))
3769 return NULL;
3770
3771 /* Only return IOMMUv2 domains */
3772 if (!(domain->flags & PD_IOMMUV2_MASK))
3773 return NULL;
3774
3775 return domain->iommu_domain;
3776 }
3777 EXPORT_SYMBOL(amd_iommu_get_v2_domain);
3778
3779 void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum)
3780 {
3781 struct iommu_dev_data *dev_data;
3782
3783 if (!amd_iommu_v2_supported())
3784 return;
3785
3786 dev_data = get_dev_data(&pdev->dev);
3787 dev_data->errata |= (1 << erratum);
3788 }
3789 EXPORT_SYMBOL(amd_iommu_enable_device_erratum);
3790
3791 int amd_iommu_device_info(struct pci_dev *pdev,
3792 struct amd_iommu_device_info *info)
3793 {
3794 int max_pasids;
3795 int pos;
3796
3797 if (pdev == NULL || info == NULL)
3798 return -EINVAL;
3799
3800 if (!amd_iommu_v2_supported())
3801 return -EINVAL;
3802
3803 memset(info, 0, sizeof(*info));
3804
3805 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);
3806 if (pos)
3807 info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
3808
3809 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
3810 if (pos)
3811 info->flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;
3812
3813 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
3814 if (pos) {
3815 int features;
3816
3817 max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));
3818 max_pasids = min(max_pasids, (1 << 20));
3819
3820 info->flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
3821 info->max_pasids = min(pci_max_pasids(pdev), max_pasids);
3822
3823 features = pci_pasid_features(pdev);
3824 if (features & PCI_PASID_CAP_EXEC)
3825 info->flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
3826 if (features & PCI_PASID_CAP_PRIV)
3827 info->flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
3828 }
3829
3830 return 0;
3831 }
3832 EXPORT_SYMBOL(amd_iommu_device_info);
3833
3834 #ifdef CONFIG_IRQ_REMAP
3835
3836 /*****************************************************************************
3837 *
3838 * Interrupt Remapping Implementation
3839 *
3840 *****************************************************************************/
3841
3842 union irte {
3843 u32 val;
3844 struct {
3845 u32 valid : 1,
3846 no_fault : 1,
3847 int_type : 3,
3848 rq_eoi : 1,
3849 dm : 1,
3850 rsvd_1 : 1,
3851 destination : 8,
3852 vector : 8,
3853 rsvd_2 : 8;
3854 } fields;
3855 };
3856
3857 #define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6)
3858 #define DTE_IRQ_REMAP_INTCTL (2ULL << 60)
3859 #define DTE_IRQ_TABLE_LEN (8ULL << 1)
3860 #define DTE_IRQ_REMAP_ENABLE 1ULL
3861
3862 static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table)
3863 {
3864 u64 dte;
3865
3866 dte = amd_iommu_dev_table[devid].data[2];
3867 dte &= ~DTE_IRQ_PHYS_ADDR_MASK;
3868 dte |= virt_to_phys(table->table);
3869 dte |= DTE_IRQ_REMAP_INTCTL;
3870 dte |= DTE_IRQ_TABLE_LEN;
3871 dte |= DTE_IRQ_REMAP_ENABLE;
3872
3873 amd_iommu_dev_table[devid].data[2] = dte;
3874 }
3875
3876 #define IRTE_ALLOCATED (~1U)
3877
3878 static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)
3879 {
3880 struct irq_remap_table *table = NULL;
3881 struct amd_iommu *iommu;
3882 unsigned long flags;
3883 u16 alias;
3884
3885 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
3886
3887 iommu = amd_iommu_rlookup_table[devid];
3888 if (!iommu)
3889 goto out_unlock;
3890
3891 table = irq_lookup_table[devid];
3892 if (table)
3893 goto out;
3894
3895 alias = amd_iommu_alias_table[devid];
3896 table = irq_lookup_table[alias];
3897 if (table) {
3898 irq_lookup_table[devid] = table;
3899 set_dte_irq_entry(devid, table);
3900 iommu_flush_dte(iommu, devid);
3901 goto out;
3902 }
3903
3904 /* Nothing there yet, allocate new irq remapping table */
3905 table = kzalloc(sizeof(*table), GFP_ATOMIC);
3906 if (!table)
3907 goto out;
3908
3909 /* Initialize table spin-lock */
3910 spin_lock_init(&table->lock);
3911
3912 if (ioapic)
3913 /* Keep the first 32 indexes free for IOAPIC interrupts */
3914 table->min_index = 32;
3915
3916 table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_ATOMIC);
3917 if (!table->table) {
3918 kfree(table);
3919 table = NULL;
3920 goto out;
3921 }
3922
3923 memset(table->table, 0, MAX_IRQS_PER_TABLE * sizeof(u32));
3924
3925 if (ioapic) {
3926 int i;
3927
3928 for (i = 0; i < 32; ++i)
3929 table->table[i] = IRTE_ALLOCATED;
3930 }
3931
3932 irq_lookup_table[devid] = table;
3933 set_dte_irq_entry(devid, table);
3934 iommu_flush_dte(iommu, devid);
3935 if (devid != alias) {
3936 irq_lookup_table[alias] = table;
3937 set_dte_irq_entry(alias, table);
3938 iommu_flush_dte(iommu, alias);
3939 }
3940
3941 out:
3942 iommu_completion_wait(iommu);
3943
3944 out_unlock:
3945 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
3946
3947 return table;
3948 }
3949
3950 static int alloc_irq_index(struct irq_cfg *cfg, u16 devid, int count)
3951 {
3952 struct irq_remap_table *table;
3953 unsigned long flags;
3954 int index, c;
3955
3956 table = get_irq_table(devid, false);
3957 if (!table)
3958 return -ENODEV;
3959
3960 spin_lock_irqsave(&table->lock, flags);
3961
3962 /* Scan table for free entries */
3963 for (c = 0, index = table->min_index;
3964 index < MAX_IRQS_PER_TABLE;
3965 ++index) {
3966 if (table->table[index] == 0)
3967 c += 1;
3968 else
3969 c = 0;
3970
3971 if (c == count) {
3972 struct irq_2_irte *irte_info;
3973
3974 for (; c != 0; --c)
3975 table->table[index - c + 1] = IRTE_ALLOCATED;
3976
3977 index -= count - 1;
3978
3979 cfg->remapped = 1;
3980 irte_info = &cfg->irq_2_irte;
3981 irte_info->devid = devid;
3982 irte_info->index = index;
3983
3984 goto out;
3985 }
3986 }
3987
3988 index = -ENOSPC;
3989
3990 out:
3991 spin_unlock_irqrestore(&table->lock, flags);
3992
3993 return index;
3994 }
3995
3996 static int get_irte(u16 devid, int index, union irte *irte)
3997 {
3998 struct irq_remap_table *table;
3999 unsigned long flags;
4000
4001 table = get_irq_table(devid, false);
4002 if (!table)
4003 return -ENOMEM;
4004
4005 spin_lock_irqsave(&table->lock, flags);
4006 irte->val = table->table[index];
4007 spin_unlock_irqrestore(&table->lock, flags);
4008
4009 return 0;
4010 }
4011
4012 static int modify_irte(u16 devid, int index, union irte irte)
4013 {
4014 struct irq_remap_table *table;
4015 struct amd_iommu *iommu;
4016 unsigned long flags;
4017
4018 iommu = amd_iommu_rlookup_table[devid];
4019 if (iommu == NULL)
4020 return -EINVAL;
4021
4022 table = get_irq_table(devid, false);
4023 if (!table)
4024 return -ENOMEM;
4025
4026 spin_lock_irqsave(&table->lock, flags);
4027 table->table[index] = irte.val;
4028 spin_unlock_irqrestore(&table->lock, flags);
4029
4030 iommu_flush_irt(iommu, devid);
4031 iommu_completion_wait(iommu);
4032
4033 return 0;
4034 }
4035
4036 static void free_irte(u16 devid, int index)
4037 {
4038 struct irq_remap_table *table;
4039 struct amd_iommu *iommu;
4040 unsigned long flags;
4041
4042 iommu = amd_iommu_rlookup_table[devid];
4043 if (iommu == NULL)
4044 return;
4045
4046 table = get_irq_table(devid, false);
4047 if (!table)
4048 return;
4049
4050 spin_lock_irqsave(&table->lock, flags);
4051 table->table[index] = 0;
4052 spin_unlock_irqrestore(&table->lock, flags);
4053
4054 iommu_flush_irt(iommu, devid);
4055 iommu_completion_wait(iommu);
4056 }
4057
4058 static int setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry,
4059 unsigned int destination, int vector,
4060 struct io_apic_irq_attr *attr)
4061 {
4062 struct irq_remap_table *table;
4063 struct irq_2_irte *irte_info;
4064 struct irq_cfg *cfg;
4065 union irte irte;
4066 int ioapic_id;
4067 int index;
4068 int devid;
4069 int ret;
4070
4071 cfg = irq_get_chip_data(irq);
4072 if (!cfg)
4073 return -EINVAL;
4074
4075 irte_info = &cfg->irq_2_irte;
4076 ioapic_id = mpc_ioapic_id(attr->ioapic);
4077 devid = get_ioapic_devid(ioapic_id);
4078
4079 if (devid < 0)
4080 return devid;
4081
4082 table = get_irq_table(devid, true);
4083 if (table == NULL)
4084 return -ENOMEM;
4085
4086 index = attr->ioapic_pin;
4087
4088 /* Setup IRQ remapping info */
4089 cfg->remapped = 1;
4090 irte_info->devid = devid;
4091 irte_info->index = index;
4092
4093 /* Setup IRTE for IOMMU */
4094 irte.val = 0;
4095 irte.fields.vector = vector;
4096 irte.fields.int_type = apic->irq_delivery_mode;
4097 irte.fields.destination = destination;
4098 irte.fields.dm = apic->irq_dest_mode;
4099 irte.fields.valid = 1;
4100
4101 ret = modify_irte(devid, index, irte);
4102 if (ret)
4103 return ret;
4104
4105 /* Setup IOAPIC entry */
4106 memset(entry, 0, sizeof(*entry));
4107
4108 entry->vector = index;
4109 entry->mask = 0;
4110 entry->trigger = attr->trigger;
4111 entry->polarity = attr->polarity;
4112
4113 /*
4114 * Mask level triggered irqs.
4115 */
4116 if (attr->trigger)
4117 entry->mask = 1;
4118
4119 return 0;
4120 }
4121
4122 static int set_affinity(struct irq_data *data, const struct cpumask *mask,
4123 bool force)
4124 {
4125 struct irq_2_irte *irte_info;
4126 unsigned int dest, irq;
4127 struct irq_cfg *cfg;
4128 union irte irte;
4129 int err;
4130
4131 if (!config_enabled(CONFIG_SMP))
4132 return -1;
4133
4134 cfg = data->chip_data;
4135 irq = data->irq;
4136 irte_info = &cfg->irq_2_irte;
4137
4138 if (!cpumask_intersects(mask, cpu_online_mask))
4139 return -EINVAL;
4140
4141 if (get_irte(irte_info->devid, irte_info->index, &irte))
4142 return -EBUSY;
4143
4144 if (assign_irq_vector(irq, cfg, mask))
4145 return -EBUSY;
4146
4147 err = apic->cpu_mask_to_apicid_and(cfg->domain, mask, &dest);
4148 if (err) {
4149 if (assign_irq_vector(irq, cfg, data->affinity))
4150 pr_err("AMD-Vi: Failed to recover vector for irq %d\n", irq);
4151 return err;
4152 }
4153
4154 irte.fields.vector = cfg->vector;
4155 irte.fields.destination = dest;
4156
4157 modify_irte(irte_info->devid, irte_info->index, irte);
4158
4159 if (cfg->move_in_progress)
4160 send_cleanup_vector(cfg);
4161
4162 cpumask_copy(data->affinity, mask);
4163
4164 return 0;
4165 }
4166
4167 static int free_irq(int irq)
4168 {
4169 struct irq_2_irte *irte_info;
4170 struct irq_cfg *cfg;
4171
4172 cfg = irq_get_chip_data(irq);
4173 if (!cfg)
4174 return -EINVAL;
4175
4176 irte_info = &cfg->irq_2_irte;
4177
4178 free_irte(irte_info->devid, irte_info->index);
4179
4180 return 0;
4181 }
4182
4183 static void compose_msi_msg(struct pci_dev *pdev,
4184 unsigned int irq, unsigned int dest,
4185 struct msi_msg *msg, u8 hpet_id)
4186 {
4187 struct irq_2_irte *irte_info;
4188 struct irq_cfg *cfg;
4189 union irte irte;
4190
4191 cfg = irq_get_chip_data(irq);
4192 if (!cfg)
4193 return;
4194
4195 irte_info = &cfg->irq_2_irte;
4196
4197 irte.val = 0;
4198 irte.fields.vector = cfg->vector;
4199 irte.fields.int_type = apic->irq_delivery_mode;
4200 irte.fields.destination = dest;
4201 irte.fields.dm = apic->irq_dest_mode;
4202 irte.fields.valid = 1;
4203
4204 modify_irte(irte_info->devid, irte_info->index, irte);
4205
4206 msg->address_hi = MSI_ADDR_BASE_HI;
4207 msg->address_lo = MSI_ADDR_BASE_LO;
4208 msg->data = irte_info->index;
4209 }
4210
4211 static int msi_alloc_irq(struct pci_dev *pdev, int irq, int nvec)
4212 {
4213 struct irq_cfg *cfg;
4214 int index;
4215 u16 devid;
4216
4217 if (!pdev)
4218 return -EINVAL;
4219
4220 cfg = irq_get_chip_data(irq);
4221 if (!cfg)
4222 return -EINVAL;
4223
4224 devid = get_device_id(&pdev->dev);
4225 index = alloc_irq_index(cfg, devid, nvec);
4226
4227 return index < 0 ? MAX_IRQS_PER_TABLE : index;
4228 }
4229
4230 static int msi_setup_irq(struct pci_dev *pdev, unsigned int irq,
4231 int index, int offset)
4232 {
4233 struct irq_2_irte *irte_info;
4234 struct irq_cfg *cfg;
4235 u16 devid;
4236
4237 if (!pdev)
4238 return -EINVAL;
4239
4240 cfg = irq_get_chip_data(irq);
4241 if (!cfg)
4242 return -EINVAL;
4243
4244 if (index >= MAX_IRQS_PER_TABLE)
4245 return 0;
4246
4247 devid = get_device_id(&pdev->dev);
4248 irte_info = &cfg->irq_2_irte;
4249
4250 cfg->remapped = 1;
4251 irte_info->devid = devid;
4252 irte_info->index = index + offset;
4253
4254 return 0;
4255 }
4256
4257 static int alloc_hpet_msi(unsigned int irq, unsigned int id)
4258 {
4259 struct irq_2_irte *irte_info;
4260 struct irq_cfg *cfg;
4261 int index, devid;
4262
4263 cfg = irq_get_chip_data(irq);
4264 if (!cfg)
4265 return -EINVAL;
4266
4267 irte_info = &cfg->irq_2_irte;
4268 devid = get_hpet_devid(id);
4269 if (devid < 0)
4270 return devid;
4271
4272 index = alloc_irq_index(cfg, devid, 1);
4273 if (index < 0)
4274 return index;
4275
4276 cfg->remapped = 1;
4277 irte_info->devid = devid;
4278 irte_info->index = index;
4279
4280 return 0;
4281 }
4282
4283 struct irq_remap_ops amd_iommu_irq_ops = {
4284 .supported = amd_iommu_supported,
4285 .prepare = amd_iommu_prepare,
4286 .enable = amd_iommu_enable,
4287 .disable = amd_iommu_disable,
4288 .reenable = amd_iommu_reenable,
4289 .enable_faulting = amd_iommu_enable_faulting,
4290 .setup_ioapic_entry = setup_ioapic_entry,
4291 .set_affinity = set_affinity,
4292 .free_irq = free_irq,
4293 .compose_msi_msg = compose_msi_msg,
4294 .msi_alloc_irq = msi_alloc_irq,
4295 .msi_setup_irq = msi_setup_irq,
4296 .alloc_hpet_msi = alloc_hpet_msi,
4297 };
4298 #endif
Linux with added FPC-III support