?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ARM: ep93xx: move timer to its own file
[linux/fpc-iii.git] / drivers / iommu / intel-iommu.c
bloba98a7b27aca1dec2cb2f53319df8a49abcf8e645
1 /*
2 * Copyright © 2006-2014 Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
19 */
20
21 #define pr_fmt(fmt) "DMAR: " fmt
22
23 #include <linux/init.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/export.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/memory.h>
36 #include <linux/timer.h>
37 #include <linux/iova.h>
38 #include <linux/iommu.h>
39 #include <linux/intel-iommu.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/tboot.h>
42 #include <linux/dmi.h>
43 #include <linux/pci-ats.h>
44 #include <linux/memblock.h>
45 #include <linux/dma-contiguous.h>
46 #include <linux/crash_dump.h>
47 #include <asm/irq_remapping.h>
48 #include <asm/cacheflush.h>
49 #include <asm/iommu.h>
50
51 #include "irq_remapping.h"
52
53 #define ROOT_SIZE VTD_PAGE_SIZE
54 #define CONTEXT_SIZE VTD_PAGE_SIZE
55
56 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
57 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
58 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
59 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
60
61 #define IOAPIC_RANGE_START (0xfee00000)
62 #define IOAPIC_RANGE_END (0xfeefffff)
63 #define IOVA_START_ADDR (0x1000)
64
65 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
66
67 #define MAX_AGAW_WIDTH 64
68 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
69
70 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
71 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
72
73 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
74 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
75 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
76 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
77 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
78
79 /* IO virtual address start page frame number */
80 #define IOVA_START_PFN (1)
81
82 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
83 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
84 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
85
86 /* page table handling */
87 #define LEVEL_STRIDE (9)
88 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
89
90 /*
91 * This bitmap is used to advertise the page sizes our hardware support
92 * to the IOMMU core, which will then use this information to split
93 * physically contiguous memory regions it is mapping into page sizes
94 * that we support.
95 *
96 * Traditionally the IOMMU core just handed us the mappings directly,
97 * after making sure the size is an order of a 4KiB page and that the
98 * mapping has natural alignment.
99 *
100 * To retain this behavior, we currently advertise that we support
101 * all page sizes that are an order of 4KiB.
102 *
103 * If at some point we'd like to utilize the IOMMU core's new behavior,
104 * we could change this to advertise the real page sizes we support.
105 */
106 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
107
108 static inline int agaw_to_level(int agaw)
109 {
110 return agaw + 2;
111 }
112
113 static inline int agaw_to_width(int agaw)
114 {
115 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
116 }
117
118 static inline int width_to_agaw(int width)
119 {
120 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
121 }
122
123 static inline unsigned int level_to_offset_bits(int level)
124 {
125 return (level - 1) * LEVEL_STRIDE;
126 }
127
128 static inline int pfn_level_offset(unsigned long pfn, int level)
129 {
130 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
131 }
132
133 static inline unsigned long level_mask(int level)
134 {
135 return -1UL << level_to_offset_bits(level);
136 }
137
138 static inline unsigned long level_size(int level)
139 {
140 return 1UL << level_to_offset_bits(level);
141 }
142
143 static inline unsigned long align_to_level(unsigned long pfn, int level)
144 {
145 return (pfn + level_size(level) - 1) & level_mask(level);
146 }
147
148 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
149 {
150 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
151 }
152
153 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
154 are never going to work. */
155 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
156 {
157 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
158 }
159
160 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
161 {
162 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
163 }
164 static inline unsigned long page_to_dma_pfn(struct page *pg)
165 {
166 return mm_to_dma_pfn(page_to_pfn(pg));
167 }
168 static inline unsigned long virt_to_dma_pfn(void *p)
169 {
170 return page_to_dma_pfn(virt_to_page(p));
171 }
172
173 /* global iommu list, set NULL for ignored DMAR units */
174 static struct intel_iommu **g_iommus;
175
176 static void __init check_tylersburg_isoch(void);
177 static int rwbf_quirk;
178
179 /*
180 * set to 1 to panic kernel if can't successfully enable VT-d
181 * (used when kernel is launched w/ TXT)
182 */
183 static int force_on = 0;
184
185 /*
186 * 0: Present
187 * 1-11: Reserved
188 * 12-63: Context Ptr (12 - (haw-1))
189 * 64-127: Reserved
190 */
191 struct root_entry {
192 u64 lo;
193 u64 hi;
194 };
195 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
196
197 /*
198 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
199 * if marked present.
200 */
201 static phys_addr_t root_entry_lctp(struct root_entry *re)
202 {
203 if (!(re->lo & 1))
204 return 0;
205
206 return re->lo & VTD_PAGE_MASK;
207 }
208
209 /*
210 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
211 * if marked present.
212 */
213 static phys_addr_t root_entry_uctp(struct root_entry *re)
214 {
215 if (!(re->hi & 1))
216 return 0;
217
218 return re->hi & VTD_PAGE_MASK;
219 }
220 /*
221 * low 64 bits:
222 * 0: present
223 * 1: fault processing disable
224 * 2-3: translation type
225 * 12-63: address space root
226 * high 64 bits:
227 * 0-2: address width
228 * 3-6: aval
229 * 8-23: domain id
230 */
231 struct context_entry {
232 u64 lo;
233 u64 hi;
234 };
235
236 static inline void context_clear_pasid_enable(struct context_entry *context)
237 {
238 context->lo &= ~(1ULL << 11);
239 }
240
241 static inline bool context_pasid_enabled(struct context_entry *context)
242 {
243 return !!(context->lo & (1ULL << 11));
244 }
245
246 static inline void context_set_copied(struct context_entry *context)
247 {
248 context->hi |= (1ull << 3);
249 }
250
251 static inline bool context_copied(struct context_entry *context)
252 {
253 return !!(context->hi & (1ULL << 3));
254 }
255
256 static inline bool __context_present(struct context_entry *context)
257 {
258 return (context->lo & 1);
259 }
260
261 static inline bool context_present(struct context_entry *context)
262 {
263 return context_pasid_enabled(context) ?
264 __context_present(context) :
265 __context_present(context) && !context_copied(context);
266 }
267
268 static inline void context_set_present(struct context_entry *context)
269 {
270 context->lo |= 1;
271 }
272
273 static inline void context_set_fault_enable(struct context_entry *context)
274 {
275 context->lo &= (((u64)-1) << 2) | 1;
276 }
277
278 static inline void context_set_translation_type(struct context_entry *context,
279 unsigned long value)
280 {
281 context->lo &= (((u64)-1) << 4) | 3;
282 context->lo |= (value & 3) << 2;
283 }
284
285 static inline void context_set_address_root(struct context_entry *context,
286 unsigned long value)
287 {
288 context->lo &= ~VTD_PAGE_MASK;
289 context->lo |= value & VTD_PAGE_MASK;
290 }
291
292 static inline void context_set_address_width(struct context_entry *context,
293 unsigned long value)
294 {
295 context->hi |= value & 7;
296 }
297
298 static inline void context_set_domain_id(struct context_entry *context,
299 unsigned long value)
300 {
301 context->hi |= (value & ((1 << 16) - 1)) << 8;
302 }
303
304 static inline int context_domain_id(struct context_entry *c)
305 {
306 return((c->hi >> 8) & 0xffff);
307 }
308
309 static inline void context_clear_entry(struct context_entry *context)
310 {
311 context->lo = 0;
312 context->hi = 0;
313 }
314
315 /*
316 * 0: readable
317 * 1: writable
318 * 2-6: reserved
319 * 7: super page
320 * 8-10: available
321 * 11: snoop behavior
322 * 12-63: Host physcial address
323 */
324 struct dma_pte {
325 u64 val;
326 };
327
328 static inline void dma_clear_pte(struct dma_pte *pte)
329 {
330 pte->val = 0;
331 }
332
333 static inline u64 dma_pte_addr(struct dma_pte *pte)
334 {
335 #ifdef CONFIG_64BIT
336 return pte->val & VTD_PAGE_MASK;
337 #else
338 /* Must have a full atomic 64-bit read */
339 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
340 #endif
341 }
342
343 static inline bool dma_pte_present(struct dma_pte *pte)
344 {
345 return (pte->val & 3) != 0;
346 }
347
348 static inline bool dma_pte_superpage(struct dma_pte *pte)
349 {
350 return (pte->val & DMA_PTE_LARGE_PAGE);
351 }
352
353 static inline int first_pte_in_page(struct dma_pte *pte)
354 {
355 return !((unsigned long)pte & ~VTD_PAGE_MASK);
356 }
357
358 /*
359 * This domain is a statically identity mapping domain.
360 * 1. This domain creats a static 1:1 mapping to all usable memory.
361 * 2. It maps to each iommu if successful.
362 * 3. Each iommu mapps to this domain if successful.
363 */
364 static struct dmar_domain *si_domain;
365 static int hw_pass_through = 1;
366
367 /* domain represents a virtual machine, more than one devices
368 * across iommus may be owned in one domain, e.g. kvm guest.
369 */
370 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
371
372 /* si_domain contains mulitple devices */
373 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
374
375 struct dmar_domain {
376 int id; /* domain id */
377 int nid; /* node id */
378 DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
379 /* bitmap of iommus this domain uses*/
380
381 struct list_head devices; /* all devices' list */
382 struct iova_domain iovad; /* iova's that belong to this domain */
383
384 struct dma_pte *pgd; /* virtual address */
385 int gaw; /* max guest address width */
386
387 /* adjusted guest address width, 0 is level 2 30-bit */
388 int agaw;
389
390 int flags; /* flags to find out type of domain */
391
392 int iommu_coherency;/* indicate coherency of iommu access */
393 int iommu_snooping; /* indicate snooping control feature*/
394 int iommu_count; /* reference count of iommu */
395 int iommu_superpage;/* Level of superpages supported:
396 0 == 4KiB (no superpages), 1 == 2MiB,
397 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
398 spinlock_t iommu_lock; /* protect iommu set in domain */
399 u64 max_addr; /* maximum mapped address */
400
401 struct iommu_domain domain; /* generic domain data structure for
402 iommu core */
403 };
404
405 /* PCI domain-device relationship */
406 struct device_domain_info {
407 struct list_head link; /* link to domain siblings */
408 struct list_head global; /* link to global list */
409 u8 bus; /* PCI bus number */
410 u8 devfn; /* PCI devfn number */
411 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
412 struct intel_iommu *iommu; /* IOMMU used by this device */
413 struct dmar_domain *domain; /* pointer to domain */
414 };
415
416 struct dmar_rmrr_unit {
417 struct list_head list; /* list of rmrr units */
418 struct acpi_dmar_header *hdr; /* ACPI header */
419 u64 base_address; /* reserved base address*/
420 u64 end_address; /* reserved end address */
421 struct dmar_dev_scope *devices; /* target devices */
422 int devices_cnt; /* target device count */
423 };
424
425 struct dmar_atsr_unit {
426 struct list_head list; /* list of ATSR units */
427 struct acpi_dmar_header *hdr; /* ACPI header */
428 struct dmar_dev_scope *devices; /* target devices */
429 int devices_cnt; /* target device count */
430 u8 include_all:1; /* include all ports */
431 };
432
433 static LIST_HEAD(dmar_atsr_units);
434 static LIST_HEAD(dmar_rmrr_units);
435
436 #define for_each_rmrr_units(rmrr) \
437 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
438
439 static void flush_unmaps_timeout(unsigned long data);
440
441 static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
442
443 #define HIGH_WATER_MARK 250
444 struct deferred_flush_tables {
445 int next;
446 struct iova *iova[HIGH_WATER_MARK];
447 struct dmar_domain *domain[HIGH_WATER_MARK];
448 struct page *freelist[HIGH_WATER_MARK];
449 };
450
451 static struct deferred_flush_tables *deferred_flush;
452
453 /* bitmap for indexing intel_iommus */
454 static int g_num_of_iommus;
455
456 static DEFINE_SPINLOCK(async_umap_flush_lock);
457 static LIST_HEAD(unmaps_to_do);
458
459 static int timer_on;
460 static long list_size;
461
462 static void domain_exit(struct dmar_domain *domain);
463 static void domain_remove_dev_info(struct dmar_domain *domain);
464 static void domain_remove_one_dev_info(struct dmar_domain *domain,
465 struct device *dev);
466 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
467 struct device *dev);
468 static int domain_detach_iommu(struct dmar_domain *domain,
469 struct intel_iommu *iommu);
470
471 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
472 int dmar_disabled = 0;
473 #else
474 int dmar_disabled = 1;
475 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
476
477 int intel_iommu_enabled = 0;
478 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
479
480 static int dmar_map_gfx = 1;
481 static int dmar_forcedac;
482 static int intel_iommu_strict;
483 static int intel_iommu_superpage = 1;
484 static int intel_iommu_ecs = 1;
485
486 /* We only actually use ECS when PASID support (on the new bit 40)
487 * is also advertised. Some early implementations — the ones with
488 * PASID support on bit 28 — have issues even when we *only* use
489 * extended root/context tables. */
490 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
491 ecap_pasid(iommu->ecap))
492
493 int intel_iommu_gfx_mapped;
494 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
495
496 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
497 static DEFINE_SPINLOCK(device_domain_lock);
498 static LIST_HEAD(device_domain_list);
499
500 static const struct iommu_ops intel_iommu_ops;
501
502 static bool translation_pre_enabled(struct intel_iommu *iommu)
503 {
504 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
505 }
506
507 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
508 {
509 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
510 }
511
512 static void init_translation_status(struct intel_iommu *iommu)
513 {
514 u32 gsts;
515
516 gsts = readl(iommu->reg + DMAR_GSTS_REG);
517 if (gsts & DMA_GSTS_TES)
518 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
519 }
520
521 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
522 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
523 {
524 return container_of(dom, struct dmar_domain, domain);
525 }
526
527 static int __init intel_iommu_setup(char *str)
528 {
529 if (!str)
530 return -EINVAL;
531 while (*str) {
532 if (!strncmp(str, "on", 2)) {
533 dmar_disabled = 0;
534 pr_info("IOMMU enabled\n");
535 } else if (!strncmp(str, "off", 3)) {
536 dmar_disabled = 1;
537 pr_info("IOMMU disabled\n");
538 } else if (!strncmp(str, "igfx_off", 8)) {
539 dmar_map_gfx = 0;
540 pr_info("Disable GFX device mapping\n");
541 } else if (!strncmp(str, "forcedac", 8)) {
542 pr_info("Forcing DAC for PCI devices\n");
543 dmar_forcedac = 1;
544 } else if (!strncmp(str, "strict", 6)) {
545 pr_info("Disable batched IOTLB flush\n");
546 intel_iommu_strict = 1;
547 } else if (!strncmp(str, "sp_off", 6)) {
548 pr_info("Disable supported super page\n");
549 intel_iommu_superpage = 0;
550 } else if (!strncmp(str, "ecs_off", 7)) {
551 printk(KERN_INFO
552 "Intel-IOMMU: disable extended context table support\n");
553 intel_iommu_ecs = 0;
554 }
555
556 str += strcspn(str, ",");
557 while (*str == ',')
558 str++;
559 }
560 return 0;
561 }
562 __setup("intel_iommu=", intel_iommu_setup);
563
564 static struct kmem_cache *iommu_domain_cache;
565 static struct kmem_cache *iommu_devinfo_cache;
566
567 static inline void *alloc_pgtable_page(int node)
568 {
569 struct page *page;
570 void *vaddr = NULL;
571
572 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
573 if (page)
574 vaddr = page_address(page);
575 return vaddr;
576 }
577
578 static inline void free_pgtable_page(void *vaddr)
579 {
580 free_page((unsigned long)vaddr);
581 }
582
583 static inline void *alloc_domain_mem(void)
584 {
585 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
586 }
587
588 static void free_domain_mem(void *vaddr)
589 {
590 kmem_cache_free(iommu_domain_cache, vaddr);
591 }
592
593 static inline void * alloc_devinfo_mem(void)
594 {
595 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
596 }
597
598 static inline void free_devinfo_mem(void *vaddr)
599 {
600 kmem_cache_free(iommu_devinfo_cache, vaddr);
601 }
602
603 static inline int domain_type_is_vm(struct dmar_domain *domain)
604 {
605 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
606 }
607
608 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
609 {
610 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
611 DOMAIN_FLAG_STATIC_IDENTITY);
612 }
613
614 static inline int domain_pfn_supported(struct dmar_domain *domain,
615 unsigned long pfn)
616 {
617 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
618
619 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
620 }
621
622 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
623 {
624 unsigned long sagaw;
625 int agaw = -1;
626
627 sagaw = cap_sagaw(iommu->cap);
628 for (agaw = width_to_agaw(max_gaw);
629 agaw >= 0; agaw--) {
630 if (test_bit(agaw, &sagaw))
631 break;
632 }
633
634 return agaw;
635 }
636
637 /*
638 * Calculate max SAGAW for each iommu.
639 */
640 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
641 {
642 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
643 }
644
645 /*
646 * calculate agaw for each iommu.
647 * "SAGAW" may be different across iommus, use a default agaw, and
648 * get a supported less agaw for iommus that don't support the default agaw.
649 */
650 int iommu_calculate_agaw(struct intel_iommu *iommu)
651 {
652 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
653 }
654
655 /* This functionin only returns single iommu in a domain */
656 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
657 {
658 int iommu_id;
659
660 /* si_domain and vm domain should not get here. */
661 BUG_ON(domain_type_is_vm_or_si(domain));
662 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
663 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
664 return NULL;
665
666 return g_iommus[iommu_id];
667 }
668
669 static void domain_update_iommu_coherency(struct dmar_domain *domain)
670 {
671 struct dmar_drhd_unit *drhd;
672 struct intel_iommu *iommu;
673 bool found = false;
674 int i;
675
676 domain->iommu_coherency = 1;
677
678 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
679 found = true;
680 if (!ecap_coherent(g_iommus[i]->ecap)) {
681 domain->iommu_coherency = 0;
682 break;
683 }
684 }
685 if (found)
686 return;
687
688 /* No hardware attached; use lowest common denominator */
689 rcu_read_lock();
690 for_each_active_iommu(iommu, drhd) {
691 if (!ecap_coherent(iommu->ecap)) {
692 domain->iommu_coherency = 0;
693 break;
694 }
695 }
696 rcu_read_unlock();
697 }
698
699 static int domain_update_iommu_snooping(struct intel_iommu *skip)
700 {
701 struct dmar_drhd_unit *drhd;
702 struct intel_iommu *iommu;
703 int ret = 1;
704
705 rcu_read_lock();
706 for_each_active_iommu(iommu, drhd) {
707 if (iommu != skip) {
708 if (!ecap_sc_support(iommu->ecap)) {
709 ret = 0;
710 break;
711 }
712 }
713 }
714 rcu_read_unlock();
715
716 return ret;
717 }
718
719 static int domain_update_iommu_superpage(struct intel_iommu *skip)
720 {
721 struct dmar_drhd_unit *drhd;
722 struct intel_iommu *iommu;
723 int mask = 0xf;
724
725 if (!intel_iommu_superpage) {
726 return 0;
727 }
728
729 /* set iommu_superpage to the smallest common denominator */
730 rcu_read_lock();
731 for_each_active_iommu(iommu, drhd) {
732 if (iommu != skip) {
733 mask &= cap_super_page_val(iommu->cap);
734 if (!mask)
735 break;
736 }
737 }
738 rcu_read_unlock();
739
740 return fls(mask);
741 }
742
743 /* Some capabilities may be different across iommus */
744 static void domain_update_iommu_cap(struct dmar_domain *domain)
745 {
746 domain_update_iommu_coherency(domain);
747 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
748 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
749 }
750
751 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
752 u8 bus, u8 devfn, int alloc)
753 {
754 struct root_entry *root = &iommu->root_entry[bus];
755 struct context_entry *context;
756 u64 *entry;
757
758 if (ecs_enabled(iommu)) {
759 if (devfn >= 0x80) {
760 devfn -= 0x80;
761 entry = &root->hi;
762 }
763 devfn *= 2;
764 }
765 entry = &root->lo;
766 if (*entry & 1)
767 context = phys_to_virt(*entry & VTD_PAGE_MASK);
768 else {
769 unsigned long phy_addr;
770 if (!alloc)
771 return NULL;
772
773 context = alloc_pgtable_page(iommu->node);
774 if (!context)
775 return NULL;
776
777 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
778 phy_addr = virt_to_phys((void *)context);
779 *entry = phy_addr | 1;
780 __iommu_flush_cache(iommu, entry, sizeof(*entry));
781 }
782 return &context[devfn];
783 }
784
785 static int iommu_dummy(struct device *dev)
786 {
787 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
788 }
789
790 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
791 {
792 struct dmar_drhd_unit *drhd = NULL;
793 struct intel_iommu *iommu;
794 struct device *tmp;
795 struct pci_dev *ptmp, *pdev = NULL;
796 u16 segment = 0;
797 int i;
798
799 if (iommu_dummy(dev))
800 return NULL;
801
802 if (dev_is_pci(dev)) {
803 pdev = to_pci_dev(dev);
804 segment = pci_domain_nr(pdev->bus);
805 } else if (has_acpi_companion(dev))
806 dev = &ACPI_COMPANION(dev)->dev;
807
808 rcu_read_lock();
809 for_each_active_iommu(iommu, drhd) {
810 if (pdev && segment != drhd->segment)
811 continue;
812
813 for_each_active_dev_scope(drhd->devices,
814 drhd->devices_cnt, i, tmp) {
815 if (tmp == dev) {
816 *bus = drhd->devices[i].bus;
817 *devfn = drhd->devices[i].devfn;
818 goto out;
819 }
820
821 if (!pdev || !dev_is_pci(tmp))
822 continue;
823
824 ptmp = to_pci_dev(tmp);
825 if (ptmp->subordinate &&
826 ptmp->subordinate->number <= pdev->bus->number &&
827 ptmp->subordinate->busn_res.end >= pdev->bus->number)
828 goto got_pdev;
829 }
830
831 if (pdev && drhd->include_all) {
832 got_pdev:
833 *bus = pdev->bus->number;
834 *devfn = pdev->devfn;
835 goto out;
836 }
837 }
838 iommu = NULL;
839 out:
840 rcu_read_unlock();
841
842 return iommu;
843 }
844
845 static void domain_flush_cache(struct dmar_domain *domain,
846 void *addr, int size)
847 {
848 if (!domain->iommu_coherency)
849 clflush_cache_range(addr, size);
850 }
851
852 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
853 {
854 struct context_entry *context;
855 int ret = 0;
856 unsigned long flags;
857
858 spin_lock_irqsave(&iommu->lock, flags);
859 context = iommu_context_addr(iommu, bus, devfn, 0);
860 if (context)
861 ret = context_present(context);
862 spin_unlock_irqrestore(&iommu->lock, flags);
863 return ret;
864 }
865
866 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
867 {
868 struct context_entry *context;
869 unsigned long flags;
870
871 spin_lock_irqsave(&iommu->lock, flags);
872 context = iommu_context_addr(iommu, bus, devfn, 0);
873 if (context) {
874 context_clear_entry(context);
875 __iommu_flush_cache(iommu, context, sizeof(*context));
876 }
877 spin_unlock_irqrestore(&iommu->lock, flags);
878 }
879
880 static void free_context_table(struct intel_iommu *iommu)
881 {
882 int i;
883 unsigned long flags;
884 struct context_entry *context;
885
886 spin_lock_irqsave(&iommu->lock, flags);
887 if (!iommu->root_entry) {
888 goto out;
889 }
890 for (i = 0; i < ROOT_ENTRY_NR; i++) {
891 context = iommu_context_addr(iommu, i, 0, 0);
892 if (context)
893 free_pgtable_page(context);
894
895 if (!ecs_enabled(iommu))
896 continue;
897
898 context = iommu_context_addr(iommu, i, 0x80, 0);
899 if (context)
900 free_pgtable_page(context);
901
902 }
903 free_pgtable_page(iommu->root_entry);
904 iommu->root_entry = NULL;
905 out:
906 spin_unlock_irqrestore(&iommu->lock, flags);
907 }
908
909 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
910 unsigned long pfn, int *target_level)
911 {
912 struct dma_pte *parent, *pte = NULL;
913 int level = agaw_to_level(domain->agaw);
914 int offset;
915
916 BUG_ON(!domain->pgd);
917
918 if (!domain_pfn_supported(domain, pfn))
919 /* Address beyond IOMMU's addressing capabilities. */
920 return NULL;
921
922 parent = domain->pgd;
923
924 while (1) {
925 void *tmp_page;
926
927 offset = pfn_level_offset(pfn, level);
928 pte = &parent[offset];
929 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
930 break;
931 if (level == *target_level)
932 break;
933
934 if (!dma_pte_present(pte)) {
935 uint64_t pteval;
936
937 tmp_page = alloc_pgtable_page(domain->nid);
938
939 if (!tmp_page)
940 return NULL;
941
942 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
943 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
944 if (cmpxchg64(&pte->val, 0ULL, pteval))
945 /* Someone else set it while we were thinking; use theirs. */
946 free_pgtable_page(tmp_page);
947 else
948 domain_flush_cache(domain, pte, sizeof(*pte));
949 }
950 if (level == 1)
951 break;
952
953 parent = phys_to_virt(dma_pte_addr(pte));
954 level--;
955 }
956
957 if (!*target_level)
958 *target_level = level;
959
960 return pte;
961 }
962
963
964 /* return address's pte at specific level */
965 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
966 unsigned long pfn,
967 int level, int *large_page)
968 {
969 struct dma_pte *parent, *pte = NULL;
970 int total = agaw_to_level(domain->agaw);
971 int offset;
972
973 parent = domain->pgd;
974 while (level <= total) {
975 offset = pfn_level_offset(pfn, total);
976 pte = &parent[offset];
977 if (level == total)
978 return pte;
979
980 if (!dma_pte_present(pte)) {
981 *large_page = total;
982 break;
983 }
984
985 if (dma_pte_superpage(pte)) {
986 *large_page = total;
987 return pte;
988 }
989
990 parent = phys_to_virt(dma_pte_addr(pte));
991 total--;
992 }
993 return NULL;
994 }
995
996 /* clear last level pte, a tlb flush should be followed */
997 static void dma_pte_clear_range(struct dmar_domain *domain,
998 unsigned long start_pfn,
999 unsigned long last_pfn)
1000 {
1001 unsigned int large_page = 1;
1002 struct dma_pte *first_pte, *pte;
1003
1004 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1005 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1006 BUG_ON(start_pfn > last_pfn);
1007
1008 /* we don't need lock here; nobody else touches the iova range */
1009 do {
1010 large_page = 1;
1011 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1012 if (!pte) {
1013 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1014 continue;
1015 }
1016 do {
1017 dma_clear_pte(pte);
1018 start_pfn += lvl_to_nr_pages(large_page);
1019 pte++;
1020 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1021
1022 domain_flush_cache(domain, first_pte,
1023 (void *)pte - (void *)first_pte);
1024
1025 } while (start_pfn && start_pfn <= last_pfn);
1026 }
1027
1028 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1029 struct dma_pte *pte, unsigned long pfn,
1030 unsigned long start_pfn, unsigned long last_pfn)
1031 {
1032 pfn = max(start_pfn, pfn);
1033 pte = &pte[pfn_level_offset(pfn, level)];
1034
1035 do {
1036 unsigned long level_pfn;
1037 struct dma_pte *level_pte;
1038
1039 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1040 goto next;
1041
1042 level_pfn = pfn & level_mask(level - 1);
1043 level_pte = phys_to_virt(dma_pte_addr(pte));
1044
1045 if (level > 2)
1046 dma_pte_free_level(domain, level - 1, level_pte,
1047 level_pfn, start_pfn, last_pfn);
1048
1049 /* If range covers entire pagetable, free it */
1050 if (!(start_pfn > level_pfn ||
1051 last_pfn < level_pfn + level_size(level) - 1)) {
1052 dma_clear_pte(pte);
1053 domain_flush_cache(domain, pte, sizeof(*pte));
1054 free_pgtable_page(level_pte);
1055 }
1056 next:
1057 pfn += level_size(level);
1058 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1059 }
1060
1061 /* free page table pages. last level pte should already be cleared */
1062 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1063 unsigned long start_pfn,
1064 unsigned long last_pfn)
1065 {
1066 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1067 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1068 BUG_ON(start_pfn > last_pfn);
1069
1070 dma_pte_clear_range(domain, start_pfn, last_pfn);
1071
1072 /* We don't need lock here; nobody else touches the iova range */
1073 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1074 domain->pgd, 0, start_pfn, last_pfn);
1075
1076 /* free pgd */
1077 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1078 free_pgtable_page(domain->pgd);
1079 domain->pgd = NULL;
1080 }
1081 }
1082
1083 /* When a page at a given level is being unlinked from its parent, we don't
1084 need to *modify* it at all. All we need to do is make a list of all the
1085 pages which can be freed just as soon as we've flushed the IOTLB and we
1086 know the hardware page-walk will no longer touch them.
1087 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1088 be freed. */
1089 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1090 int level, struct dma_pte *pte,
1091 struct page *freelist)
1092 {
1093 struct page *pg;
1094
1095 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1096 pg->freelist = freelist;
1097 freelist = pg;
1098
1099 if (level == 1)
1100 return freelist;
1101
1102 pte = page_address(pg);
1103 do {
1104 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1105 freelist = dma_pte_list_pagetables(domain, level - 1,
1106 pte, freelist);
1107 pte++;
1108 } while (!first_pte_in_page(pte));
1109
1110 return freelist;
1111 }
1112
1113 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1114 struct dma_pte *pte, unsigned long pfn,
1115 unsigned long start_pfn,
1116 unsigned long last_pfn,
1117 struct page *freelist)
1118 {
1119 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1120
1121 pfn = max(start_pfn, pfn);
1122 pte = &pte[pfn_level_offset(pfn, level)];
1123
1124 do {
1125 unsigned long level_pfn;
1126
1127 if (!dma_pte_present(pte))
1128 goto next;
1129
1130 level_pfn = pfn & level_mask(level);
1131
1132 /* If range covers entire pagetable, free it */
1133 if (start_pfn <= level_pfn &&
1134 last_pfn >= level_pfn + level_size(level) - 1) {
1135 /* These suborbinate page tables are going away entirely. Don't
1136 bother to clear them; we're just going to *free* them. */
1137 if (level > 1 && !dma_pte_superpage(pte))
1138 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1139
1140 dma_clear_pte(pte);
1141 if (!first_pte)
1142 first_pte = pte;
1143 last_pte = pte;
1144 } else if (level > 1) {
1145 /* Recurse down into a level that isn't *entirely* obsolete */
1146 freelist = dma_pte_clear_level(domain, level - 1,
1147 phys_to_virt(dma_pte_addr(pte)),
1148 level_pfn, start_pfn, last_pfn,
1149 freelist);
1150 }
1151 next:
1152 pfn += level_size(level);
1153 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1154
1155 if (first_pte)
1156 domain_flush_cache(domain, first_pte,
1157 (void *)++last_pte - (void *)first_pte);
1158
1159 return freelist;
1160 }
1161
1162 /* We can't just free the pages because the IOMMU may still be walking
1163 the page tables, and may have cached the intermediate levels. The
1164 pages can only be freed after the IOTLB flush has been done. */
1165 struct page *domain_unmap(struct dmar_domain *domain,
1166 unsigned long start_pfn,
1167 unsigned long last_pfn)
1168 {
1169 struct page *freelist = NULL;
1170
1171 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1172 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1173 BUG_ON(start_pfn > last_pfn);
1174
1175 /* we don't need lock here; nobody else touches the iova range */
1176 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1177 domain->pgd, 0, start_pfn, last_pfn, NULL);
1178
1179 /* free pgd */
1180 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1181 struct page *pgd_page = virt_to_page(domain->pgd);
1182 pgd_page->freelist = freelist;
1183 freelist = pgd_page;
1184
1185 domain->pgd = NULL;
1186 }
1187
1188 return freelist;
1189 }
1190
1191 void dma_free_pagelist(struct page *freelist)
1192 {
1193 struct page *pg;
1194
1195 while ((pg = freelist)) {
1196 freelist = pg->freelist;
1197 free_pgtable_page(page_address(pg));
1198 }
1199 }
1200
1201 /* iommu handling */
1202 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1203 {
1204 struct root_entry *root;
1205 unsigned long flags;
1206
1207 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1208 if (!root) {
1209 pr_err("Allocating root entry for %s failed\n",
1210 iommu->name);
1211 return -ENOMEM;
1212 }
1213
1214 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1215
1216 spin_lock_irqsave(&iommu->lock, flags);
1217 iommu->root_entry = root;
1218 spin_unlock_irqrestore(&iommu->lock, flags);
1219
1220 return 0;
1221 }
1222
1223 static void iommu_set_root_entry(struct intel_iommu *iommu)
1224 {
1225 u64 addr;
1226 u32 sts;
1227 unsigned long flag;
1228
1229 addr = virt_to_phys(iommu->root_entry);
1230 if (ecs_enabled(iommu))
1231 addr |= DMA_RTADDR_RTT;
1232
1233 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1234 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1235
1236 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1237
1238 /* Make sure hardware complete it */
1239 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1240 readl, (sts & DMA_GSTS_RTPS), sts);
1241
1242 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1243 }
1244
1245 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1246 {
1247 u32 val;
1248 unsigned long flag;
1249
1250 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1251 return;
1252
1253 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1254 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1255
1256 /* Make sure hardware complete it */
1257 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1258 readl, (!(val & DMA_GSTS_WBFS)), val);
1259
1260 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1261 }
1262
1263 /* return value determine if we need a write buffer flush */
1264 static void __iommu_flush_context(struct intel_iommu *iommu,
1265 u16 did, u16 source_id, u8 function_mask,
1266 u64 type)
1267 {
1268 u64 val = 0;
1269 unsigned long flag;
1270
1271 switch (type) {
1272 case DMA_CCMD_GLOBAL_INVL:
1273 val = DMA_CCMD_GLOBAL_INVL;
1274 break;
1275 case DMA_CCMD_DOMAIN_INVL:
1276 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1277 break;
1278 case DMA_CCMD_DEVICE_INVL:
1279 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1280 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1281 break;
1282 default:
1283 BUG();
1284 }
1285 val |= DMA_CCMD_ICC;
1286
1287 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1288 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1289
1290 /* Make sure hardware complete it */
1291 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1292 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1293
1294 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1295 }
1296
1297 /* return value determine if we need a write buffer flush */
1298 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1299 u64 addr, unsigned int size_order, u64 type)
1300 {
1301 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1302 u64 val = 0, val_iva = 0;
1303 unsigned long flag;
1304
1305 switch (type) {
1306 case DMA_TLB_GLOBAL_FLUSH:
1307 /* global flush doesn't need set IVA_REG */
1308 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1309 break;
1310 case DMA_TLB_DSI_FLUSH:
1311 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1312 break;
1313 case DMA_TLB_PSI_FLUSH:
1314 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1315 /* IH bit is passed in as part of address */
1316 val_iva = size_order | addr;
1317 break;
1318 default:
1319 BUG();
1320 }
1321 /* Note: set drain read/write */
1322 #if 0
1323 /*
1324 * This is probably to be super secure.. Looks like we can
1325 * ignore it without any impact.
1326 */
1327 if (cap_read_drain(iommu->cap))
1328 val |= DMA_TLB_READ_DRAIN;
1329 #endif
1330 if (cap_write_drain(iommu->cap))
1331 val |= DMA_TLB_WRITE_DRAIN;
1332
1333 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1334 /* Note: Only uses first TLB reg currently */
1335 if (val_iva)
1336 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1337 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1338
1339 /* Make sure hardware complete it */
1340 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1341 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1342
1343 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1344
1345 /* check IOTLB invalidation granularity */
1346 if (DMA_TLB_IAIG(val) == 0)
1347 pr_err("Flush IOTLB failed\n");
1348 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1349 pr_debug("TLB flush request %Lx, actual %Lx\n",
1350 (unsigned long long)DMA_TLB_IIRG(type),
1351 (unsigned long long)DMA_TLB_IAIG(val));
1352 }
1353
1354 static struct device_domain_info *
1355 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1356 u8 bus, u8 devfn)
1357 {
1358 bool found = false;
1359 unsigned long flags;
1360 struct device_domain_info *info;
1361 struct pci_dev *pdev;
1362
1363 if (!ecap_dev_iotlb_support(iommu->ecap))
1364 return NULL;
1365
1366 if (!iommu->qi)
1367 return NULL;
1368
1369 spin_lock_irqsave(&device_domain_lock, flags);
1370 list_for_each_entry(info, &domain->devices, link)
1371 if (info->iommu == iommu && info->bus == bus &&
1372 info->devfn == devfn) {
1373 found = true;
1374 break;
1375 }
1376 spin_unlock_irqrestore(&device_domain_lock, flags);
1377
1378 if (!found || !info->dev || !dev_is_pci(info->dev))
1379 return NULL;
1380
1381 pdev = to_pci_dev(info->dev);
1382
1383 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
1384 return NULL;
1385
1386 if (!dmar_find_matched_atsr_unit(pdev))
1387 return NULL;
1388
1389 return info;
1390 }
1391
1392 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1393 {
1394 if (!info || !dev_is_pci(info->dev))
1395 return;
1396
1397 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
1398 }
1399
1400 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1401 {
1402 if (!info->dev || !dev_is_pci(info->dev) ||
1403 !pci_ats_enabled(to_pci_dev(info->dev)))
1404 return;
1405
1406 pci_disable_ats(to_pci_dev(info->dev));
1407 }
1408
1409 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1410 u64 addr, unsigned mask)
1411 {
1412 u16 sid, qdep;
1413 unsigned long flags;
1414 struct device_domain_info *info;
1415
1416 spin_lock_irqsave(&device_domain_lock, flags);
1417 list_for_each_entry(info, &domain->devices, link) {
1418 struct pci_dev *pdev;
1419 if (!info->dev || !dev_is_pci(info->dev))
1420 continue;
1421
1422 pdev = to_pci_dev(info->dev);
1423 if (!pci_ats_enabled(pdev))
1424 continue;
1425
1426 sid = info->bus << 8 | info->devfn;
1427 qdep = pci_ats_queue_depth(pdev);
1428 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1429 }
1430 spin_unlock_irqrestore(&device_domain_lock, flags);
1431 }
1432
1433 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1434 unsigned long pfn, unsigned int pages, int ih, int map)
1435 {
1436 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1437 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1438
1439 BUG_ON(pages == 0);
1440
1441 if (ih)
1442 ih = 1 << 6;
1443 /*
1444 * Fallback to domain selective flush if no PSI support or the size is
1445 * too big.
1446 * PSI requires page size to be 2 ^ x, and the base address is naturally
1447 * aligned to the size
1448 */
1449 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1450 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1451 DMA_TLB_DSI_FLUSH);
1452 else
1453 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1454 DMA_TLB_PSI_FLUSH);
1455
1456 /*
1457 * In caching mode, changes of pages from non-present to present require
1458 * flush. However, device IOTLB doesn't need to be flushed in this case.
1459 */
1460 if (!cap_caching_mode(iommu->cap) || !map)
1461 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1462 }
1463
1464 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1465 {
1466 u32 pmen;
1467 unsigned long flags;
1468
1469 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1470 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1471 pmen &= ~DMA_PMEN_EPM;
1472 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1473
1474 /* wait for the protected region status bit to clear */
1475 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1476 readl, !(pmen & DMA_PMEN_PRS), pmen);
1477
1478 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1479 }
1480
1481 static void iommu_enable_translation(struct intel_iommu *iommu)
1482 {
1483 u32 sts;
1484 unsigned long flags;
1485
1486 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1487 iommu->gcmd |= DMA_GCMD_TE;
1488 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1489
1490 /* Make sure hardware complete it */
1491 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1492 readl, (sts & DMA_GSTS_TES), sts);
1493
1494 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1495 }
1496
1497 static void iommu_disable_translation(struct intel_iommu *iommu)
1498 {
1499 u32 sts;
1500 unsigned long flag;
1501
1502 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1503 iommu->gcmd &= ~DMA_GCMD_TE;
1504 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1505
1506 /* Make sure hardware complete it */
1507 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1508 readl, (!(sts & DMA_GSTS_TES)), sts);
1509
1510 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1511 }
1512
1513
1514 static int iommu_init_domains(struct intel_iommu *iommu)
1515 {
1516 unsigned long ndomains;
1517 unsigned long nlongs;
1518
1519 ndomains = cap_ndoms(iommu->cap);
1520 pr_debug("%s: Number of Domains supported <%ld>\n",
1521 iommu->name, ndomains);
1522 nlongs = BITS_TO_LONGS(ndomains);
1523
1524 spin_lock_init(&iommu->lock);
1525
1526 /* TBD: there might be 64K domains,
1527 * consider other allocation for future chip
1528 */
1529 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1530 if (!iommu->domain_ids) {
1531 pr_err("%s: Allocating domain id array failed\n",
1532 iommu->name);
1533 return -ENOMEM;
1534 }
1535 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1536 GFP_KERNEL);
1537 if (!iommu->domains) {
1538 pr_err("%s: Allocating domain array failed\n",
1539 iommu->name);
1540 kfree(iommu->domain_ids);
1541 iommu->domain_ids = NULL;
1542 return -ENOMEM;
1543 }
1544
1545 /*
1546 * if Caching mode is set, then invalid translations are tagged
1547 * with domainid 0. Hence we need to pre-allocate it.
1548 */
1549 if (cap_caching_mode(iommu->cap))
1550 set_bit(0, iommu->domain_ids);
1551 return 0;
1552 }
1553
1554 static void disable_dmar_iommu(struct intel_iommu *iommu)
1555 {
1556 struct dmar_domain *domain;
1557 int i;
1558
1559 if ((iommu->domains) && (iommu->domain_ids)) {
1560 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1561 /*
1562 * Domain id 0 is reserved for invalid translation
1563 * if hardware supports caching mode.
1564 */
1565 if (cap_caching_mode(iommu->cap) && i == 0)
1566 continue;
1567
1568 domain = iommu->domains[i];
1569 clear_bit(i, iommu->domain_ids);
1570 if (domain_detach_iommu(domain, iommu) == 0 &&
1571 !domain_type_is_vm(domain))
1572 domain_exit(domain);
1573 }
1574 }
1575
1576 if (iommu->gcmd & DMA_GCMD_TE)
1577 iommu_disable_translation(iommu);
1578 }
1579
1580 static void free_dmar_iommu(struct intel_iommu *iommu)
1581 {
1582 if ((iommu->domains) && (iommu->domain_ids)) {
1583 kfree(iommu->domains);
1584 kfree(iommu->domain_ids);
1585 iommu->domains = NULL;
1586 iommu->domain_ids = NULL;
1587 }
1588
1589 g_iommus[iommu->seq_id] = NULL;
1590
1591 /* free context mapping */
1592 free_context_table(iommu);
1593 }
1594
1595 static struct dmar_domain *alloc_domain(int flags)
1596 {
1597 /* domain id for virtual machine, it won't be set in context */
1598 static atomic_t vm_domid = ATOMIC_INIT(0);
1599 struct dmar_domain *domain;
1600
1601 domain = alloc_domain_mem();
1602 if (!domain)
1603 return NULL;
1604
1605 memset(domain, 0, sizeof(*domain));
1606 domain->nid = -1;
1607 domain->flags = flags;
1608 spin_lock_init(&domain->iommu_lock);
1609 INIT_LIST_HEAD(&domain->devices);
1610 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1611 domain->id = atomic_inc_return(&vm_domid);
1612
1613 return domain;
1614 }
1615
1616 static int __iommu_attach_domain(struct dmar_domain *domain,
1617 struct intel_iommu *iommu)
1618 {
1619 int num;
1620 unsigned long ndomains;
1621
1622 ndomains = cap_ndoms(iommu->cap);
1623 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1624 if (num < ndomains) {
1625 set_bit(num, iommu->domain_ids);
1626 iommu->domains[num] = domain;
1627 } else {
1628 num = -ENOSPC;
1629 }
1630
1631 return num;
1632 }
1633
1634 static int iommu_attach_domain(struct dmar_domain *domain,
1635 struct intel_iommu *iommu)
1636 {
1637 int num;
1638 unsigned long flags;
1639
1640 spin_lock_irqsave(&iommu->lock, flags);
1641 num = __iommu_attach_domain(domain, iommu);
1642 spin_unlock_irqrestore(&iommu->lock, flags);
1643 if (num < 0)
1644 pr_err("%s: No free domain ids\n", iommu->name);
1645
1646 return num;
1647 }
1648
1649 static int iommu_attach_vm_domain(struct dmar_domain *domain,
1650 struct intel_iommu *iommu)
1651 {
1652 int num;
1653 unsigned long ndomains;
1654
1655 ndomains = cap_ndoms(iommu->cap);
1656 for_each_set_bit(num, iommu->domain_ids, ndomains)
1657 if (iommu->domains[num] == domain)
1658 return num;
1659
1660 return __iommu_attach_domain(domain, iommu);
1661 }
1662
1663 static void iommu_detach_domain(struct dmar_domain *domain,
1664 struct intel_iommu *iommu)
1665 {
1666 unsigned long flags;
1667 int num, ndomains;
1668
1669 spin_lock_irqsave(&iommu->lock, flags);
1670 if (domain_type_is_vm_or_si(domain)) {
1671 ndomains = cap_ndoms(iommu->cap);
1672 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1673 if (iommu->domains[num] == domain) {
1674 clear_bit(num, iommu->domain_ids);
1675 iommu->domains[num] = NULL;
1676 break;
1677 }
1678 }
1679 } else {
1680 clear_bit(domain->id, iommu->domain_ids);
1681 iommu->domains[domain->id] = NULL;
1682 }
1683 spin_unlock_irqrestore(&iommu->lock, flags);
1684 }
1685
1686 static void domain_attach_iommu(struct dmar_domain *domain,
1687 struct intel_iommu *iommu)
1688 {
1689 unsigned long flags;
1690
1691 spin_lock_irqsave(&domain->iommu_lock, flags);
1692 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1693 domain->iommu_count++;
1694 if (domain->iommu_count == 1)
1695 domain->nid = iommu->node;
1696 domain_update_iommu_cap(domain);
1697 }
1698 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1699 }
1700
1701 static int domain_detach_iommu(struct dmar_domain *domain,
1702 struct intel_iommu *iommu)
1703 {
1704 unsigned long flags;
1705 int count = INT_MAX;
1706
1707 spin_lock_irqsave(&domain->iommu_lock, flags);
1708 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1709 count = --domain->iommu_count;
1710 domain_update_iommu_cap(domain);
1711 }
1712 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1713
1714 return count;
1715 }
1716
1717 static struct iova_domain reserved_iova_list;
1718 static struct lock_class_key reserved_rbtree_key;
1719
1720 static int dmar_init_reserved_ranges(void)
1721 {
1722 struct pci_dev *pdev = NULL;
1723 struct iova *iova;
1724 int i;
1725
1726 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1727 DMA_32BIT_PFN);
1728
1729 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1730 &reserved_rbtree_key);
1731
1732 /* IOAPIC ranges shouldn't be accessed by DMA */
1733 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1734 IOVA_PFN(IOAPIC_RANGE_END));
1735 if (!iova) {
1736 pr_err("Reserve IOAPIC range failed\n");
1737 return -ENODEV;
1738 }
1739
1740 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1741 for_each_pci_dev(pdev) {
1742 struct resource *r;
1743
1744 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1745 r = &pdev->resource[i];
1746 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1747 continue;
1748 iova = reserve_iova(&reserved_iova_list,
1749 IOVA_PFN(r->start),
1750 IOVA_PFN(r->end));
1751 if (!iova) {
1752 pr_err("Reserve iova failed\n");
1753 return -ENODEV;
1754 }
1755 }
1756 }
1757 return 0;
1758 }
1759
1760 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1761 {
1762 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1763 }
1764
1765 static inline int guestwidth_to_adjustwidth(int gaw)
1766 {
1767 int agaw;
1768 int r = (gaw - 12) % 9;
1769
1770 if (r == 0)
1771 agaw = gaw;
1772 else
1773 agaw = gaw + 9 - r;
1774 if (agaw > 64)
1775 agaw = 64;
1776 return agaw;
1777 }
1778
1779 static int domain_init(struct dmar_domain *domain, int guest_width)
1780 {
1781 struct intel_iommu *iommu;
1782 int adjust_width, agaw;
1783 unsigned long sagaw;
1784
1785 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1786 DMA_32BIT_PFN);
1787 domain_reserve_special_ranges(domain);
1788
1789 /* calculate AGAW */
1790 iommu = domain_get_iommu(domain);
1791 if (guest_width > cap_mgaw(iommu->cap))
1792 guest_width = cap_mgaw(iommu->cap);
1793 domain->gaw = guest_width;
1794 adjust_width = guestwidth_to_adjustwidth(guest_width);
1795 agaw = width_to_agaw(adjust_width);
1796 sagaw = cap_sagaw(iommu->cap);
1797 if (!test_bit(agaw, &sagaw)) {
1798 /* hardware doesn't support it, choose a bigger one */
1799 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1800 agaw = find_next_bit(&sagaw, 5, agaw);
1801 if (agaw >= 5)
1802 return -ENODEV;
1803 }
1804 domain->agaw = agaw;
1805
1806 if (ecap_coherent(iommu->ecap))
1807 domain->iommu_coherency = 1;
1808 else
1809 domain->iommu_coherency = 0;
1810
1811 if (ecap_sc_support(iommu->ecap))
1812 domain->iommu_snooping = 1;
1813 else
1814 domain->iommu_snooping = 0;
1815
1816 if (intel_iommu_superpage)
1817 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1818 else
1819 domain->iommu_superpage = 0;
1820
1821 domain->nid = iommu->node;
1822
1823 /* always allocate the top pgd */
1824 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1825 if (!domain->pgd)
1826 return -ENOMEM;
1827 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1828 return 0;
1829 }
1830
1831 static void domain_exit(struct dmar_domain *domain)
1832 {
1833 struct page *freelist = NULL;
1834 int i;
1835
1836 /* Domain 0 is reserved, so dont process it */
1837 if (!domain)
1838 return;
1839
1840 /* Flush any lazy unmaps that may reference this domain */
1841 if (!intel_iommu_strict)
1842 flush_unmaps_timeout(0);
1843
1844 /* remove associated devices */
1845 domain_remove_dev_info(domain);
1846
1847 /* destroy iovas */
1848 put_iova_domain(&domain->iovad);
1849
1850 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1851
1852 /* clear attached or cached domains */
1853 rcu_read_lock();
1854 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus)
1855 iommu_detach_domain(domain, g_iommus[i]);
1856 rcu_read_unlock();
1857
1858 dma_free_pagelist(freelist);
1859
1860 free_domain_mem(domain);
1861 }
1862
1863 static int domain_context_mapping_one(struct dmar_domain *domain,
1864 struct intel_iommu *iommu,
1865 u8 bus, u8 devfn, int translation)
1866 {
1867 struct context_entry *context;
1868 unsigned long flags;
1869 struct dma_pte *pgd;
1870 int id;
1871 int agaw;
1872 struct device_domain_info *info = NULL;
1873
1874 pr_debug("Set context mapping for %02x:%02x.%d\n",
1875 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1876
1877 BUG_ON(!domain->pgd);
1878 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1879 translation != CONTEXT_TT_MULTI_LEVEL);
1880
1881 spin_lock_irqsave(&iommu->lock, flags);
1882 context = iommu_context_addr(iommu, bus, devfn, 1);
1883 spin_unlock_irqrestore(&iommu->lock, flags);
1884 if (!context)
1885 return -ENOMEM;
1886 spin_lock_irqsave(&iommu->lock, flags);
1887 if (context_present(context)) {
1888 spin_unlock_irqrestore(&iommu->lock, flags);
1889 return 0;
1890 }
1891
1892 context_clear_entry(context);
1893
1894 id = domain->id;
1895 pgd = domain->pgd;
1896
1897 if (domain_type_is_vm_or_si(domain)) {
1898 if (domain_type_is_vm(domain)) {
1899 id = iommu_attach_vm_domain(domain, iommu);
1900 if (id < 0) {
1901 spin_unlock_irqrestore(&iommu->lock, flags);
1902 pr_err("%s: No free domain ids\n", iommu->name);
1903 return -EFAULT;
1904 }
1905 }
1906
1907 /* Skip top levels of page tables for
1908 * iommu which has less agaw than default.
1909 * Unnecessary for PT mode.
1910 */
1911 if (translation != CONTEXT_TT_PASS_THROUGH) {
1912 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1913 pgd = phys_to_virt(dma_pte_addr(pgd));
1914 if (!dma_pte_present(pgd)) {
1915 spin_unlock_irqrestore(&iommu->lock, flags);
1916 return -ENOMEM;
1917 }
1918 }
1919 }
1920 }
1921
1922 context_set_domain_id(context, id);
1923
1924 if (translation != CONTEXT_TT_PASS_THROUGH) {
1925 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
1926 translation = info ? CONTEXT_TT_DEV_IOTLB :
1927 CONTEXT_TT_MULTI_LEVEL;
1928 }
1929 /*
1930 * In pass through mode, AW must be programmed to indicate the largest
1931 * AGAW value supported by hardware. And ASR is ignored by hardware.
1932 */
1933 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1934 context_set_address_width(context, iommu->msagaw);
1935 else {
1936 context_set_address_root(context, virt_to_phys(pgd));
1937 context_set_address_width(context, iommu->agaw);
1938 }
1939
1940 context_set_translation_type(context, translation);
1941 context_set_fault_enable(context);
1942 context_set_present(context);
1943 domain_flush_cache(domain, context, sizeof(*context));
1944
1945 /*
1946 * It's a non-present to present mapping. If hardware doesn't cache
1947 * non-present entry we only need to flush the write-buffer. If the
1948 * _does_ cache non-present entries, then it does so in the special
1949 * domain #0, which we have to flush:
1950 */
1951 if (cap_caching_mode(iommu->cap)) {
1952 iommu->flush.flush_context(iommu, 0,
1953 (((u16)bus) << 8) | devfn,
1954 DMA_CCMD_MASK_NOBIT,
1955 DMA_CCMD_DEVICE_INVL);
1956 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
1957 } else {
1958 iommu_flush_write_buffer(iommu);
1959 }
1960 iommu_enable_dev_iotlb(info);
1961 spin_unlock_irqrestore(&iommu->lock, flags);
1962
1963 domain_attach_iommu(domain, iommu);
1964
1965 return 0;
1966 }
1967
1968 struct domain_context_mapping_data {
1969 struct dmar_domain *domain;
1970 struct intel_iommu *iommu;
1971 int translation;
1972 };
1973
1974 static int domain_context_mapping_cb(struct pci_dev *pdev,
1975 u16 alias, void *opaque)
1976 {
1977 struct domain_context_mapping_data *data = opaque;
1978
1979 return domain_context_mapping_one(data->domain, data->iommu,
1980 PCI_BUS_NUM(alias), alias & 0xff,
1981 data->translation);
1982 }
1983
1984 static int
1985 domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1986 int translation)
1987 {
1988 struct intel_iommu *iommu;
1989 u8 bus, devfn;
1990 struct domain_context_mapping_data data;
1991
1992 iommu = device_to_iommu(dev, &bus, &devfn);
1993 if (!iommu)
1994 return -ENODEV;
1995
1996 if (!dev_is_pci(dev))
1997 return domain_context_mapping_one(domain, iommu, bus, devfn,
1998 translation);
1999
2000 data.domain = domain;
2001 data.iommu = iommu;
2002 data.translation = translation;
2003
2004 return pci_for_each_dma_alias(to_pci_dev(dev),
2005 &domain_context_mapping_cb, &data);
2006 }
2007
2008 static int domain_context_mapped_cb(struct pci_dev *pdev,
2009 u16 alias, void *opaque)
2010 {
2011 struct intel_iommu *iommu = opaque;
2012
2013 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2014 }
2015
2016 static int domain_context_mapped(struct device *dev)
2017 {
2018 struct intel_iommu *iommu;
2019 u8 bus, devfn;
2020
2021 iommu = device_to_iommu(dev, &bus, &devfn);
2022 if (!iommu)
2023 return -ENODEV;
2024
2025 if (!dev_is_pci(dev))
2026 return device_context_mapped(iommu, bus, devfn);
2027
2028 return !pci_for_each_dma_alias(to_pci_dev(dev),
2029 domain_context_mapped_cb, iommu);
2030 }
2031
2032 /* Returns a number of VTD pages, but aligned to MM page size */
2033 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2034 size_t size)
2035 {
2036 host_addr &= ~PAGE_MASK;
2037 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2038 }
2039
2040 /* Return largest possible superpage level for a given mapping */
2041 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2042 unsigned long iov_pfn,
2043 unsigned long phy_pfn,
2044 unsigned long pages)
2045 {
2046 int support, level = 1;
2047 unsigned long pfnmerge;
2048
2049 support = domain->iommu_superpage;
2050
2051 /* To use a large page, the virtual *and* physical addresses
2052 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2053 of them will mean we have to use smaller pages. So just
2054 merge them and check both at once. */
2055 pfnmerge = iov_pfn | phy_pfn;
2056
2057 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2058 pages >>= VTD_STRIDE_SHIFT;
2059 if (!pages)
2060 break;
2061 pfnmerge >>= VTD_STRIDE_SHIFT;
2062 level++;
2063 support--;
2064 }
2065 return level;
2066 }
2067
2068 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2069 struct scatterlist *sg, unsigned long phys_pfn,
2070 unsigned long nr_pages, int prot)
2071 {
2072 struct dma_pte *first_pte = NULL, *pte = NULL;
2073 phys_addr_t uninitialized_var(pteval);
2074 unsigned long sg_res = 0;
2075 unsigned int largepage_lvl = 0;
2076 unsigned long lvl_pages = 0;
2077
2078 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2079
2080 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2081 return -EINVAL;
2082
2083 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2084
2085 if (!sg) {
2086 sg_res = nr_pages;
2087 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2088 }
2089
2090 while (nr_pages > 0) {
2091 uint64_t tmp;
2092
2093 if (!sg_res) {
2094 sg_res = aligned_nrpages(sg->offset, sg->length);
2095 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2096 sg->dma_length = sg->length;
2097 pteval = page_to_phys(sg_page(sg)) | prot;
2098 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2099 }
2100
2101 if (!pte) {
2102 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2103
2104 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2105 if (!pte)
2106 return -ENOMEM;
2107 /* It is large page*/
2108 if (largepage_lvl > 1) {
2109 pteval |= DMA_PTE_LARGE_PAGE;
2110 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2111 /*
2112 * Ensure that old small page tables are
2113 * removed to make room for superpage,
2114 * if they exist.
2115 */
2116 dma_pte_free_pagetable(domain, iov_pfn,
2117 iov_pfn + lvl_pages - 1);
2118 } else {
2119 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2120 }
2121
2122 }
2123 /* We don't need lock here, nobody else
2124 * touches the iova range
2125 */
2126 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2127 if (tmp) {
2128 static int dumps = 5;
2129 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2130 iov_pfn, tmp, (unsigned long long)pteval);
2131 if (dumps) {
2132 dumps--;
2133 debug_dma_dump_mappings(NULL);
2134 }
2135 WARN_ON(1);
2136 }
2137
2138 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2139
2140 BUG_ON(nr_pages < lvl_pages);
2141 BUG_ON(sg_res < lvl_pages);
2142
2143 nr_pages -= lvl_pages;
2144 iov_pfn += lvl_pages;
2145 phys_pfn += lvl_pages;
2146 pteval += lvl_pages * VTD_PAGE_SIZE;
2147 sg_res -= lvl_pages;
2148
2149 /* If the next PTE would be the first in a new page, then we
2150 need to flush the cache on the entries we've just written.
2151 And then we'll need to recalculate 'pte', so clear it and
2152 let it get set again in the if (!pte) block above.
2153
2154 If we're done (!nr_pages) we need to flush the cache too.
2155
2156 Also if we've been setting superpages, we may need to
2157 recalculate 'pte' and switch back to smaller pages for the
2158 end of the mapping, if the trailing size is not enough to
2159 use another superpage (i.e. sg_res < lvl_pages). */
2160 pte++;
2161 if (!nr_pages || first_pte_in_page(pte) ||
2162 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2163 domain_flush_cache(domain, first_pte,
2164 (void *)pte - (void *)first_pte);
2165 pte = NULL;
2166 }
2167
2168 if (!sg_res && nr_pages)
2169 sg = sg_next(sg);
2170 }
2171 return 0;
2172 }
2173
2174 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2175 struct scatterlist *sg, unsigned long nr_pages,
2176 int prot)
2177 {
2178 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2179 }
2180
2181 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2182 unsigned long phys_pfn, unsigned long nr_pages,
2183 int prot)
2184 {
2185 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2186 }
2187
2188 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
2189 {
2190 if (!iommu)
2191 return;
2192
2193 clear_context_table(iommu, bus, devfn);
2194 iommu->flush.flush_context(iommu, 0, 0, 0,
2195 DMA_CCMD_GLOBAL_INVL);
2196 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2197 }
2198
2199 static inline void unlink_domain_info(struct device_domain_info *info)
2200 {
2201 assert_spin_locked(&device_domain_lock);
2202 list_del(&info->link);
2203 list_del(&info->global);
2204 if (info->dev)
2205 info->dev->archdata.iommu = NULL;
2206 }
2207
2208 static void domain_remove_dev_info(struct dmar_domain *domain)
2209 {
2210 struct device_domain_info *info, *tmp;
2211 unsigned long flags;
2212
2213 spin_lock_irqsave(&device_domain_lock, flags);
2214 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
2215 unlink_domain_info(info);
2216 spin_unlock_irqrestore(&device_domain_lock, flags);
2217
2218 iommu_disable_dev_iotlb(info);
2219 iommu_detach_dev(info->iommu, info->bus, info->devfn);
2220
2221 if (domain_type_is_vm(domain)) {
2222 iommu_detach_dependent_devices(info->iommu, info->dev);
2223 domain_detach_iommu(domain, info->iommu);
2224 }
2225
2226 free_devinfo_mem(info);
2227 spin_lock_irqsave(&device_domain_lock, flags);
2228 }
2229 spin_unlock_irqrestore(&device_domain_lock, flags);
2230 }
2231
2232 /*
2233 * find_domain
2234 * Note: we use struct device->archdata.iommu stores the info
2235 */
2236 static struct dmar_domain *find_domain(struct device *dev)
2237 {
2238 struct device_domain_info *info;
2239
2240 /* No lock here, assumes no domain exit in normal case */
2241 info = dev->archdata.iommu;
2242 if (info)
2243 return info->domain;
2244 return NULL;
2245 }
2246
2247 static inline struct device_domain_info *
2248 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2249 {
2250 struct device_domain_info *info;
2251
2252 list_for_each_entry(info, &device_domain_list, global)
2253 if (info->iommu->segment == segment && info->bus == bus &&
2254 info->devfn == devfn)
2255 return info;
2256
2257 return NULL;
2258 }
2259
2260 static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
2261 int bus, int devfn,
2262 struct device *dev,
2263 struct dmar_domain *domain)
2264 {
2265 struct dmar_domain *found = NULL;
2266 struct device_domain_info *info;
2267 unsigned long flags;
2268
2269 info = alloc_devinfo_mem();
2270 if (!info)
2271 return NULL;
2272
2273 info->bus = bus;
2274 info->devfn = devfn;
2275 info->dev = dev;
2276 info->domain = domain;
2277 info->iommu = iommu;
2278
2279 spin_lock_irqsave(&device_domain_lock, flags);
2280 if (dev)
2281 found = find_domain(dev);
2282 else {
2283 struct device_domain_info *info2;
2284 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2285 if (info2)
2286 found = info2->domain;
2287 }
2288 if (found) {
2289 spin_unlock_irqrestore(&device_domain_lock, flags);
2290 free_devinfo_mem(info);
2291 /* Caller must free the original domain */
2292 return found;
2293 }
2294
2295 list_add(&info->link, &domain->devices);
2296 list_add(&info->global, &device_domain_list);
2297 if (dev)
2298 dev->archdata.iommu = info;
2299 spin_unlock_irqrestore(&device_domain_lock, flags);
2300
2301 return domain;
2302 }
2303
2304 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2305 {
2306 *(u16 *)opaque = alias;
2307 return 0;
2308 }
2309
2310 /* domain is initialized */
2311 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2312 {
2313 struct dmar_domain *domain, *tmp;
2314 struct intel_iommu *iommu;
2315 struct device_domain_info *info;
2316 u16 dma_alias;
2317 unsigned long flags;
2318 u8 bus, devfn;
2319
2320 domain = find_domain(dev);
2321 if (domain)
2322 return domain;
2323
2324 iommu = device_to_iommu(dev, &bus, &devfn);
2325 if (!iommu)
2326 return NULL;
2327
2328 if (dev_is_pci(dev)) {
2329 struct pci_dev *pdev = to_pci_dev(dev);
2330
2331 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2332
2333 spin_lock_irqsave(&device_domain_lock, flags);
2334 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2335 PCI_BUS_NUM(dma_alias),
2336 dma_alias & 0xff);
2337 if (info) {
2338 iommu = info->iommu;
2339 domain = info->domain;
2340 }
2341 spin_unlock_irqrestore(&device_domain_lock, flags);
2342
2343 /* DMA alias already has a domain, uses it */
2344 if (info)
2345 goto found_domain;
2346 }
2347
2348 /* Allocate and initialize new domain for the device */
2349 domain = alloc_domain(0);
2350 if (!domain)
2351 return NULL;
2352 domain->id = iommu_attach_domain(domain, iommu);
2353 if (domain->id < 0) {
2354 free_domain_mem(domain);
2355 return NULL;
2356 }
2357 domain_attach_iommu(domain, iommu);
2358 if (domain_init(domain, gaw)) {
2359 domain_exit(domain);
2360 return NULL;
2361 }
2362
2363 /* register PCI DMA alias device */
2364 if (dev_is_pci(dev)) {
2365 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2366 dma_alias & 0xff, NULL, domain);
2367
2368 if (!tmp || tmp != domain) {
2369 domain_exit(domain);
2370 domain = tmp;
2371 }
2372
2373 if (!domain)
2374 return NULL;
2375 }
2376
2377 found_domain:
2378 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2379
2380 if (!tmp || tmp != domain) {
2381 domain_exit(domain);
2382 domain = tmp;
2383 }
2384
2385 return domain;
2386 }
2387
2388 static int iommu_identity_mapping;
2389 #define IDENTMAP_ALL 1
2390 #define IDENTMAP_GFX 2
2391 #define IDENTMAP_AZALIA 4
2392
2393 static int iommu_domain_identity_map(struct dmar_domain *domain,
2394 unsigned long long start,
2395 unsigned long long end)
2396 {
2397 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2398 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2399
2400 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2401 dma_to_mm_pfn(last_vpfn))) {
2402 pr_err("Reserving iova failed\n");
2403 return -ENOMEM;
2404 }
2405
2406 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2407 start, end, domain->id);
2408 /*
2409 * RMRR range might have overlap with physical memory range,
2410 * clear it first
2411 */
2412 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2413
2414 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2415 last_vpfn - first_vpfn + 1,
2416 DMA_PTE_READ|DMA_PTE_WRITE);
2417 }
2418
2419 static int iommu_prepare_identity_map(struct device *dev,
2420 unsigned long long start,
2421 unsigned long long end)
2422 {
2423 struct dmar_domain *domain;
2424 int ret;
2425
2426 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2427 if (!domain)
2428 return -ENOMEM;
2429
2430 /* For _hardware_ passthrough, don't bother. But for software
2431 passthrough, we do it anyway -- it may indicate a memory
2432 range which is reserved in E820, so which didn't get set
2433 up to start with in si_domain */
2434 if (domain == si_domain && hw_pass_through) {
2435 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2436 dev_name(dev), start, end);
2437 return 0;
2438 }
2439
2440 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2441 dev_name(dev), start, end);
2442
2443 if (end < start) {
2444 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2445 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2446 dmi_get_system_info(DMI_BIOS_VENDOR),
2447 dmi_get_system_info(DMI_BIOS_VERSION),
2448 dmi_get_system_info(DMI_PRODUCT_VERSION));
2449 ret = -EIO;
2450 goto error;
2451 }
2452
2453 if (end >> agaw_to_width(domain->agaw)) {
2454 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2455 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2456 agaw_to_width(domain->agaw),
2457 dmi_get_system_info(DMI_BIOS_VENDOR),
2458 dmi_get_system_info(DMI_BIOS_VERSION),
2459 dmi_get_system_info(DMI_PRODUCT_VERSION));
2460 ret = -EIO;
2461 goto error;
2462 }
2463
2464 ret = iommu_domain_identity_map(domain, start, end);
2465 if (ret)
2466 goto error;
2467
2468 /* context entry init */
2469 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2470 if (ret)
2471 goto error;
2472
2473 return 0;
2474
2475 error:
2476 domain_exit(domain);
2477 return ret;
2478 }
2479
2480 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2481 struct device *dev)
2482 {
2483 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2484 return 0;
2485 return iommu_prepare_identity_map(dev, rmrr->base_address,
2486 rmrr->end_address);
2487 }
2488
2489 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2490 static inline void iommu_prepare_isa(void)
2491 {
2492 struct pci_dev *pdev;
2493 int ret;
2494
2495 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2496 if (!pdev)
2497 return;
2498
2499 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2500 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2501
2502 if (ret)
2503 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2504
2505 pci_dev_put(pdev);
2506 }
2507 #else
2508 static inline void iommu_prepare_isa(void)
2509 {
2510 return;
2511 }
2512 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2513
2514 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2515
2516 static int __init si_domain_init(int hw)
2517 {
2518 struct dmar_drhd_unit *drhd;
2519 struct intel_iommu *iommu;
2520 int nid, ret = 0;
2521 bool first = true;
2522
2523 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2524 if (!si_domain)
2525 return -EFAULT;
2526
2527 for_each_active_iommu(iommu, drhd) {
2528 ret = iommu_attach_domain(si_domain, iommu);
2529 if (ret < 0) {
2530 domain_exit(si_domain);
2531 return -EFAULT;
2532 } else if (first) {
2533 si_domain->id = ret;
2534 first = false;
2535 } else if (si_domain->id != ret) {
2536 domain_exit(si_domain);
2537 return -EFAULT;
2538 }
2539 domain_attach_iommu(si_domain, iommu);
2540 }
2541
2542 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2543 domain_exit(si_domain);
2544 return -EFAULT;
2545 }
2546
2547 pr_debug("Identity mapping domain is domain %d\n",
2548 si_domain->id);
2549
2550 if (hw)
2551 return 0;
2552
2553 for_each_online_node(nid) {
2554 unsigned long start_pfn, end_pfn;
2555 int i;
2556
2557 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2558 ret = iommu_domain_identity_map(si_domain,
2559 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2560 if (ret)
2561 return ret;
2562 }
2563 }
2564
2565 return 0;
2566 }
2567
2568 static int identity_mapping(struct device *dev)
2569 {
2570 struct device_domain_info *info;
2571
2572 if (likely(!iommu_identity_mapping))
2573 return 0;
2574
2575 info = dev->archdata.iommu;
2576 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2577 return (info->domain == si_domain);
2578
2579 return 0;
2580 }
2581
2582 static int domain_add_dev_info(struct dmar_domain *domain,
2583 struct device *dev, int translation)
2584 {
2585 struct dmar_domain *ndomain;
2586 struct intel_iommu *iommu;
2587 u8 bus, devfn;
2588 int ret;
2589
2590 iommu = device_to_iommu(dev, &bus, &devfn);
2591 if (!iommu)
2592 return -ENODEV;
2593
2594 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2595 if (ndomain != domain)
2596 return -EBUSY;
2597
2598 ret = domain_context_mapping(domain, dev, translation);
2599 if (ret) {
2600 domain_remove_one_dev_info(domain, dev);
2601 return ret;
2602 }
2603
2604 return 0;
2605 }
2606
2607 static bool device_has_rmrr(struct device *dev)
2608 {
2609 struct dmar_rmrr_unit *rmrr;
2610 struct device *tmp;
2611 int i;
2612
2613 rcu_read_lock();
2614 for_each_rmrr_units(rmrr) {
2615 /*
2616 * Return TRUE if this RMRR contains the device that
2617 * is passed in.
2618 */
2619 for_each_active_dev_scope(rmrr->devices,
2620 rmrr->devices_cnt, i, tmp)
2621 if (tmp == dev) {
2622 rcu_read_unlock();
2623 return true;
2624 }
2625 }
2626 rcu_read_unlock();
2627 return false;
2628 }
2629
2630 /*
2631 * There are a couple cases where we need to restrict the functionality of
2632 * devices associated with RMRRs. The first is when evaluating a device for
2633 * identity mapping because problems exist when devices are moved in and out
2634 * of domains and their respective RMRR information is lost. This means that
2635 * a device with associated RMRRs will never be in a "passthrough" domain.
2636 * The second is use of the device through the IOMMU API. This interface
2637 * expects to have full control of the IOVA space for the device. We cannot
2638 * satisfy both the requirement that RMRR access is maintained and have an
2639 * unencumbered IOVA space. We also have no ability to quiesce the device's
2640 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2641 * We therefore prevent devices associated with an RMRR from participating in
2642 * the IOMMU API, which eliminates them from device assignment.
2643 *
2644 * In both cases we assume that PCI USB devices with RMRRs have them largely
2645 * for historical reasons and that the RMRR space is not actively used post
2646 * boot. This exclusion may change if vendors begin to abuse it.
2647 *
2648 * The same exception is made for graphics devices, with the requirement that
2649 * any use of the RMRR regions will be torn down before assigning the device
2650 * to a guest.
2651 */
2652 static bool device_is_rmrr_locked(struct device *dev)
2653 {
2654 if (!device_has_rmrr(dev))
2655 return false;
2656
2657 if (dev_is_pci(dev)) {
2658 struct pci_dev *pdev = to_pci_dev(dev);
2659
2660 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2661 return false;
2662 }
2663
2664 return true;
2665 }
2666
2667 static int iommu_should_identity_map(struct device *dev, int startup)
2668 {
2669
2670 if (dev_is_pci(dev)) {
2671 struct pci_dev *pdev = to_pci_dev(dev);
2672
2673 if (device_is_rmrr_locked(dev))
2674 return 0;
2675
2676 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2677 return 1;
2678
2679 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2680 return 1;
2681
2682 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2683 return 0;
2684
2685 /*
2686 * We want to start off with all devices in the 1:1 domain, and
2687 * take them out later if we find they can't access all of memory.
2688 *
2689 * However, we can't do this for PCI devices behind bridges,
2690 * because all PCI devices behind the same bridge will end up
2691 * with the same source-id on their transactions.
2692 *
2693 * Practically speaking, we can't change things around for these
2694 * devices at run-time, because we can't be sure there'll be no
2695 * DMA transactions in flight for any of their siblings.
2696 *
2697 * So PCI devices (unless they're on the root bus) as well as
2698 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2699 * the 1:1 domain, just in _case_ one of their siblings turns out
2700 * not to be able to map all of memory.
2701 */
2702 if (!pci_is_pcie(pdev)) {
2703 if (!pci_is_root_bus(pdev->bus))
2704 return 0;
2705 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2706 return 0;
2707 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2708 return 0;
2709 } else {
2710 if (device_has_rmrr(dev))
2711 return 0;
2712 }
2713
2714 /*
2715 * At boot time, we don't yet know if devices will be 64-bit capable.
2716 * Assume that they will — if they turn out not to be, then we can
2717 * take them out of the 1:1 domain later.
2718 */
2719 if (!startup) {
2720 /*
2721 * If the device's dma_mask is less than the system's memory
2722 * size then this is not a candidate for identity mapping.
2723 */
2724 u64 dma_mask = *dev->dma_mask;
2725
2726 if (dev->coherent_dma_mask &&
2727 dev->coherent_dma_mask < dma_mask)
2728 dma_mask = dev->coherent_dma_mask;
2729
2730 return dma_mask >= dma_get_required_mask(dev);
2731 }
2732
2733 return 1;
2734 }
2735
2736 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2737 {
2738 int ret;
2739
2740 if (!iommu_should_identity_map(dev, 1))
2741 return 0;
2742
2743 ret = domain_add_dev_info(si_domain, dev,
2744 hw ? CONTEXT_TT_PASS_THROUGH :
2745 CONTEXT_TT_MULTI_LEVEL);
2746 if (!ret)
2747 pr_info("%s identity mapping for device %s\n",
2748 hw ? "Hardware" : "Software", dev_name(dev));
2749 else if (ret == -ENODEV)
2750 /* device not associated with an iommu */
2751 ret = 0;
2752
2753 return ret;
2754 }
2755
2756
2757 static int __init iommu_prepare_static_identity_mapping(int hw)
2758 {
2759 struct pci_dev *pdev = NULL;
2760 struct dmar_drhd_unit *drhd;
2761 struct intel_iommu *iommu;
2762 struct device *dev;
2763 int i;
2764 int ret = 0;
2765
2766 for_each_pci_dev(pdev) {
2767 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2768 if (ret)
2769 return ret;
2770 }
2771
2772 for_each_active_iommu(iommu, drhd)
2773 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2774 struct acpi_device_physical_node *pn;
2775 struct acpi_device *adev;
2776
2777 if (dev->bus != &acpi_bus_type)
2778 continue;
2779
2780 adev= to_acpi_device(dev);
2781 mutex_lock(&adev->physical_node_lock);
2782 list_for_each_entry(pn, &adev->physical_node_list, node) {
2783 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2784 if (ret)
2785 break;
2786 }
2787 mutex_unlock(&adev->physical_node_lock);
2788 if (ret)
2789 return ret;
2790 }
2791
2792 return 0;
2793 }
2794
2795 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2796 {
2797 /*
2798 * Start from the sane iommu hardware state.
2799 * If the queued invalidation is already initialized by us
2800 * (for example, while enabling interrupt-remapping) then
2801 * we got the things already rolling from a sane state.
2802 */
2803 if (!iommu->qi) {
2804 /*
2805 * Clear any previous faults.
2806 */
2807 dmar_fault(-1, iommu);
2808 /*
2809 * Disable queued invalidation if supported and already enabled
2810 * before OS handover.
2811 */
2812 dmar_disable_qi(iommu);
2813 }
2814
2815 if (dmar_enable_qi(iommu)) {
2816 /*
2817 * Queued Invalidate not enabled, use Register Based Invalidate
2818 */
2819 iommu->flush.flush_context = __iommu_flush_context;
2820 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2821 pr_info("%s: Using Register based invalidation\n",
2822 iommu->name);
2823 } else {
2824 iommu->flush.flush_context = qi_flush_context;
2825 iommu->flush.flush_iotlb = qi_flush_iotlb;
2826 pr_info("%s: Using Queued invalidation\n", iommu->name);
2827 }
2828 }
2829
2830 static int copy_context_table(struct intel_iommu *iommu,
2831 struct root_entry *old_re,
2832 struct context_entry **tbl,
2833 int bus, bool ext)
2834 {
2835 struct context_entry *old_ce = NULL, *new_ce = NULL, ce;
2836 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2837 phys_addr_t old_ce_phys;
2838
2839 tbl_idx = ext ? bus * 2 : bus;
2840
2841 for (devfn = 0; devfn < 256; devfn++) {
2842 /* First calculate the correct index */
2843 idx = (ext ? devfn * 2 : devfn) % 256;
2844
2845 if (idx == 0) {
2846 /* First save what we may have and clean up */
2847 if (new_ce) {
2848 tbl[tbl_idx] = new_ce;
2849 __iommu_flush_cache(iommu, new_ce,
2850 VTD_PAGE_SIZE);
2851 pos = 1;
2852 }
2853
2854 if (old_ce)
2855 iounmap(old_ce);
2856
2857 ret = 0;
2858 if (devfn < 0x80)
2859 old_ce_phys = root_entry_lctp(old_re);
2860 else
2861 old_ce_phys = root_entry_uctp(old_re);
2862
2863 if (!old_ce_phys) {
2864 if (ext && devfn == 0) {
2865 /* No LCTP, try UCTP */
2866 devfn = 0x7f;
2867 continue;
2868 } else {
2869 goto out;
2870 }
2871 }
2872
2873 ret = -ENOMEM;
2874 old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2875 if (!old_ce)
2876 goto out;
2877
2878 new_ce = alloc_pgtable_page(iommu->node);
2879 if (!new_ce)
2880 goto out_unmap;
2881
2882 ret = 0;
2883 }
2884
2885 /* Now copy the context entry */
2886 ce = old_ce[idx];
2887
2888 if (!__context_present(&ce))
2889 continue;
2890
2891 did = context_domain_id(&ce);
2892 if (did >= 0 && did < cap_ndoms(iommu->cap))
2893 set_bit(did, iommu->domain_ids);
2894
2895 /*
2896 * We need a marker for copied context entries. This
2897 * marker needs to work for the old format as well as
2898 * for extended context entries.
2899 *
2900 * Bit 67 of the context entry is used. In the old
2901 * format this bit is available to software, in the
2902 * extended format it is the PGE bit, but PGE is ignored
2903 * by HW if PASIDs are disabled (and thus still
2904 * available).
2905 *
2906 * So disable PASIDs first and then mark the entry
2907 * copied. This means that we don't copy PASID
2908 * translations from the old kernel, but this is fine as
2909 * faults there are not fatal.
2910 */
2911 context_clear_pasid_enable(&ce);
2912 context_set_copied(&ce);
2913
2914 new_ce[idx] = ce;
2915 }
2916
2917 tbl[tbl_idx + pos] = new_ce;
2918
2919 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2920
2921 out_unmap:
2922 iounmap(old_ce);
2923
2924 out:
2925 return ret;
2926 }
2927
2928 static int copy_translation_tables(struct intel_iommu *iommu)
2929 {
2930 struct context_entry **ctxt_tbls;
2931 struct root_entry *old_rt;
2932 phys_addr_t old_rt_phys;
2933 int ctxt_table_entries;
2934 unsigned long flags;
2935 u64 rtaddr_reg;
2936 int bus, ret;
2937 bool new_ext, ext;
2938
2939 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2940 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
2941 new_ext = !!ecap_ecs(iommu->ecap);
2942
2943 /*
2944 * The RTT bit can only be changed when translation is disabled,
2945 * but disabling translation means to open a window for data
2946 * corruption. So bail out and don't copy anything if we would
2947 * have to change the bit.
2948 */
2949 if (new_ext != ext)
2950 return -EINVAL;
2951
2952 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2953 if (!old_rt_phys)
2954 return -EINVAL;
2955
2956 old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
2957 if (!old_rt)
2958 return -ENOMEM;
2959
2960 /* This is too big for the stack - allocate it from slab */
2961 ctxt_table_entries = ext ? 512 : 256;
2962 ret = -ENOMEM;
2963 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
2964 if (!ctxt_tbls)
2965 goto out_unmap;
2966
2967 for (bus = 0; bus < 256; bus++) {
2968 ret = copy_context_table(iommu, &old_rt[bus],
2969 ctxt_tbls, bus, ext);
2970 if (ret) {
2971 pr_err("%s: Failed to copy context table for bus %d\n",
2972 iommu->name, bus);
2973 continue;
2974 }
2975 }
2976
2977 spin_lock_irqsave(&iommu->lock, flags);
2978
2979 /* Context tables are copied, now write them to the root_entry table */
2980 for (bus = 0; bus < 256; bus++) {
2981 int idx = ext ? bus * 2 : bus;
2982 u64 val;
2983
2984 if (ctxt_tbls[idx]) {
2985 val = virt_to_phys(ctxt_tbls[idx]) | 1;
2986 iommu->root_entry[bus].lo = val;
2987 }
2988
2989 if (!ext || !ctxt_tbls[idx + 1])
2990 continue;
2991
2992 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2993 iommu->root_entry[bus].hi = val;
2994 }
2995
2996 spin_unlock_irqrestore(&iommu->lock, flags);
2997
2998 kfree(ctxt_tbls);
2999
3000 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3001
3002 ret = 0;
3003
3004 out_unmap:
3005 iounmap(old_rt);
3006
3007 return ret;
3008 }
3009
3010 static int __init init_dmars(void)
3011 {
3012 struct dmar_drhd_unit *drhd;
3013 struct dmar_rmrr_unit *rmrr;
3014 bool copied_tables = false;
3015 struct device *dev;
3016 struct intel_iommu *iommu;
3017 int i, ret;
3018
3019 /*
3020 * for each drhd
3021 * allocate root
3022 * initialize and program root entry to not present
3023 * endfor
3024 */
3025 for_each_drhd_unit(drhd) {
3026 /*
3027 * lock not needed as this is only incremented in the single
3028 * threaded kernel __init code path all other access are read
3029 * only
3030 */
3031 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3032 g_num_of_iommus++;
3033 continue;
3034 }
3035 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3036 }
3037
3038 /* Preallocate enough resources for IOMMU hot-addition */
3039 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3040 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3041
3042 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3043 GFP_KERNEL);
3044 if (!g_iommus) {
3045 pr_err("Allocating global iommu array failed\n");
3046 ret = -ENOMEM;
3047 goto error;
3048 }
3049
3050 deferred_flush = kzalloc(g_num_of_iommus *
3051 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3052 if (!deferred_flush) {
3053 ret = -ENOMEM;
3054 goto free_g_iommus;
3055 }
3056
3057 for_each_active_iommu(iommu, drhd) {
3058 g_iommus[iommu->seq_id] = iommu;
3059
3060 intel_iommu_init_qi(iommu);
3061
3062 ret = iommu_init_domains(iommu);
3063 if (ret)
3064 goto free_iommu;
3065
3066 init_translation_status(iommu);
3067
3068 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3069 iommu_disable_translation(iommu);
3070 clear_translation_pre_enabled(iommu);
3071 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3072 iommu->name);
3073 }
3074
3075 /*
3076 * TBD:
3077 * we could share the same root & context tables
3078 * among all IOMMU's. Need to Split it later.
3079 */
3080 ret = iommu_alloc_root_entry(iommu);
3081 if (ret)
3082 goto free_iommu;
3083
3084 if (translation_pre_enabled(iommu)) {
3085 pr_info("Translation already enabled - trying to copy translation structures\n");
3086
3087 ret = copy_translation_tables(iommu);
3088 if (ret) {
3089 /*
3090 * We found the IOMMU with translation
3091 * enabled - but failed to copy over the
3092 * old root-entry table. Try to proceed
3093 * by disabling translation now and
3094 * allocating a clean root-entry table.
3095 * This might cause DMAR faults, but
3096 * probably the dump will still succeed.
3097 */
3098 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3099 iommu->name);
3100 iommu_disable_translation(iommu);
3101 clear_translation_pre_enabled(iommu);
3102 } else {
3103 pr_info("Copied translation tables from previous kernel for %s\n",
3104 iommu->name);
3105 copied_tables = true;
3106 }
3107 }
3108
3109 iommu_flush_write_buffer(iommu);
3110 iommu_set_root_entry(iommu);
3111 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3112 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3113
3114 if (!ecap_pass_through(iommu->ecap))
3115 hw_pass_through = 0;
3116 }
3117
3118 if (iommu_pass_through)
3119 iommu_identity_mapping |= IDENTMAP_ALL;
3120
3121 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3122 iommu_identity_mapping |= IDENTMAP_GFX;
3123 #endif
3124
3125 if (iommu_identity_mapping) {
3126 ret = si_domain_init(hw_pass_through);
3127 if (ret)
3128 goto free_iommu;
3129 }
3130
3131 check_tylersburg_isoch();
3132
3133 /*
3134 * If we copied translations from a previous kernel in the kdump
3135 * case, we can not assign the devices to domains now, as that
3136 * would eliminate the old mappings. So skip this part and defer
3137 * the assignment to device driver initialization time.
3138 */
3139 if (copied_tables)
3140 goto domains_done;
3141
3142 /*
3143 * If pass through is not set or not enabled, setup context entries for
3144 * identity mappings for rmrr, gfx, and isa and may fall back to static
3145 * identity mapping if iommu_identity_mapping is set.
3146 */
3147 if (iommu_identity_mapping) {
3148 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3149 if (ret) {
3150 pr_crit("Failed to setup IOMMU pass-through\n");
3151 goto free_iommu;
3152 }
3153 }
3154 /*
3155 * For each rmrr
3156 * for each dev attached to rmrr
3157 * do
3158 * locate drhd for dev, alloc domain for dev
3159 * allocate free domain
3160 * allocate page table entries for rmrr
3161 * if context not allocated for bus
3162 * allocate and init context
3163 * set present in root table for this bus
3164 * init context with domain, translation etc
3165 * endfor
3166 * endfor
3167 */
3168 pr_info("Setting RMRR:\n");
3169 for_each_rmrr_units(rmrr) {
3170 /* some BIOS lists non-exist devices in DMAR table. */
3171 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3172 i, dev) {
3173 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3174 if (ret)
3175 pr_err("Mapping reserved region failed\n");
3176 }
3177 }
3178
3179 iommu_prepare_isa();
3180
3181 domains_done:
3182
3183 /*
3184 * for each drhd
3185 * enable fault log
3186 * global invalidate context cache
3187 * global invalidate iotlb
3188 * enable translation
3189 */
3190 for_each_iommu(iommu, drhd) {
3191 if (drhd->ignored) {
3192 /*
3193 * we always have to disable PMRs or DMA may fail on
3194 * this device
3195 */
3196 if (force_on)
3197 iommu_disable_protect_mem_regions(iommu);
3198 continue;
3199 }
3200
3201 iommu_flush_write_buffer(iommu);
3202
3203 ret = dmar_set_interrupt(iommu);
3204 if (ret)
3205 goto free_iommu;
3206
3207 if (!translation_pre_enabled(iommu))
3208 iommu_enable_translation(iommu);
3209
3210 iommu_disable_protect_mem_regions(iommu);
3211 }
3212
3213 return 0;
3214
3215 free_iommu:
3216 for_each_active_iommu(iommu, drhd) {
3217 disable_dmar_iommu(iommu);
3218 free_dmar_iommu(iommu);
3219 }
3220 kfree(deferred_flush);
3221 free_g_iommus:
3222 kfree(g_iommus);
3223 error:
3224 return ret;
3225 }
3226
3227 /* This takes a number of _MM_ pages, not VTD pages */
3228 static struct iova *intel_alloc_iova(struct device *dev,
3229 struct dmar_domain *domain,
3230 unsigned long nrpages, uint64_t dma_mask)
3231 {
3232 struct iova *iova = NULL;
3233
3234 /* Restrict dma_mask to the width that the iommu can handle */
3235 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3236
3237 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3238 /*
3239 * First try to allocate an io virtual address in
3240 * DMA_BIT_MASK(32) and if that fails then try allocating
3241 * from higher range
3242 */
3243 iova = alloc_iova(&domain->iovad, nrpages,
3244 IOVA_PFN(DMA_BIT_MASK(32)), 1);
3245 if (iova)
3246 return iova;
3247 }
3248 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3249 if (unlikely(!iova)) {
3250 pr_err("Allocating %ld-page iova for %s failed",
3251 nrpages, dev_name(dev));
3252 return NULL;
3253 }
3254
3255 return iova;
3256 }
3257
3258 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3259 {
3260 struct dmar_domain *domain;
3261 int ret;
3262
3263 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3264 if (!domain) {
3265 pr_err("Allocating domain for %s failed\n",
3266 dev_name(dev));
3267 return NULL;
3268 }
3269
3270 /* make sure context mapping is ok */
3271 if (unlikely(!domain_context_mapped(dev))) {
3272 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
3273 if (ret) {
3274 pr_err("Domain context map for %s failed\n",
3275 dev_name(dev));
3276 return NULL;
3277 }
3278 }
3279
3280 return domain;
3281 }
3282
3283 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3284 {
3285 struct device_domain_info *info;
3286
3287 /* No lock here, assumes no domain exit in normal case */
3288 info = dev->archdata.iommu;
3289 if (likely(info))
3290 return info->domain;
3291
3292 return __get_valid_domain_for_dev(dev);
3293 }
3294
3295 /* Check if the dev needs to go through non-identity map and unmap process.*/
3296 static int iommu_no_mapping(struct device *dev)
3297 {
3298 int found;
3299
3300 if (iommu_dummy(dev))
3301 return 1;
3302
3303 if (!iommu_identity_mapping)
3304 return 0;
3305
3306 found = identity_mapping(dev);
3307 if (found) {
3308 if (iommu_should_identity_map(dev, 0))
3309 return 1;
3310 else {
3311 /*
3312 * 32 bit DMA is removed from si_domain and fall back
3313 * to non-identity mapping.
3314 */
3315 domain_remove_one_dev_info(si_domain, dev);
3316 pr_info("32bit %s uses non-identity mapping\n",
3317 dev_name(dev));
3318 return 0;
3319 }
3320 } else {
3321 /*
3322 * In case of a detached 64 bit DMA device from vm, the device
3323 * is put into si_domain for identity mapping.
3324 */
3325 if (iommu_should_identity_map(dev, 0)) {
3326 int ret;
3327 ret = domain_add_dev_info(si_domain, dev,
3328 hw_pass_through ?
3329 CONTEXT_TT_PASS_THROUGH :
3330 CONTEXT_TT_MULTI_LEVEL);
3331 if (!ret) {
3332 pr_info("64bit %s uses identity mapping\n",
3333 dev_name(dev));
3334 return 1;
3335 }
3336 }
3337 }
3338
3339 return 0;
3340 }
3341
3342 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3343 size_t size, int dir, u64 dma_mask)
3344 {
3345 struct dmar_domain *domain;
3346 phys_addr_t start_paddr;
3347 struct iova *iova;
3348 int prot = 0;
3349 int ret;
3350 struct intel_iommu *iommu;
3351 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3352
3353 BUG_ON(dir == DMA_NONE);
3354
3355 if (iommu_no_mapping(dev))
3356 return paddr;
3357
3358 domain = get_valid_domain_for_dev(dev);
3359 if (!domain)
3360 return 0;
3361
3362 iommu = domain_get_iommu(domain);
3363 size = aligned_nrpages(paddr, size);
3364
3365 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3366 if (!iova)
3367 goto error;
3368
3369 /*
3370 * Check if DMAR supports zero-length reads on write only
3371 * mappings..
3372 */
3373 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3374 !cap_zlr(iommu->cap))
3375 prot |= DMA_PTE_READ;
3376 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3377 prot |= DMA_PTE_WRITE;
3378 /*
3379 * paddr - (paddr + size) might be partial page, we should map the whole
3380 * page. Note: if two part of one page are separately mapped, we
3381 * might have two guest_addr mapping to the same host paddr, but this
3382 * is not a big problem
3383 */
3384 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
3385 mm_to_dma_pfn(paddr_pfn), size, prot);
3386 if (ret)
3387 goto error;
3388
3389 /* it's a non-present to present mapping. Only flush if caching mode */
3390 if (cap_caching_mode(iommu->cap))
3391 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
3392 else
3393 iommu_flush_write_buffer(iommu);
3394
3395 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3396 start_paddr += paddr & ~PAGE_MASK;
3397 return start_paddr;
3398
3399 error:
3400 if (iova)
3401 __free_iova(&domain->iovad, iova);
3402 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3403 dev_name(dev), size, (unsigned long long)paddr, dir);
3404 return 0;
3405 }
3406
3407 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3408 unsigned long offset, size_t size,
3409 enum dma_data_direction dir,
3410 struct dma_attrs *attrs)
3411 {
3412 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3413 dir, *dev->dma_mask);
3414 }
3415
3416 static void flush_unmaps(void)
3417 {
3418 int i, j;
3419
3420 timer_on = 0;
3421
3422 /* just flush them all */
3423 for (i = 0; i < g_num_of_iommus; i++) {
3424 struct intel_iommu *iommu = g_iommus[i];
3425 if (!iommu)
3426 continue;
3427
3428 if (!deferred_flush[i].next)
3429 continue;
3430
3431 /* In caching mode, global flushes turn emulation expensive */
3432 if (!cap_caching_mode(iommu->cap))
3433 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3434 DMA_TLB_GLOBAL_FLUSH);
3435 for (j = 0; j < deferred_flush[i].next; j++) {
3436 unsigned long mask;
3437 struct iova *iova = deferred_flush[i].iova[j];
3438 struct dmar_domain *domain = deferred_flush[i].domain[j];
3439
3440 /* On real hardware multiple invalidations are expensive */
3441 if (cap_caching_mode(iommu->cap))
3442 iommu_flush_iotlb_psi(iommu, domain->id,
3443 iova->pfn_lo, iova_size(iova),
3444 !deferred_flush[i].freelist[j], 0);
3445 else {
3446 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
3447 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3448 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3449 }
3450 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
3451 if (deferred_flush[i].freelist[j])
3452 dma_free_pagelist(deferred_flush[i].freelist[j]);
3453 }
3454 deferred_flush[i].next = 0;
3455 }
3456
3457 list_size = 0;
3458 }
3459
3460 static void flush_unmaps_timeout(unsigned long data)
3461 {
3462 unsigned long flags;
3463
3464 spin_lock_irqsave(&async_umap_flush_lock, flags);
3465 flush_unmaps();
3466 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3467 }
3468
3469 static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
3470 {
3471 unsigned long flags;
3472 int next, iommu_id;
3473 struct intel_iommu *iommu;
3474
3475 spin_lock_irqsave(&async_umap_flush_lock, flags);
3476 if (list_size == HIGH_WATER_MARK)
3477 flush_unmaps();
3478
3479 iommu = domain_get_iommu(dom);
3480 iommu_id = iommu->seq_id;
3481
3482 next = deferred_flush[iommu_id].next;
3483 deferred_flush[iommu_id].domain[next] = dom;
3484 deferred_flush[iommu_id].iova[next] = iova;
3485 deferred_flush[iommu_id].freelist[next] = freelist;
3486 deferred_flush[iommu_id].next++;
3487
3488 if (!timer_on) {
3489 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3490 timer_on = 1;
3491 }
3492 list_size++;
3493 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3494 }
3495
3496 static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
3497 {
3498 struct dmar_domain *domain;
3499 unsigned long start_pfn, last_pfn;
3500 struct iova *iova;
3501 struct intel_iommu *iommu;
3502 struct page *freelist;
3503
3504 if (iommu_no_mapping(dev))
3505 return;
3506
3507 domain = find_domain(dev);
3508 BUG_ON(!domain);
3509
3510 iommu = domain_get_iommu(domain);
3511
3512 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
3513 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3514 (unsigned long long)dev_addr))
3515 return;
3516
3517 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3518 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3519
3520 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3521 dev_name(dev), start_pfn, last_pfn);
3522
3523 freelist = domain_unmap(domain, start_pfn, last_pfn);
3524
3525 if (intel_iommu_strict) {
3526 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3527 last_pfn - start_pfn + 1, !freelist, 0);
3528 /* free iova */
3529 __free_iova(&domain->iovad, iova);
3530 dma_free_pagelist(freelist);
3531 } else {
3532 add_unmap(domain, iova, freelist);
3533 /*
3534 * queue up the release of the unmap to save the 1/6th of the
3535 * cpu used up by the iotlb flush operation...
3536 */
3537 }
3538 }
3539
3540 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3541 size_t size, enum dma_data_direction dir,
3542 struct dma_attrs *attrs)
3543 {
3544 intel_unmap(dev, dev_addr);
3545 }
3546
3547 static void *intel_alloc_coherent(struct device *dev, size_t size,
3548 dma_addr_t *dma_handle, gfp_t flags,
3549 struct dma_attrs *attrs)
3550 {
3551 struct page *page = NULL;
3552 int order;
3553
3554 size = PAGE_ALIGN(size);
3555 order = get_order(size);
3556
3557 if (!iommu_no_mapping(dev))
3558 flags &= ~(GFP_DMA | GFP_DMA32);
3559 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3560 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3561 flags |= GFP_DMA;
3562 else
3563 flags |= GFP_DMA32;
3564 }
3565
3566 if (flags & __GFP_WAIT) {
3567 unsigned int count = size >> PAGE_SHIFT;
3568
3569 page = dma_alloc_from_contiguous(dev, count, order);
3570 if (page && iommu_no_mapping(dev) &&
3571 page_to_phys(page) + size > dev->coherent_dma_mask) {
3572 dma_release_from_contiguous(dev, page, count);
3573 page = NULL;
3574 }
3575 }
3576
3577 if (!page)
3578 page = alloc_pages(flags, order);
3579 if (!page)
3580 return NULL;
3581 memset(page_address(page), 0, size);
3582
3583 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3584 DMA_BIDIRECTIONAL,
3585 dev->coherent_dma_mask);
3586 if (*dma_handle)
3587 return page_address(page);
3588 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3589 __free_pages(page, order);
3590
3591 return NULL;
3592 }
3593
3594 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3595 dma_addr_t dma_handle, struct dma_attrs *attrs)
3596 {
3597 int order;
3598 struct page *page = virt_to_page(vaddr);
3599
3600 size = PAGE_ALIGN(size);
3601 order = get_order(size);
3602
3603 intel_unmap(dev, dma_handle);
3604 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3605 __free_pages(page, order);
3606 }
3607
3608 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3609 int nelems, enum dma_data_direction dir,
3610 struct dma_attrs *attrs)
3611 {
3612 intel_unmap(dev, sglist[0].dma_address);
3613 }
3614
3615 static int intel_nontranslate_map_sg(struct device *hddev,
3616 struct scatterlist *sglist, int nelems, int dir)
3617 {
3618 int i;
3619 struct scatterlist *sg;
3620
3621 for_each_sg(sglist, sg, nelems, i) {
3622 BUG_ON(!sg_page(sg));
3623 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3624 sg->dma_length = sg->length;
3625 }
3626 return nelems;
3627 }
3628
3629 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3630 enum dma_data_direction dir, struct dma_attrs *attrs)
3631 {
3632 int i;
3633 struct dmar_domain *domain;
3634 size_t size = 0;
3635 int prot = 0;
3636 struct iova *iova = NULL;
3637 int ret;
3638 struct scatterlist *sg;
3639 unsigned long start_vpfn;
3640 struct intel_iommu *iommu;
3641
3642 BUG_ON(dir == DMA_NONE);
3643 if (iommu_no_mapping(dev))
3644 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3645
3646 domain = get_valid_domain_for_dev(dev);
3647 if (!domain)
3648 return 0;
3649
3650 iommu = domain_get_iommu(domain);
3651
3652 for_each_sg(sglist, sg, nelems, i)
3653 size += aligned_nrpages(sg->offset, sg->length);
3654
3655 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3656 *dev->dma_mask);
3657 if (!iova) {
3658 sglist->dma_length = 0;
3659 return 0;
3660 }
3661
3662 /*
3663 * Check if DMAR supports zero-length reads on write only
3664 * mappings..
3665 */
3666 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3667 !cap_zlr(iommu->cap))
3668 prot |= DMA_PTE_READ;
3669 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3670 prot |= DMA_PTE_WRITE;
3671
3672 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3673
3674 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3675 if (unlikely(ret)) {
3676 dma_pte_free_pagetable(domain, start_vpfn,
3677 start_vpfn + size - 1);
3678 __free_iova(&domain->iovad, iova);
3679 return 0;
3680 }
3681
3682 /* it's a non-present to present mapping. Only flush if caching mode */
3683 if (cap_caching_mode(iommu->cap))
3684 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
3685 else
3686 iommu_flush_write_buffer(iommu);
3687
3688 return nelems;
3689 }
3690
3691 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3692 {
3693 return !dma_addr;
3694 }
3695
3696 struct dma_map_ops intel_dma_ops = {
3697 .alloc = intel_alloc_coherent,
3698 .free = intel_free_coherent,
3699 .map_sg = intel_map_sg,
3700 .unmap_sg = intel_unmap_sg,
3701 .map_page = intel_map_page,
3702 .unmap_page = intel_unmap_page,
3703 .mapping_error = intel_mapping_error,
3704 };
3705
3706 static inline int iommu_domain_cache_init(void)
3707 {
3708 int ret = 0;
3709
3710 iommu_domain_cache = kmem_cache_create("iommu_domain",
3711 sizeof(struct dmar_domain),
3712 0,
3713 SLAB_HWCACHE_ALIGN,
3714
3715 NULL);
3716 if (!iommu_domain_cache) {
3717 pr_err("Couldn't create iommu_domain cache\n");
3718 ret = -ENOMEM;
3719 }
3720
3721 return ret;
3722 }
3723
3724 static inline int iommu_devinfo_cache_init(void)
3725 {
3726 int ret = 0;
3727
3728 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3729 sizeof(struct device_domain_info),
3730 0,
3731 SLAB_HWCACHE_ALIGN,
3732 NULL);
3733 if (!iommu_devinfo_cache) {
3734 pr_err("Couldn't create devinfo cache\n");
3735 ret = -ENOMEM;
3736 }
3737
3738 return ret;
3739 }
3740
3741 static int __init iommu_init_mempool(void)
3742 {
3743 int ret;
3744 ret = iommu_iova_cache_init();
3745 if (ret)
3746 return ret;
3747
3748 ret = iommu_domain_cache_init();
3749 if (ret)
3750 goto domain_error;
3751
3752 ret = iommu_devinfo_cache_init();
3753 if (!ret)
3754 return ret;
3755
3756 kmem_cache_destroy(iommu_domain_cache);
3757 domain_error:
3758 iommu_iova_cache_destroy();
3759
3760 return -ENOMEM;
3761 }
3762
3763 static void __init iommu_exit_mempool(void)
3764 {
3765 kmem_cache_destroy(iommu_devinfo_cache);
3766 kmem_cache_destroy(iommu_domain_cache);
3767 iommu_iova_cache_destroy();
3768 }
3769
3770 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3771 {
3772 struct dmar_drhd_unit *drhd;
3773 u32 vtbar;
3774 int rc;
3775
3776 /* We know that this device on this chipset has its own IOMMU.
3777 * If we find it under a different IOMMU, then the BIOS is lying
3778 * to us. Hope that the IOMMU for this device is actually
3779 * disabled, and it needs no translation...
3780 */
3781 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3782 if (rc) {
3783 /* "can't" happen */
3784 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3785 return;
3786 }
3787 vtbar &= 0xffff0000;
3788
3789 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3790 drhd = dmar_find_matched_drhd_unit(pdev);
3791 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3792 TAINT_FIRMWARE_WORKAROUND,
3793 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3794 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3795 }
3796 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3797
3798 static void __init init_no_remapping_devices(void)
3799 {
3800 struct dmar_drhd_unit *drhd;
3801 struct device *dev;
3802 int i;
3803
3804 for_each_drhd_unit(drhd) {
3805 if (!drhd->include_all) {
3806 for_each_active_dev_scope(drhd->devices,
3807 drhd->devices_cnt, i, dev)
3808 break;
3809 /* ignore DMAR unit if no devices exist */
3810 if (i == drhd->devices_cnt)
3811 drhd->ignored = 1;
3812 }
3813 }
3814
3815 for_each_active_drhd_unit(drhd) {
3816 if (drhd->include_all)
3817 continue;
3818
3819 for_each_active_dev_scope(drhd->devices,
3820 drhd->devices_cnt, i, dev)
3821 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3822 break;
3823 if (i < drhd->devices_cnt)
3824 continue;
3825
3826 /* This IOMMU has *only* gfx devices. Either bypass it or
3827 set the gfx_mapped flag, as appropriate */
3828 if (dmar_map_gfx) {
3829 intel_iommu_gfx_mapped = 1;
3830 } else {
3831 drhd->ignored = 1;
3832 for_each_active_dev_scope(drhd->devices,
3833 drhd->devices_cnt, i, dev)
3834 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3835 }
3836 }
3837 }
3838
3839 #ifdef CONFIG_SUSPEND
3840 static int init_iommu_hw(void)
3841 {
3842 struct dmar_drhd_unit *drhd;
3843 struct intel_iommu *iommu = NULL;
3844
3845 for_each_active_iommu(iommu, drhd)
3846 if (iommu->qi)
3847 dmar_reenable_qi(iommu);
3848
3849 for_each_iommu(iommu, drhd) {
3850 if (drhd->ignored) {
3851 /*
3852 * we always have to disable PMRs or DMA may fail on
3853 * this device
3854 */
3855 if (force_on)
3856 iommu_disable_protect_mem_regions(iommu);
3857 continue;
3858 }
3859
3860 iommu_flush_write_buffer(iommu);
3861
3862 iommu_set_root_entry(iommu);
3863
3864 iommu->flush.flush_context(iommu, 0, 0, 0,
3865 DMA_CCMD_GLOBAL_INVL);
3866 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3867 iommu_enable_translation(iommu);
3868 iommu_disable_protect_mem_regions(iommu);
3869 }
3870
3871 return 0;
3872 }
3873
3874 static void iommu_flush_all(void)
3875 {
3876 struct dmar_drhd_unit *drhd;
3877 struct intel_iommu *iommu;
3878
3879 for_each_active_iommu(iommu, drhd) {
3880 iommu->flush.flush_context(iommu, 0, 0, 0,
3881 DMA_CCMD_GLOBAL_INVL);
3882 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3883 DMA_TLB_GLOBAL_FLUSH);
3884 }
3885 }
3886
3887 static int iommu_suspend(void)
3888 {
3889 struct dmar_drhd_unit *drhd;
3890 struct intel_iommu *iommu = NULL;
3891 unsigned long flag;
3892
3893 for_each_active_iommu(iommu, drhd) {
3894 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3895 GFP_ATOMIC);
3896 if (!iommu->iommu_state)
3897 goto nomem;
3898 }
3899
3900 iommu_flush_all();
3901
3902 for_each_active_iommu(iommu, drhd) {
3903 iommu_disable_translation(iommu);
3904
3905 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3906
3907 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3908 readl(iommu->reg + DMAR_FECTL_REG);
3909 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3910 readl(iommu->reg + DMAR_FEDATA_REG);
3911 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3912 readl(iommu->reg + DMAR_FEADDR_REG);
3913 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3914 readl(iommu->reg + DMAR_FEUADDR_REG);
3915
3916 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3917 }
3918 return 0;
3919
3920 nomem:
3921 for_each_active_iommu(iommu, drhd)
3922 kfree(iommu->iommu_state);
3923
3924 return -ENOMEM;
3925 }
3926
3927 static void iommu_resume(void)
3928 {
3929 struct dmar_drhd_unit *drhd;
3930 struct intel_iommu *iommu = NULL;
3931 unsigned long flag;
3932
3933 if (init_iommu_hw()) {
3934 if (force_on)
3935 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3936 else
3937 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3938 return;
3939 }
3940
3941 for_each_active_iommu(iommu, drhd) {
3942
3943 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3944
3945 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3946 iommu->reg + DMAR_FECTL_REG);
3947 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3948 iommu->reg + DMAR_FEDATA_REG);
3949 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3950 iommu->reg + DMAR_FEADDR_REG);
3951 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3952 iommu->reg + DMAR_FEUADDR_REG);
3953
3954 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3955 }
3956
3957 for_each_active_iommu(iommu, drhd)
3958 kfree(iommu->iommu_state);
3959 }
3960
3961 static struct syscore_ops iommu_syscore_ops = {
3962 .resume = iommu_resume,
3963 .suspend = iommu_suspend,
3964 };
3965
3966 static void __init init_iommu_pm_ops(void)
3967 {
3968 register_syscore_ops(&iommu_syscore_ops);
3969 }
3970
3971 #else
3972 static inline void init_iommu_pm_ops(void) {}
3973 #endif /* CONFIG_PM */
3974
3975
3976 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
3977 {
3978 struct acpi_dmar_reserved_memory *rmrr;
3979 struct dmar_rmrr_unit *rmrru;
3980
3981 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3982 if (!rmrru)
3983 return -ENOMEM;
3984
3985 rmrru->hdr = header;
3986 rmrr = (struct acpi_dmar_reserved_memory *)header;
3987 rmrru->base_address = rmrr->base_address;
3988 rmrru->end_address = rmrr->end_address;
3989 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3990 ((void *)rmrr) + rmrr->header.length,
3991 &rmrru->devices_cnt);
3992 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3993 kfree(rmrru);
3994 return -ENOMEM;
3995 }
3996
3997 list_add(&rmrru->list, &dmar_rmrr_units);
3998
3999 return 0;
4000 }
4001
4002 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4003 {
4004 struct dmar_atsr_unit *atsru;
4005 struct acpi_dmar_atsr *tmp;
4006
4007 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4008 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4009 if (atsr->segment != tmp->segment)
4010 continue;
4011 if (atsr->header.length != tmp->header.length)
4012 continue;
4013 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4014 return atsru;
4015 }
4016
4017 return NULL;
4018 }
4019
4020 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4021 {
4022 struct acpi_dmar_atsr *atsr;
4023 struct dmar_atsr_unit *atsru;
4024
4025 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4026 return 0;
4027
4028 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4029 atsru = dmar_find_atsr(atsr);
4030 if (atsru)
4031 return 0;
4032
4033 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4034 if (!atsru)
4035 return -ENOMEM;
4036
4037 /*
4038 * If memory is allocated from slab by ACPI _DSM method, we need to
4039 * copy the memory content because the memory buffer will be freed
4040 * on return.
4041 */
4042 atsru->hdr = (void *)(atsru + 1);
4043 memcpy(atsru->hdr, hdr, hdr->length);
4044 atsru->include_all = atsr->flags & 0x1;
4045 if (!atsru->include_all) {
4046 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4047 (void *)atsr + atsr->header.length,
4048 &atsru->devices_cnt);
4049 if (atsru->devices_cnt && atsru->devices == NULL) {
4050 kfree(atsru);
4051 return -ENOMEM;
4052 }
4053 }
4054
4055 list_add_rcu(&atsru->list, &dmar_atsr_units);
4056
4057 return 0;
4058 }
4059
4060 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4061 {
4062 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4063 kfree(atsru);
4064 }
4065
4066 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4067 {
4068 struct acpi_dmar_atsr *atsr;
4069 struct dmar_atsr_unit *atsru;
4070
4071 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4072 atsru = dmar_find_atsr(atsr);
4073 if (atsru) {
4074 list_del_rcu(&atsru->list);
4075 synchronize_rcu();
4076 intel_iommu_free_atsr(atsru);
4077 }
4078
4079 return 0;
4080 }
4081
4082 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4083 {
4084 int i;
4085 struct device *dev;
4086 struct acpi_dmar_atsr *atsr;
4087 struct dmar_atsr_unit *atsru;
4088
4089 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4090 atsru = dmar_find_atsr(atsr);
4091 if (!atsru)
4092 return 0;
4093
4094 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
4095 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4096 i, dev)
4097 return -EBUSY;
4098
4099 return 0;
4100 }
4101
4102 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4103 {
4104 int sp, ret = 0;
4105 struct intel_iommu *iommu = dmaru->iommu;
4106
4107 if (g_iommus[iommu->seq_id])
4108 return 0;
4109
4110 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4111 pr_warn("%s: Doesn't support hardware pass through.\n",
4112 iommu->name);
4113 return -ENXIO;
4114 }
4115 if (!ecap_sc_support(iommu->ecap) &&
4116 domain_update_iommu_snooping(iommu)) {
4117 pr_warn("%s: Doesn't support snooping.\n",
4118 iommu->name);
4119 return -ENXIO;
4120 }
4121 sp = domain_update_iommu_superpage(iommu) - 1;
4122 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4123 pr_warn("%s: Doesn't support large page.\n",
4124 iommu->name);
4125 return -ENXIO;
4126 }
4127
4128 /*
4129 * Disable translation if already enabled prior to OS handover.
4130 */
4131 if (iommu->gcmd & DMA_GCMD_TE)
4132 iommu_disable_translation(iommu);
4133
4134 g_iommus[iommu->seq_id] = iommu;
4135 ret = iommu_init_domains(iommu);
4136 if (ret == 0)
4137 ret = iommu_alloc_root_entry(iommu);
4138 if (ret)
4139 goto out;
4140
4141 if (dmaru->ignored) {
4142 /*
4143 * we always have to disable PMRs or DMA may fail on this device
4144 */
4145 if (force_on)
4146 iommu_disable_protect_mem_regions(iommu);
4147 return 0;
4148 }
4149
4150 intel_iommu_init_qi(iommu);
4151 iommu_flush_write_buffer(iommu);
4152 ret = dmar_set_interrupt(iommu);
4153 if (ret)
4154 goto disable_iommu;
4155
4156 iommu_set_root_entry(iommu);
4157 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4158 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4159 iommu_enable_translation(iommu);
4160
4161 if (si_domain) {
4162 ret = iommu_attach_domain(si_domain, iommu);
4163 if (ret < 0 || si_domain->id != ret)
4164 goto disable_iommu;
4165 domain_attach_iommu(si_domain, iommu);
4166 }
4167
4168 iommu_disable_protect_mem_regions(iommu);
4169 return 0;
4170
4171 disable_iommu:
4172 disable_dmar_iommu(iommu);
4173 out:
4174 free_dmar_iommu(iommu);
4175 return ret;
4176 }
4177
4178 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4179 {
4180 int ret = 0;
4181 struct intel_iommu *iommu = dmaru->iommu;
4182
4183 if (!intel_iommu_enabled)
4184 return 0;
4185 if (iommu == NULL)
4186 return -EINVAL;
4187
4188 if (insert) {
4189 ret = intel_iommu_add(dmaru);
4190 } else {
4191 disable_dmar_iommu(iommu);
4192 free_dmar_iommu(iommu);
4193 }
4194
4195 return ret;
4196 }
4197
4198 static void intel_iommu_free_dmars(void)
4199 {
4200 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4201 struct dmar_atsr_unit *atsru, *atsr_n;
4202
4203 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4204 list_del(&rmrru->list);
4205 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4206 kfree(rmrru);
4207 }
4208
4209 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4210 list_del(&atsru->list);
4211 intel_iommu_free_atsr(atsru);
4212 }
4213 }
4214
4215 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4216 {
4217 int i, ret = 1;
4218 struct pci_bus *bus;
4219 struct pci_dev *bridge = NULL;
4220 struct device *tmp;
4221 struct acpi_dmar_atsr *atsr;
4222 struct dmar_atsr_unit *atsru;
4223
4224 dev = pci_physfn(dev);
4225 for (bus = dev->bus; bus; bus = bus->parent) {
4226 bridge = bus->self;
4227 if (!bridge || !pci_is_pcie(bridge) ||
4228 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4229 return 0;
4230 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4231 break;
4232 }
4233 if (!bridge)
4234 return 0;
4235
4236 rcu_read_lock();
4237 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4238 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4239 if (atsr->segment != pci_domain_nr(dev->bus))
4240 continue;
4241
4242 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4243 if (tmp == &bridge->dev)
4244 goto out;
4245
4246 if (atsru->include_all)
4247 goto out;
4248 }
4249 ret = 0;
4250 out:
4251 rcu_read_unlock();
4252
4253 return ret;
4254 }
4255
4256 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4257 {
4258 int ret = 0;
4259 struct dmar_rmrr_unit *rmrru;
4260 struct dmar_atsr_unit *atsru;
4261 struct acpi_dmar_atsr *atsr;
4262 struct acpi_dmar_reserved_memory *rmrr;
4263
4264 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4265 return 0;
4266
4267 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4268 rmrr = container_of(rmrru->hdr,
4269 struct acpi_dmar_reserved_memory, header);
4270 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4271 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4272 ((void *)rmrr) + rmrr->header.length,
4273 rmrr->segment, rmrru->devices,
4274 rmrru->devices_cnt);
4275 if(ret < 0)
4276 return ret;
4277 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4278 dmar_remove_dev_scope(info, rmrr->segment,
4279 rmrru->devices, rmrru->devices_cnt);
4280 }
4281 }
4282
4283 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4284 if (atsru->include_all)
4285 continue;
4286
4287 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4288 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4289 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4290 (void *)atsr + atsr->header.length,
4291 atsr->segment, atsru->devices,
4292 atsru->devices_cnt);
4293 if (ret > 0)
4294 break;
4295 else if(ret < 0)
4296 return ret;
4297 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4298 if (dmar_remove_dev_scope(info, atsr->segment,
4299 atsru->devices, atsru->devices_cnt))
4300 break;
4301 }
4302 }
4303
4304 return 0;
4305 }
4306
4307 /*
4308 * Here we only respond to action of unbound device from driver.
4309 *
4310 * Added device is not attached to its DMAR domain here yet. That will happen
4311 * when mapping the device to iova.
4312 */
4313 static int device_notifier(struct notifier_block *nb,
4314 unsigned long action, void *data)
4315 {
4316 struct device *dev = data;
4317 struct dmar_domain *domain;
4318
4319 if (iommu_dummy(dev))
4320 return 0;
4321
4322 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4323 return 0;
4324
4325 domain = find_domain(dev);
4326 if (!domain)
4327 return 0;
4328
4329 down_read(&dmar_global_lock);
4330 domain_remove_one_dev_info(domain, dev);
4331 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4332 domain_exit(domain);
4333 up_read(&dmar_global_lock);
4334
4335 return 0;
4336 }
4337
4338 static struct notifier_block device_nb = {
4339 .notifier_call = device_notifier,
4340 };
4341
4342 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4343 unsigned long val, void *v)
4344 {
4345 struct memory_notify *mhp = v;
4346 unsigned long long start, end;
4347 unsigned long start_vpfn, last_vpfn;
4348
4349 switch (val) {
4350 case MEM_GOING_ONLINE:
4351 start = mhp->start_pfn << PAGE_SHIFT;
4352 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4353 if (iommu_domain_identity_map(si_domain, start, end)) {
4354 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4355 start, end);
4356 return NOTIFY_BAD;
4357 }
4358 break;
4359
4360 case MEM_OFFLINE:
4361 case MEM_CANCEL_ONLINE:
4362 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4363 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4364 while (start_vpfn <= last_vpfn) {
4365 struct iova *iova;
4366 struct dmar_drhd_unit *drhd;
4367 struct intel_iommu *iommu;
4368 struct page *freelist;
4369
4370 iova = find_iova(&si_domain->iovad, start_vpfn);
4371 if (iova == NULL) {
4372 pr_debug("Failed get IOVA for PFN %lx\n",
4373 start_vpfn);
4374 break;
4375 }
4376
4377 iova = split_and_remove_iova(&si_domain->iovad, iova,
4378 start_vpfn, last_vpfn);
4379 if (iova == NULL) {
4380 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4381 start_vpfn, last_vpfn);
4382 return NOTIFY_BAD;
4383 }
4384
4385 freelist = domain_unmap(si_domain, iova->pfn_lo,
4386 iova->pfn_hi);
4387
4388 rcu_read_lock();
4389 for_each_active_iommu(iommu, drhd)
4390 iommu_flush_iotlb_psi(iommu, si_domain->id,
4391 iova->pfn_lo, iova_size(iova),
4392 !freelist, 0);
4393 rcu_read_unlock();
4394 dma_free_pagelist(freelist);
4395
4396 start_vpfn = iova->pfn_hi + 1;
4397 free_iova_mem(iova);
4398 }
4399 break;
4400 }
4401
4402 return NOTIFY_OK;
4403 }
4404
4405 static struct notifier_block intel_iommu_memory_nb = {
4406 .notifier_call = intel_iommu_memory_notifier,
4407 .priority = 0
4408 };
4409
4410
4411 static ssize_t intel_iommu_show_version(struct device *dev,
4412 struct device_attribute *attr,
4413 char *buf)
4414 {
4415 struct intel_iommu *iommu = dev_get_drvdata(dev);
4416 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4417 return sprintf(buf, "%d:%d\n",
4418 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4419 }
4420 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4421
4422 static ssize_t intel_iommu_show_address(struct device *dev,
4423 struct device_attribute *attr,
4424 char *buf)
4425 {
4426 struct intel_iommu *iommu = dev_get_drvdata(dev);
4427 return sprintf(buf, "%llx\n", iommu->reg_phys);
4428 }
4429 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4430
4431 static ssize_t intel_iommu_show_cap(struct device *dev,
4432 struct device_attribute *attr,
4433 char *buf)
4434 {
4435 struct intel_iommu *iommu = dev_get_drvdata(dev);
4436 return sprintf(buf, "%llx\n", iommu->cap);
4437 }
4438 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4439
4440 static ssize_t intel_iommu_show_ecap(struct device *dev,
4441 struct device_attribute *attr,
4442 char *buf)
4443 {
4444 struct intel_iommu *iommu = dev_get_drvdata(dev);
4445 return sprintf(buf, "%llx\n", iommu->ecap);
4446 }
4447 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4448
4449 static struct attribute *intel_iommu_attrs[] = {
4450 &dev_attr_version.attr,
4451 &dev_attr_address.attr,
4452 &dev_attr_cap.attr,
4453 &dev_attr_ecap.attr,
4454 NULL,
4455 };
4456
4457 static struct attribute_group intel_iommu_group = {
4458 .name = "intel-iommu",
4459 .attrs = intel_iommu_attrs,
4460 };
4461
4462 const struct attribute_group *intel_iommu_groups[] = {
4463 &intel_iommu_group,
4464 NULL,
4465 };
4466
4467 int __init intel_iommu_init(void)
4468 {
4469 int ret = -ENODEV;
4470 struct dmar_drhd_unit *drhd;
4471 struct intel_iommu *iommu;
4472
4473 /* VT-d is required for a TXT/tboot launch, so enforce that */
4474 force_on = tboot_force_iommu();
4475
4476 if (iommu_init_mempool()) {
4477 if (force_on)
4478 panic("tboot: Failed to initialize iommu memory\n");
4479 return -ENOMEM;
4480 }
4481
4482 down_write(&dmar_global_lock);
4483 if (dmar_table_init()) {
4484 if (force_on)
4485 panic("tboot: Failed to initialize DMAR table\n");
4486 goto out_free_dmar;
4487 }
4488
4489 if (dmar_dev_scope_init() < 0) {
4490 if (force_on)
4491 panic("tboot: Failed to initialize DMAR device scope\n");
4492 goto out_free_dmar;
4493 }
4494
4495 if (no_iommu || dmar_disabled)
4496 goto out_free_dmar;
4497
4498 if (list_empty(&dmar_rmrr_units))
4499 pr_info("No RMRR found\n");
4500
4501 if (list_empty(&dmar_atsr_units))
4502 pr_info("No ATSR found\n");
4503
4504 if (dmar_init_reserved_ranges()) {
4505 if (force_on)
4506 panic("tboot: Failed to reserve iommu ranges\n");
4507 goto out_free_reserved_range;
4508 }
4509
4510 init_no_remapping_devices();
4511
4512 ret = init_dmars();
4513 if (ret) {
4514 if (force_on)
4515 panic("tboot: Failed to initialize DMARs\n");
4516 pr_err("Initialization failed\n");
4517 goto out_free_reserved_range;
4518 }
4519 up_write(&dmar_global_lock);
4520 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4521
4522 init_timer(&unmap_timer);
4523 #ifdef CONFIG_SWIOTLB
4524 swiotlb = 0;
4525 #endif
4526 dma_ops = &intel_dma_ops;
4527
4528 init_iommu_pm_ops();
4529
4530 for_each_active_iommu(iommu, drhd)
4531 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4532 intel_iommu_groups,
4533 iommu->name);
4534
4535 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4536 bus_register_notifier(&pci_bus_type, &device_nb);
4537 if (si_domain && !hw_pass_through)
4538 register_memory_notifier(&intel_iommu_memory_nb);
4539
4540 intel_iommu_enabled = 1;
4541
4542 return 0;
4543
4544 out_free_reserved_range:
4545 put_iova_domain(&reserved_iova_list);
4546 out_free_dmar:
4547 intel_iommu_free_dmars();
4548 up_write(&dmar_global_lock);
4549 iommu_exit_mempool();
4550 return ret;
4551 }
4552
4553 static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4554 {
4555 struct intel_iommu *iommu = opaque;
4556
4557 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4558 return 0;
4559 }
4560
4561 /*
4562 * NB - intel-iommu lacks any sort of reference counting for the users of
4563 * dependent devices. If multiple endpoints have intersecting dependent
4564 * devices, unbinding the driver from any one of them will possibly leave
4565 * the others unable to operate.
4566 */
4567 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
4568 struct device *dev)
4569 {
4570 if (!iommu || !dev || !dev_is_pci(dev))
4571 return;
4572
4573 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
4574 }
4575
4576 static void domain_remove_one_dev_info(struct dmar_domain *domain,
4577 struct device *dev)
4578 {
4579 struct device_domain_info *info, *tmp;
4580 struct intel_iommu *iommu;
4581 unsigned long flags;
4582 bool found = false;
4583 u8 bus, devfn;
4584
4585 iommu = device_to_iommu(dev, &bus, &devfn);
4586 if (!iommu)
4587 return;
4588
4589 spin_lock_irqsave(&device_domain_lock, flags);
4590 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
4591 if (info->iommu == iommu && info->bus == bus &&
4592 info->devfn == devfn) {
4593 unlink_domain_info(info);
4594 spin_unlock_irqrestore(&device_domain_lock, flags);
4595
4596 iommu_disable_dev_iotlb(info);
4597 iommu_detach_dev(iommu, info->bus, info->devfn);
4598 iommu_detach_dependent_devices(iommu, dev);
4599 free_devinfo_mem(info);
4600
4601 spin_lock_irqsave(&device_domain_lock, flags);
4602
4603 if (found)
4604 break;
4605 else
4606 continue;
4607 }
4608
4609 /* if there is no other devices under the same iommu
4610 * owned by this domain, clear this iommu in iommu_bmp
4611 * update iommu count and coherency
4612 */
4613 if (info->iommu == iommu)
4614 found = true;
4615 }
4616
4617 spin_unlock_irqrestore(&device_domain_lock, flags);
4618
4619 if (found == 0) {
4620 domain_detach_iommu(domain, iommu);
4621 if (!domain_type_is_vm_or_si(domain))
4622 iommu_detach_domain(domain, iommu);
4623 }
4624 }
4625
4626 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4627 {
4628 int adjust_width;
4629
4630 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4631 DMA_32BIT_PFN);
4632 domain_reserve_special_ranges(domain);
4633
4634 /* calculate AGAW */
4635 domain->gaw = guest_width;
4636 adjust_width = guestwidth_to_adjustwidth(guest_width);
4637 domain->agaw = width_to_agaw(adjust_width);
4638
4639 domain->iommu_coherency = 0;
4640 domain->iommu_snooping = 0;
4641 domain->iommu_superpage = 0;
4642 domain->max_addr = 0;
4643
4644 /* always allocate the top pgd */
4645 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4646 if (!domain->pgd)
4647 return -ENOMEM;
4648 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4649 return 0;
4650 }
4651
4652 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4653 {
4654 struct dmar_domain *dmar_domain;
4655 struct iommu_domain *domain;
4656
4657 if (type != IOMMU_DOMAIN_UNMANAGED)
4658 return NULL;
4659
4660 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4661 if (!dmar_domain) {
4662 pr_err("Can't allocate dmar_domain\n");
4663 return NULL;
4664 }
4665 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4666 pr_err("Domain initialization failed\n");
4667 domain_exit(dmar_domain);
4668 return NULL;
4669 }
4670 domain_update_iommu_cap(dmar_domain);
4671
4672 domain = &dmar_domain->domain;
4673 domain->geometry.aperture_start = 0;
4674 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4675 domain->geometry.force_aperture = true;
4676
4677 return domain;
4678 }
4679
4680 static void intel_iommu_domain_free(struct iommu_domain *domain)
4681 {
4682 domain_exit(to_dmar_domain(domain));
4683 }
4684
4685 static int intel_iommu_attach_device(struct iommu_domain *domain,
4686 struct device *dev)
4687 {
4688 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4689 struct intel_iommu *iommu;
4690 int addr_width;
4691 u8 bus, devfn;
4692
4693 if (device_is_rmrr_locked(dev)) {
4694 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4695 return -EPERM;
4696 }
4697
4698 /* normally dev is not mapped */
4699 if (unlikely(domain_context_mapped(dev))) {
4700 struct dmar_domain *old_domain;
4701
4702 old_domain = find_domain(dev);
4703 if (old_domain) {
4704 if (domain_type_is_vm_or_si(dmar_domain))
4705 domain_remove_one_dev_info(old_domain, dev);
4706 else
4707 domain_remove_dev_info(old_domain);
4708
4709 if (!domain_type_is_vm_or_si(old_domain) &&
4710 list_empty(&old_domain->devices))
4711 domain_exit(old_domain);
4712 }
4713 }
4714
4715 iommu = device_to_iommu(dev, &bus, &devfn);
4716 if (!iommu)
4717 return -ENODEV;
4718
4719 /* check if this iommu agaw is sufficient for max mapped address */
4720 addr_width = agaw_to_width(iommu->agaw);
4721 if (addr_width > cap_mgaw(iommu->cap))
4722 addr_width = cap_mgaw(iommu->cap);
4723
4724 if (dmar_domain->max_addr > (1LL << addr_width)) {
4725 pr_err("%s: iommu width (%d) is not "
4726 "sufficient for the mapped address (%llx)\n",
4727 __func__, addr_width, dmar_domain->max_addr);
4728 return -EFAULT;
4729 }
4730 dmar_domain->gaw = addr_width;
4731
4732 /*
4733 * Knock out extra levels of page tables if necessary
4734 */
4735 while (iommu->agaw < dmar_domain->agaw) {
4736 struct dma_pte *pte;
4737
4738 pte = dmar_domain->pgd;
4739 if (dma_pte_present(pte)) {
4740 dmar_domain->pgd = (struct dma_pte *)
4741 phys_to_virt(dma_pte_addr(pte));
4742 free_pgtable_page(pte);
4743 }
4744 dmar_domain->agaw--;
4745 }
4746
4747 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
4748 }
4749
4750 static void intel_iommu_detach_device(struct iommu_domain *domain,
4751 struct device *dev)
4752 {
4753 domain_remove_one_dev_info(to_dmar_domain(domain), dev);
4754 }
4755
4756 static int intel_iommu_map(struct iommu_domain *domain,
4757 unsigned long iova, phys_addr_t hpa,
4758 size_t size, int iommu_prot)
4759 {
4760 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4761 u64 max_addr;
4762 int prot = 0;
4763 int ret;
4764
4765 if (iommu_prot & IOMMU_READ)
4766 prot |= DMA_PTE_READ;
4767 if (iommu_prot & IOMMU_WRITE)
4768 prot |= DMA_PTE_WRITE;
4769 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4770 prot |= DMA_PTE_SNP;
4771
4772 max_addr = iova + size;
4773 if (dmar_domain->max_addr < max_addr) {
4774 u64 end;
4775
4776 /* check if minimum agaw is sufficient for mapped address */
4777 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4778 if (end < max_addr) {
4779 pr_err("%s: iommu width (%d) is not "
4780 "sufficient for the mapped address (%llx)\n",
4781 __func__, dmar_domain->gaw, max_addr);
4782 return -EFAULT;
4783 }
4784 dmar_domain->max_addr = max_addr;
4785 }
4786 /* Round up size to next multiple of PAGE_SIZE, if it and
4787 the low bits of hpa would take us onto the next page */
4788 size = aligned_nrpages(hpa, size);
4789 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4790 hpa >> VTD_PAGE_SHIFT, size, prot);
4791 return ret;
4792 }
4793
4794 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4795 unsigned long iova, size_t size)
4796 {
4797 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4798 struct page *freelist = NULL;
4799 struct intel_iommu *iommu;
4800 unsigned long start_pfn, last_pfn;
4801 unsigned int npages;
4802 int iommu_id, num, ndomains, level = 0;
4803
4804 /* Cope with horrid API which requires us to unmap more than the
4805 size argument if it happens to be a large-page mapping. */
4806 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4807 BUG();
4808
4809 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4810 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4811
4812 start_pfn = iova >> VTD_PAGE_SHIFT;
4813 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4814
4815 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4816
4817 npages = last_pfn - start_pfn + 1;
4818
4819 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4820 iommu = g_iommus[iommu_id];
4821
4822 /*
4823 * find bit position of dmar_domain
4824 */
4825 ndomains = cap_ndoms(iommu->cap);
4826 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4827 if (iommu->domains[num] == dmar_domain)
4828 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4829 npages, !freelist, 0);
4830 }
4831
4832 }
4833
4834 dma_free_pagelist(freelist);
4835
4836 if (dmar_domain->max_addr == iova + size)
4837 dmar_domain->max_addr = iova;
4838
4839 return size;
4840 }
4841
4842 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4843 dma_addr_t iova)
4844 {
4845 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4846 struct dma_pte *pte;
4847 int level = 0;
4848 u64 phys = 0;
4849
4850 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
4851 if (pte)
4852 phys = dma_pte_addr(pte);
4853
4854 return phys;
4855 }
4856
4857 static bool intel_iommu_capable(enum iommu_cap cap)
4858 {
4859 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4860 return domain_update_iommu_snooping(NULL) == 1;
4861 if (cap == IOMMU_CAP_INTR_REMAP)
4862 return irq_remapping_enabled == 1;
4863
4864 return false;
4865 }
4866
4867 static int intel_iommu_add_device(struct device *dev)
4868 {
4869 struct intel_iommu *iommu;
4870 struct iommu_group *group;
4871 u8 bus, devfn;
4872
4873 iommu = device_to_iommu(dev, &bus, &devfn);
4874 if (!iommu)
4875 return -ENODEV;
4876
4877 iommu_device_link(iommu->iommu_dev, dev);
4878
4879 group = iommu_group_get_for_dev(dev);
4880
4881 if (IS_ERR(group))
4882 return PTR_ERR(group);
4883
4884 iommu_group_put(group);
4885 return 0;
4886 }
4887
4888 static void intel_iommu_remove_device(struct device *dev)
4889 {
4890 struct intel_iommu *iommu;
4891 u8 bus, devfn;
4892
4893 iommu = device_to_iommu(dev, &bus, &devfn);
4894 if (!iommu)
4895 return;
4896
4897 iommu_group_remove_device(dev);
4898
4899 iommu_device_unlink(iommu->iommu_dev, dev);
4900 }
4901
4902 static const struct iommu_ops intel_iommu_ops = {
4903 .capable = intel_iommu_capable,
4904 .domain_alloc = intel_iommu_domain_alloc,
4905 .domain_free = intel_iommu_domain_free,
4906 .attach_dev = intel_iommu_attach_device,
4907 .detach_dev = intel_iommu_detach_device,
4908 .map = intel_iommu_map,
4909 .unmap = intel_iommu_unmap,
4910 .map_sg = default_iommu_map_sg,
4911 .iova_to_phys = intel_iommu_iova_to_phys,
4912 .add_device = intel_iommu_add_device,
4913 .remove_device = intel_iommu_remove_device,
4914 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
4915 };
4916
4917 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4918 {
4919 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4920 pr_info("Disabling IOMMU for graphics on this chipset\n");
4921 dmar_map_gfx = 0;
4922 }
4923
4924 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4925 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4926 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4927 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4928 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4929 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4930 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4931
4932 static void quirk_iommu_rwbf(struct pci_dev *dev)
4933 {
4934 /*
4935 * Mobile 4 Series Chipset neglects to set RWBF capability,
4936 * but needs it. Same seems to hold for the desktop versions.
4937 */
4938 pr_info("Forcing write-buffer flush capability\n");
4939 rwbf_quirk = 1;
4940 }
4941
4942 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4943 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4944 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4945 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4946 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4947 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4948 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4949
4950 #define GGC 0x52
4951 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
4952 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4953 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
4954 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
4955 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4956 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4957 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4958 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4959
4960 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4961 {
4962 unsigned short ggc;
4963
4964 if (pci_read_config_word(dev, GGC, &ggc))
4965 return;
4966
4967 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4968 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4969 dmar_map_gfx = 0;
4970 } else if (dmar_map_gfx) {
4971 /* we have to ensure the gfx device is idle before we flush */
4972 pr_info("Disabling batched IOTLB flush on Ironlake\n");
4973 intel_iommu_strict = 1;
4974 }
4975 }
4976 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4977 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4978 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4979 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4980
4981 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4982 ISOCH DMAR unit for the Azalia sound device, but not give it any
4983 TLB entries, which causes it to deadlock. Check for that. We do
4984 this in a function called from init_dmars(), instead of in a PCI
4985 quirk, because we don't want to print the obnoxious "BIOS broken"
4986 message if VT-d is actually disabled.
4987 */
4988 static void __init check_tylersburg_isoch(void)
4989 {
4990 struct pci_dev *pdev;
4991 uint32_t vtisochctrl;
4992
4993 /* If there's no Azalia in the system anyway, forget it. */
4994 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4995 if (!pdev)
4996 return;
4997 pci_dev_put(pdev);
4998
4999 /* System Management Registers. Might be hidden, in which case
5000 we can't do the sanity check. But that's OK, because the
5001 known-broken BIOSes _don't_ actually hide it, so far. */
5002 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5003 if (!pdev)
5004 return;
5005
5006 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5007 pci_dev_put(pdev);
5008 return;
5009 }
5010
5011 pci_dev_put(pdev);
5012
5013 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5014 if (vtisochctrl & 1)
5015 return;
5016
5017 /* Drop all bits other than the number of TLB entries */
5018 vtisochctrl &= 0x1c;
5019
5020 /* If we have the recommended number of TLB entries (16), fine. */
5021 if (vtisochctrl == 0x10)
5022 return;
5023
5024 /* Zero TLB entries? You get to ride the short bus to school. */
5025 if (!vtisochctrl) {
5026 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5027 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5028 dmi_get_system_info(DMI_BIOS_VENDOR),
5029 dmi_get_system_info(DMI_BIOS_VERSION),
5030 dmi_get_system_info(DMI_PRODUCT_VERSION));
5031 iommu_identity_mapping |= IDENTMAP_AZALIA;
5032 return;
5033 }
5034
5035 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5036 vtisochctrl);
5037 }
Linux with added FPC-III support