?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[cris-mirror.git] / drivers / iommu / intel-iommu.c
blobebb5bf3ddbd9424586b4d22068bd7a41656881b4
1 /*
2 * Copyright © 2006-2014 Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
19 */
20
21 #define pr_fmt(fmt) "DMAR: " fmt
22
23 #include <linux/init.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/export.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/memory.h>
36 #include <linux/cpu.h>
37 #include <linux/timer.h>
38 #include <linux/io.h>
39 #include <linux/iova.h>
40 #include <linux/iommu.h>
41 #include <linux/intel-iommu.h>
42 #include <linux/syscore_ops.h>
43 #include <linux/tboot.h>
44 #include <linux/dmi.h>
45 #include <linux/pci-ats.h>
46 #include <linux/memblock.h>
47 #include <linux/dma-contiguous.h>
48 #include <linux/crash_dump.h>
49 #include <asm/irq_remapping.h>
50 #include <asm/cacheflush.h>
51 #include <asm/iommu.h>
52
53 #include "irq_remapping.h"
54
55 #define ROOT_SIZE VTD_PAGE_SIZE
56 #define CONTEXT_SIZE VTD_PAGE_SIZE
57
58 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
59 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
60 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
61 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
62
63 #define IOAPIC_RANGE_START (0xfee00000)
64 #define IOAPIC_RANGE_END (0xfeefffff)
65 #define IOVA_START_ADDR (0x1000)
66
67 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
68
69 #define MAX_AGAW_WIDTH 64
70 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
71
72 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
73 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
74
75 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
76 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
77 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
78 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
79 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
80
81 /* IO virtual address start page frame number */
82 #define IOVA_START_PFN (1)
83
84 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
85 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
86 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
87
88 /* page table handling */
89 #define LEVEL_STRIDE (9)
90 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
91
92 /*
93 * This bitmap is used to advertise the page sizes our hardware support
94 * to the IOMMU core, which will then use this information to split
95 * physically contiguous memory regions it is mapping into page sizes
96 * that we support.
97 *
98 * Traditionally the IOMMU core just handed us the mappings directly,
99 * after making sure the size is an order of a 4KiB page and that the
100 * mapping has natural alignment.
101 *
102 * To retain this behavior, we currently advertise that we support
103 * all page sizes that are an order of 4KiB.
104 *
105 * If at some point we'd like to utilize the IOMMU core's new behavior,
106 * we could change this to advertise the real page sizes we support.
107 */
108 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
109
110 static inline int agaw_to_level(int agaw)
111 {
112 return agaw + 2;
113 }
114
115 static inline int agaw_to_width(int agaw)
116 {
117 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
118 }
119
120 static inline int width_to_agaw(int width)
121 {
122 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
123 }
124
125 static inline unsigned int level_to_offset_bits(int level)
126 {
127 return (level - 1) * LEVEL_STRIDE;
128 }
129
130 static inline int pfn_level_offset(unsigned long pfn, int level)
131 {
132 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
133 }
134
135 static inline unsigned long level_mask(int level)
136 {
137 return -1UL << level_to_offset_bits(level);
138 }
139
140 static inline unsigned long level_size(int level)
141 {
142 return 1UL << level_to_offset_bits(level);
143 }
144
145 static inline unsigned long align_to_level(unsigned long pfn, int level)
146 {
147 return (pfn + level_size(level) - 1) & level_mask(level);
148 }
149
150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
151 {
152 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
153 }
154
155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
156 are never going to work. */
157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
158 {
159 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
160 }
161
162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
163 {
164 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
165 }
166 static inline unsigned long page_to_dma_pfn(struct page *pg)
167 {
168 return mm_to_dma_pfn(page_to_pfn(pg));
169 }
170 static inline unsigned long virt_to_dma_pfn(void *p)
171 {
172 return page_to_dma_pfn(virt_to_page(p));
173 }
174
175 /* global iommu list, set NULL for ignored DMAR units */
176 static struct intel_iommu **g_iommus;
177
178 static void __init check_tylersburg_isoch(void);
179 static int rwbf_quirk;
180
181 /*
182 * set to 1 to panic kernel if can't successfully enable VT-d
183 * (used when kernel is launched w/ TXT)
184 */
185 static int force_on = 0;
186
187 /*
188 * 0: Present
189 * 1-11: Reserved
190 * 12-63: Context Ptr (12 - (haw-1))
191 * 64-127: Reserved
192 */
193 struct root_entry {
194 u64 lo;
195 u64 hi;
196 };
197 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
198
199 /*
200 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
201 * if marked present.
202 */
203 static phys_addr_t root_entry_lctp(struct root_entry *re)
204 {
205 if (!(re->lo & 1))
206 return 0;
207
208 return re->lo & VTD_PAGE_MASK;
209 }
210
211 /*
212 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
213 * if marked present.
214 */
215 static phys_addr_t root_entry_uctp(struct root_entry *re)
216 {
217 if (!(re->hi & 1))
218 return 0;
219
220 return re->hi & VTD_PAGE_MASK;
221 }
222 /*
223 * low 64 bits:
224 * 0: present
225 * 1: fault processing disable
226 * 2-3: translation type
227 * 12-63: address space root
228 * high 64 bits:
229 * 0-2: address width
230 * 3-6: aval
231 * 8-23: domain id
232 */
233 struct context_entry {
234 u64 lo;
235 u64 hi;
236 };
237
238 static inline void context_clear_pasid_enable(struct context_entry *context)
239 {
240 context->lo &= ~(1ULL << 11);
241 }
242
243 static inline bool context_pasid_enabled(struct context_entry *context)
244 {
245 return !!(context->lo & (1ULL << 11));
246 }
247
248 static inline void context_set_copied(struct context_entry *context)
249 {
250 context->hi |= (1ull << 3);
251 }
252
253 static inline bool context_copied(struct context_entry *context)
254 {
255 return !!(context->hi & (1ULL << 3));
256 }
257
258 static inline bool __context_present(struct context_entry *context)
259 {
260 return (context->lo & 1);
261 }
262
263 static inline bool context_present(struct context_entry *context)
264 {
265 return context_pasid_enabled(context) ?
266 __context_present(context) :
267 __context_present(context) && !context_copied(context);
268 }
269
270 static inline void context_set_present(struct context_entry *context)
271 {
272 context->lo |= 1;
273 }
274
275 static inline void context_set_fault_enable(struct context_entry *context)
276 {
277 context->lo &= (((u64)-1) << 2) | 1;
278 }
279
280 static inline void context_set_translation_type(struct context_entry *context,
281 unsigned long value)
282 {
283 context->lo &= (((u64)-1) << 4) | 3;
284 context->lo |= (value & 3) << 2;
285 }
286
287 static inline void context_set_address_root(struct context_entry *context,
288 unsigned long value)
289 {
290 context->lo &= ~VTD_PAGE_MASK;
291 context->lo |= value & VTD_PAGE_MASK;
292 }
293
294 static inline void context_set_address_width(struct context_entry *context,
295 unsigned long value)
296 {
297 context->hi |= value & 7;
298 }
299
300 static inline void context_set_domain_id(struct context_entry *context,
301 unsigned long value)
302 {
303 context->hi |= (value & ((1 << 16) - 1)) << 8;
304 }
305
306 static inline int context_domain_id(struct context_entry *c)
307 {
308 return((c->hi >> 8) & 0xffff);
309 }
310
311 static inline void context_clear_entry(struct context_entry *context)
312 {
313 context->lo = 0;
314 context->hi = 0;
315 }
316
317 /*
318 * 0: readable
319 * 1: writable
320 * 2-6: reserved
321 * 7: super page
322 * 8-10: available
323 * 11: snoop behavior
324 * 12-63: Host physcial address
325 */
326 struct dma_pte {
327 u64 val;
328 };
329
330 static inline void dma_clear_pte(struct dma_pte *pte)
331 {
332 pte->val = 0;
333 }
334
335 static inline u64 dma_pte_addr(struct dma_pte *pte)
336 {
337 #ifdef CONFIG_64BIT
338 return pte->val & VTD_PAGE_MASK;
339 #else
340 /* Must have a full atomic 64-bit read */
341 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
342 #endif
343 }
344
345 static inline bool dma_pte_present(struct dma_pte *pte)
346 {
347 return (pte->val & 3) != 0;
348 }
349
350 static inline bool dma_pte_superpage(struct dma_pte *pte)
351 {
352 return (pte->val & DMA_PTE_LARGE_PAGE);
353 }
354
355 static inline int first_pte_in_page(struct dma_pte *pte)
356 {
357 return !((unsigned long)pte & ~VTD_PAGE_MASK);
358 }
359
360 /*
361 * This domain is a statically identity mapping domain.
362 * 1. This domain creats a static 1:1 mapping to all usable memory.
363 * 2. It maps to each iommu if successful.
364 * 3. Each iommu mapps to this domain if successful.
365 */
366 static struct dmar_domain *si_domain;
367 static int hw_pass_through = 1;
368
369 /*
370 * Domain represents a virtual machine, more than one devices
371 * across iommus may be owned in one domain, e.g. kvm guest.
372 */
373 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
374
375 /* si_domain contains mulitple devices */
376 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
377
378 #define for_each_domain_iommu(idx, domain) \
379 for (idx = 0; idx < g_num_of_iommus; idx++) \
380 if (domain->iommu_refcnt[idx])
381
382 struct dmar_domain {
383 int nid; /* node id */
384
385 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
386 /* Refcount of devices per iommu */
387
388
389 u16 iommu_did[DMAR_UNITS_SUPPORTED];
390 /* Domain ids per IOMMU. Use u16 since
391 * domain ids are 16 bit wide according
392 * to VT-d spec, section 9.3 */
393
394 bool has_iotlb_device;
395 struct list_head devices; /* all devices' list */
396 struct iova_domain iovad; /* iova's that belong to this domain */
397
398 struct dma_pte *pgd; /* virtual address */
399 int gaw; /* max guest address width */
400
401 /* adjusted guest address width, 0 is level 2 30-bit */
402 int agaw;
403
404 int flags; /* flags to find out type of domain */
405
406 int iommu_coherency;/* indicate coherency of iommu access */
407 int iommu_snooping; /* indicate snooping control feature*/
408 int iommu_count; /* reference count of iommu */
409 int iommu_superpage;/* Level of superpages supported:
410 0 == 4KiB (no superpages), 1 == 2MiB,
411 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
412 u64 max_addr; /* maximum mapped address */
413
414 struct iommu_domain domain; /* generic domain data structure for
415 iommu core */
416 };
417
418 /* PCI domain-device relationship */
419 struct device_domain_info {
420 struct list_head link; /* link to domain siblings */
421 struct list_head global; /* link to global list */
422 u8 bus; /* PCI bus number */
423 u8 devfn; /* PCI devfn number */
424 u8 pasid_supported:3;
425 u8 pasid_enabled:1;
426 u8 pri_supported:1;
427 u8 pri_enabled:1;
428 u8 ats_supported:1;
429 u8 ats_enabled:1;
430 u8 ats_qdep;
431 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
432 struct intel_iommu *iommu; /* IOMMU used by this device */
433 struct dmar_domain *domain; /* pointer to domain */
434 };
435
436 struct dmar_rmrr_unit {
437 struct list_head list; /* list of rmrr units */
438 struct acpi_dmar_header *hdr; /* ACPI header */
439 u64 base_address; /* reserved base address*/
440 u64 end_address; /* reserved end address */
441 struct dmar_dev_scope *devices; /* target devices */
442 int devices_cnt; /* target device count */
443 };
444
445 struct dmar_atsr_unit {
446 struct list_head list; /* list of ATSR units */
447 struct acpi_dmar_header *hdr; /* ACPI header */
448 struct dmar_dev_scope *devices; /* target devices */
449 int devices_cnt; /* target device count */
450 u8 include_all:1; /* include all ports */
451 };
452
453 static LIST_HEAD(dmar_atsr_units);
454 static LIST_HEAD(dmar_rmrr_units);
455
456 #define for_each_rmrr_units(rmrr) \
457 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
458
459 static void flush_unmaps_timeout(unsigned long data);
460
461 struct deferred_flush_entry {
462 unsigned long iova_pfn;
463 unsigned long nrpages;
464 struct dmar_domain *domain;
465 struct page *freelist;
466 };
467
468 #define HIGH_WATER_MARK 250
469 struct deferred_flush_table {
470 int next;
471 struct deferred_flush_entry entries[HIGH_WATER_MARK];
472 };
473
474 struct deferred_flush_data {
475 spinlock_t lock;
476 int timer_on;
477 struct timer_list timer;
478 long size;
479 struct deferred_flush_table *tables;
480 };
481
482 DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
483
484 /* bitmap for indexing intel_iommus */
485 static int g_num_of_iommus;
486
487 static void domain_exit(struct dmar_domain *domain);
488 static void domain_remove_dev_info(struct dmar_domain *domain);
489 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
490 struct device *dev);
491 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
492 static void domain_context_clear(struct intel_iommu *iommu,
493 struct device *dev);
494 static int domain_detach_iommu(struct dmar_domain *domain,
495 struct intel_iommu *iommu);
496
497 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
498 int dmar_disabled = 0;
499 #else
500 int dmar_disabled = 1;
501 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
502
503 int intel_iommu_enabled = 0;
504 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
505
506 static int dmar_map_gfx = 1;
507 static int dmar_forcedac;
508 static int intel_iommu_strict;
509 static int intel_iommu_superpage = 1;
510 static int intel_iommu_ecs = 1;
511 static int intel_iommu_pasid28;
512 static int iommu_identity_mapping;
513
514 #define IDENTMAP_ALL 1
515 #define IDENTMAP_GFX 2
516 #define IDENTMAP_AZALIA 4
517
518 /* Broadwell and Skylake have broken ECS support — normal so-called "second
519 * level" translation of DMA requests-without-PASID doesn't actually happen
520 * unless you also set the NESTE bit in an extended context-entry. Which of
521 * course means that SVM doesn't work because it's trying to do nested
522 * translation of the physical addresses it finds in the process page tables,
523 * through the IOVA->phys mapping found in the "second level" page tables.
524 *
525 * The VT-d specification was retroactively changed to change the definition
526 * of the capability bits and pretend that Broadwell/Skylake never happened...
527 * but unfortunately the wrong bit was changed. It's ECS which is broken, but
528 * for some reason it was the PASID capability bit which was redefined (from
529 * bit 28 on BDW/SKL to bit 40 in future).
530 *
531 * So our test for ECS needs to eschew those implementations which set the old
532 * PASID capabiity bit 28, since those are the ones on which ECS is broken.
533 * Unless we are working around the 'pasid28' limitations, that is, by putting
534 * the device into passthrough mode for normal DMA and thus masking the bug.
535 */
536 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
537 (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
538 /* PASID support is thus enabled if ECS is enabled and *either* of the old
539 * or new capability bits are set. */
540 #define pasid_enabled(iommu) (ecs_enabled(iommu) && \
541 (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
542
543 int intel_iommu_gfx_mapped;
544 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
545
546 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
547 static DEFINE_SPINLOCK(device_domain_lock);
548 static LIST_HEAD(device_domain_list);
549
550 static const struct iommu_ops intel_iommu_ops;
551
552 static bool translation_pre_enabled(struct intel_iommu *iommu)
553 {
554 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
555 }
556
557 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
558 {
559 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
560 }
561
562 static void init_translation_status(struct intel_iommu *iommu)
563 {
564 u32 gsts;
565
566 gsts = readl(iommu->reg + DMAR_GSTS_REG);
567 if (gsts & DMA_GSTS_TES)
568 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
569 }
570
571 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
572 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
573 {
574 return container_of(dom, struct dmar_domain, domain);
575 }
576
577 static int __init intel_iommu_setup(char *str)
578 {
579 if (!str)
580 return -EINVAL;
581 while (*str) {
582 if (!strncmp(str, "on", 2)) {
583 dmar_disabled = 0;
584 pr_info("IOMMU enabled\n");
585 } else if (!strncmp(str, "off", 3)) {
586 dmar_disabled = 1;
587 pr_info("IOMMU disabled\n");
588 } else if (!strncmp(str, "igfx_off", 8)) {
589 dmar_map_gfx = 0;
590 pr_info("Disable GFX device mapping\n");
591 } else if (!strncmp(str, "forcedac", 8)) {
592 pr_info("Forcing DAC for PCI devices\n");
593 dmar_forcedac = 1;
594 } else if (!strncmp(str, "strict", 6)) {
595 pr_info("Disable batched IOTLB flush\n");
596 intel_iommu_strict = 1;
597 } else if (!strncmp(str, "sp_off", 6)) {
598 pr_info("Disable supported super page\n");
599 intel_iommu_superpage = 0;
600 } else if (!strncmp(str, "ecs_off", 7)) {
601 printk(KERN_INFO
602 "Intel-IOMMU: disable extended context table support\n");
603 intel_iommu_ecs = 0;
604 } else if (!strncmp(str, "pasid28", 7)) {
605 printk(KERN_INFO
606 "Intel-IOMMU: enable pre-production PASID support\n");
607 intel_iommu_pasid28 = 1;
608 iommu_identity_mapping |= IDENTMAP_GFX;
609 }
610
611 str += strcspn(str, ",");
612 while (*str == ',')
613 str++;
614 }
615 return 0;
616 }
617 __setup("intel_iommu=", intel_iommu_setup);
618
619 static struct kmem_cache *iommu_domain_cache;
620 static struct kmem_cache *iommu_devinfo_cache;
621
622 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
623 {
624 struct dmar_domain **domains;
625 int idx = did >> 8;
626
627 domains = iommu->domains[idx];
628 if (!domains)
629 return NULL;
630
631 return domains[did & 0xff];
632 }
633
634 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
635 struct dmar_domain *domain)
636 {
637 struct dmar_domain **domains;
638 int idx = did >> 8;
639
640 if (!iommu->domains[idx]) {
641 size_t size = 256 * sizeof(struct dmar_domain *);
642 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
643 }
644
645 domains = iommu->domains[idx];
646 if (WARN_ON(!domains))
647 return;
648 else
649 domains[did & 0xff] = domain;
650 }
651
652 static inline void *alloc_pgtable_page(int node)
653 {
654 struct page *page;
655 void *vaddr = NULL;
656
657 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
658 if (page)
659 vaddr = page_address(page);
660 return vaddr;
661 }
662
663 static inline void free_pgtable_page(void *vaddr)
664 {
665 free_page((unsigned long)vaddr);
666 }
667
668 static inline void *alloc_domain_mem(void)
669 {
670 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
671 }
672
673 static void free_domain_mem(void *vaddr)
674 {
675 kmem_cache_free(iommu_domain_cache, vaddr);
676 }
677
678 static inline void * alloc_devinfo_mem(void)
679 {
680 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
681 }
682
683 static inline void free_devinfo_mem(void *vaddr)
684 {
685 kmem_cache_free(iommu_devinfo_cache, vaddr);
686 }
687
688 static inline int domain_type_is_vm(struct dmar_domain *domain)
689 {
690 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
691 }
692
693 static inline int domain_type_is_si(struct dmar_domain *domain)
694 {
695 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
696 }
697
698 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
699 {
700 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
701 DOMAIN_FLAG_STATIC_IDENTITY);
702 }
703
704 static inline int domain_pfn_supported(struct dmar_domain *domain,
705 unsigned long pfn)
706 {
707 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
708
709 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
710 }
711
712 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
713 {
714 unsigned long sagaw;
715 int agaw = -1;
716
717 sagaw = cap_sagaw(iommu->cap);
718 for (agaw = width_to_agaw(max_gaw);
719 agaw >= 0; agaw--) {
720 if (test_bit(agaw, &sagaw))
721 break;
722 }
723
724 return agaw;
725 }
726
727 /*
728 * Calculate max SAGAW for each iommu.
729 */
730 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
731 {
732 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
733 }
734
735 /*
736 * calculate agaw for each iommu.
737 * "SAGAW" may be different across iommus, use a default agaw, and
738 * get a supported less agaw for iommus that don't support the default agaw.
739 */
740 int iommu_calculate_agaw(struct intel_iommu *iommu)
741 {
742 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
743 }
744
745 /* This functionin only returns single iommu in a domain */
746 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
747 {
748 int iommu_id;
749
750 /* si_domain and vm domain should not get here. */
751 BUG_ON(domain_type_is_vm_or_si(domain));
752 for_each_domain_iommu(iommu_id, domain)
753 break;
754
755 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
756 return NULL;
757
758 return g_iommus[iommu_id];
759 }
760
761 static void domain_update_iommu_coherency(struct dmar_domain *domain)
762 {
763 struct dmar_drhd_unit *drhd;
764 struct intel_iommu *iommu;
765 bool found = false;
766 int i;
767
768 domain->iommu_coherency = 1;
769
770 for_each_domain_iommu(i, domain) {
771 found = true;
772 if (!ecap_coherent(g_iommus[i]->ecap)) {
773 domain->iommu_coherency = 0;
774 break;
775 }
776 }
777 if (found)
778 return;
779
780 /* No hardware attached; use lowest common denominator */
781 rcu_read_lock();
782 for_each_active_iommu(iommu, drhd) {
783 if (!ecap_coherent(iommu->ecap)) {
784 domain->iommu_coherency = 0;
785 break;
786 }
787 }
788 rcu_read_unlock();
789 }
790
791 static int domain_update_iommu_snooping(struct intel_iommu *skip)
792 {
793 struct dmar_drhd_unit *drhd;
794 struct intel_iommu *iommu;
795 int ret = 1;
796
797 rcu_read_lock();
798 for_each_active_iommu(iommu, drhd) {
799 if (iommu != skip) {
800 if (!ecap_sc_support(iommu->ecap)) {
801 ret = 0;
802 break;
803 }
804 }
805 }
806 rcu_read_unlock();
807
808 return ret;
809 }
810
811 static int domain_update_iommu_superpage(struct intel_iommu *skip)
812 {
813 struct dmar_drhd_unit *drhd;
814 struct intel_iommu *iommu;
815 int mask = 0xf;
816
817 if (!intel_iommu_superpage) {
818 return 0;
819 }
820
821 /* set iommu_superpage to the smallest common denominator */
822 rcu_read_lock();
823 for_each_active_iommu(iommu, drhd) {
824 if (iommu != skip) {
825 mask &= cap_super_page_val(iommu->cap);
826 if (!mask)
827 break;
828 }
829 }
830 rcu_read_unlock();
831
832 return fls(mask);
833 }
834
835 /* Some capabilities may be different across iommus */
836 static void domain_update_iommu_cap(struct dmar_domain *domain)
837 {
838 domain_update_iommu_coherency(domain);
839 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
840 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
841 }
842
843 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
844 u8 bus, u8 devfn, int alloc)
845 {
846 struct root_entry *root = &iommu->root_entry[bus];
847 struct context_entry *context;
848 u64 *entry;
849
850 entry = &root->lo;
851 if (ecs_enabled(iommu)) {
852 if (devfn >= 0x80) {
853 devfn -= 0x80;
854 entry = &root->hi;
855 }
856 devfn *= 2;
857 }
858 if (*entry & 1)
859 context = phys_to_virt(*entry & VTD_PAGE_MASK);
860 else {
861 unsigned long phy_addr;
862 if (!alloc)
863 return NULL;
864
865 context = alloc_pgtable_page(iommu->node);
866 if (!context)
867 return NULL;
868
869 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
870 phy_addr = virt_to_phys((void *)context);
871 *entry = phy_addr | 1;
872 __iommu_flush_cache(iommu, entry, sizeof(*entry));
873 }
874 return &context[devfn];
875 }
876
877 static int iommu_dummy(struct device *dev)
878 {
879 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
880 }
881
882 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
883 {
884 struct dmar_drhd_unit *drhd = NULL;
885 struct intel_iommu *iommu;
886 struct device *tmp;
887 struct pci_dev *ptmp, *pdev = NULL;
888 u16 segment = 0;
889 int i;
890
891 if (iommu_dummy(dev))
892 return NULL;
893
894 if (dev_is_pci(dev)) {
895 pdev = to_pci_dev(dev);
896 segment = pci_domain_nr(pdev->bus);
897 } else if (has_acpi_companion(dev))
898 dev = &ACPI_COMPANION(dev)->dev;
899
900 rcu_read_lock();
901 for_each_active_iommu(iommu, drhd) {
902 if (pdev && segment != drhd->segment)
903 continue;
904
905 for_each_active_dev_scope(drhd->devices,
906 drhd->devices_cnt, i, tmp) {
907 if (tmp == dev) {
908 *bus = drhd->devices[i].bus;
909 *devfn = drhd->devices[i].devfn;
910 goto out;
911 }
912
913 if (!pdev || !dev_is_pci(tmp))
914 continue;
915
916 ptmp = to_pci_dev(tmp);
917 if (ptmp->subordinate &&
918 ptmp->subordinate->number <= pdev->bus->number &&
919 ptmp->subordinate->busn_res.end >= pdev->bus->number)
920 goto got_pdev;
921 }
922
923 if (pdev && drhd->include_all) {
924 got_pdev:
925 *bus = pdev->bus->number;
926 *devfn = pdev->devfn;
927 goto out;
928 }
929 }
930 iommu = NULL;
931 out:
932 rcu_read_unlock();
933
934 return iommu;
935 }
936
937 static void domain_flush_cache(struct dmar_domain *domain,
938 void *addr, int size)
939 {
940 if (!domain->iommu_coherency)
941 clflush_cache_range(addr, size);
942 }
943
944 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
945 {
946 struct context_entry *context;
947 int ret = 0;
948 unsigned long flags;
949
950 spin_lock_irqsave(&iommu->lock, flags);
951 context = iommu_context_addr(iommu, bus, devfn, 0);
952 if (context)
953 ret = context_present(context);
954 spin_unlock_irqrestore(&iommu->lock, flags);
955 return ret;
956 }
957
958 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
959 {
960 struct context_entry *context;
961 unsigned long flags;
962
963 spin_lock_irqsave(&iommu->lock, flags);
964 context = iommu_context_addr(iommu, bus, devfn, 0);
965 if (context) {
966 context_clear_entry(context);
967 __iommu_flush_cache(iommu, context, sizeof(*context));
968 }
969 spin_unlock_irqrestore(&iommu->lock, flags);
970 }
971
972 static void free_context_table(struct intel_iommu *iommu)
973 {
974 int i;
975 unsigned long flags;
976 struct context_entry *context;
977
978 spin_lock_irqsave(&iommu->lock, flags);
979 if (!iommu->root_entry) {
980 goto out;
981 }
982 for (i = 0; i < ROOT_ENTRY_NR; i++) {
983 context = iommu_context_addr(iommu, i, 0, 0);
984 if (context)
985 free_pgtable_page(context);
986
987 if (!ecs_enabled(iommu))
988 continue;
989
990 context = iommu_context_addr(iommu, i, 0x80, 0);
991 if (context)
992 free_pgtable_page(context);
993
994 }
995 free_pgtable_page(iommu->root_entry);
996 iommu->root_entry = NULL;
997 out:
998 spin_unlock_irqrestore(&iommu->lock, flags);
999 }
1000
1001 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
1002 unsigned long pfn, int *target_level)
1003 {
1004 struct dma_pte *parent, *pte = NULL;
1005 int level = agaw_to_level(domain->agaw);
1006 int offset;
1007
1008 BUG_ON(!domain->pgd);
1009
1010 if (!domain_pfn_supported(domain, pfn))
1011 /* Address beyond IOMMU's addressing capabilities. */
1012 return NULL;
1013
1014 parent = domain->pgd;
1015
1016 while (1) {
1017 void *tmp_page;
1018
1019 offset = pfn_level_offset(pfn, level);
1020 pte = &parent[offset];
1021 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1022 break;
1023 if (level == *target_level)
1024 break;
1025
1026 if (!dma_pte_present(pte)) {
1027 uint64_t pteval;
1028
1029 tmp_page = alloc_pgtable_page(domain->nid);
1030
1031 if (!tmp_page)
1032 return NULL;
1033
1034 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1035 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1036 if (cmpxchg64(&pte->val, 0ULL, pteval))
1037 /* Someone else set it while we were thinking; use theirs. */
1038 free_pgtable_page(tmp_page);
1039 else
1040 domain_flush_cache(domain, pte, sizeof(*pte));
1041 }
1042 if (level == 1)
1043 break;
1044
1045 parent = phys_to_virt(dma_pte_addr(pte));
1046 level--;
1047 }
1048
1049 if (!*target_level)
1050 *target_level = level;
1051
1052 return pte;
1053 }
1054
1055
1056 /* return address's pte at specific level */
1057 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1058 unsigned long pfn,
1059 int level, int *large_page)
1060 {
1061 struct dma_pte *parent, *pte = NULL;
1062 int total = agaw_to_level(domain->agaw);
1063 int offset;
1064
1065 parent = domain->pgd;
1066 while (level <= total) {
1067 offset = pfn_level_offset(pfn, total);
1068 pte = &parent[offset];
1069 if (level == total)
1070 return pte;
1071
1072 if (!dma_pte_present(pte)) {
1073 *large_page = total;
1074 break;
1075 }
1076
1077 if (dma_pte_superpage(pte)) {
1078 *large_page = total;
1079 return pte;
1080 }
1081
1082 parent = phys_to_virt(dma_pte_addr(pte));
1083 total--;
1084 }
1085 return NULL;
1086 }
1087
1088 /* clear last level pte, a tlb flush should be followed */
1089 static void dma_pte_clear_range(struct dmar_domain *domain,
1090 unsigned long start_pfn,
1091 unsigned long last_pfn)
1092 {
1093 unsigned int large_page = 1;
1094 struct dma_pte *first_pte, *pte;
1095
1096 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1097 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1098 BUG_ON(start_pfn > last_pfn);
1099
1100 /* we don't need lock here; nobody else touches the iova range */
1101 do {
1102 large_page = 1;
1103 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1104 if (!pte) {
1105 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1106 continue;
1107 }
1108 do {
1109 dma_clear_pte(pte);
1110 start_pfn += lvl_to_nr_pages(large_page);
1111 pte++;
1112 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1113
1114 domain_flush_cache(domain, first_pte,
1115 (void *)pte - (void *)first_pte);
1116
1117 } while (start_pfn && start_pfn <= last_pfn);
1118 }
1119
1120 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1121 struct dma_pte *pte, unsigned long pfn,
1122 unsigned long start_pfn, unsigned long last_pfn)
1123 {
1124 pfn = max(start_pfn, pfn);
1125 pte = &pte[pfn_level_offset(pfn, level)];
1126
1127 do {
1128 unsigned long level_pfn;
1129 struct dma_pte *level_pte;
1130
1131 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1132 goto next;
1133
1134 level_pfn = pfn & level_mask(level - 1);
1135 level_pte = phys_to_virt(dma_pte_addr(pte));
1136
1137 if (level > 2)
1138 dma_pte_free_level(domain, level - 1, level_pte,
1139 level_pfn, start_pfn, last_pfn);
1140
1141 /* If range covers entire pagetable, free it */
1142 if (!(start_pfn > level_pfn ||
1143 last_pfn < level_pfn + level_size(level) - 1)) {
1144 dma_clear_pte(pte);
1145 domain_flush_cache(domain, pte, sizeof(*pte));
1146 free_pgtable_page(level_pte);
1147 }
1148 next:
1149 pfn += level_size(level);
1150 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1151 }
1152
1153 /* clear last level (leaf) ptes and free page table pages. */
1154 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1155 unsigned long start_pfn,
1156 unsigned long last_pfn)
1157 {
1158 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1159 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1160 BUG_ON(start_pfn > last_pfn);
1161
1162 dma_pte_clear_range(domain, start_pfn, last_pfn);
1163
1164 /* We don't need lock here; nobody else touches the iova range */
1165 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1166 domain->pgd, 0, start_pfn, last_pfn);
1167
1168 /* free pgd */
1169 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1170 free_pgtable_page(domain->pgd);
1171 domain->pgd = NULL;
1172 }
1173 }
1174
1175 /* When a page at a given level is being unlinked from its parent, we don't
1176 need to *modify* it at all. All we need to do is make a list of all the
1177 pages which can be freed just as soon as we've flushed the IOTLB and we
1178 know the hardware page-walk will no longer touch them.
1179 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1180 be freed. */
1181 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1182 int level, struct dma_pte *pte,
1183 struct page *freelist)
1184 {
1185 struct page *pg;
1186
1187 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1188 pg->freelist = freelist;
1189 freelist = pg;
1190
1191 if (level == 1)
1192 return freelist;
1193
1194 pte = page_address(pg);
1195 do {
1196 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1197 freelist = dma_pte_list_pagetables(domain, level - 1,
1198 pte, freelist);
1199 pte++;
1200 } while (!first_pte_in_page(pte));
1201
1202 return freelist;
1203 }
1204
1205 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1206 struct dma_pte *pte, unsigned long pfn,
1207 unsigned long start_pfn,
1208 unsigned long last_pfn,
1209 struct page *freelist)
1210 {
1211 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1212
1213 pfn = max(start_pfn, pfn);
1214 pte = &pte[pfn_level_offset(pfn, level)];
1215
1216 do {
1217 unsigned long level_pfn;
1218
1219 if (!dma_pte_present(pte))
1220 goto next;
1221
1222 level_pfn = pfn & level_mask(level);
1223
1224 /* If range covers entire pagetable, free it */
1225 if (start_pfn <= level_pfn &&
1226 last_pfn >= level_pfn + level_size(level) - 1) {
1227 /* These suborbinate page tables are going away entirely. Don't
1228 bother to clear them; we're just going to *free* them. */
1229 if (level > 1 && !dma_pte_superpage(pte))
1230 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1231
1232 dma_clear_pte(pte);
1233 if (!first_pte)
1234 first_pte = pte;
1235 last_pte = pte;
1236 } else if (level > 1) {
1237 /* Recurse down into a level that isn't *entirely* obsolete */
1238 freelist = dma_pte_clear_level(domain, level - 1,
1239 phys_to_virt(dma_pte_addr(pte)),
1240 level_pfn, start_pfn, last_pfn,
1241 freelist);
1242 }
1243 next:
1244 pfn += level_size(level);
1245 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1246
1247 if (first_pte)
1248 domain_flush_cache(domain, first_pte,
1249 (void *)++last_pte - (void *)first_pte);
1250
1251 return freelist;
1252 }
1253
1254 /* We can't just free the pages because the IOMMU may still be walking
1255 the page tables, and may have cached the intermediate levels. The
1256 pages can only be freed after the IOTLB flush has been done. */
1257 static struct page *domain_unmap(struct dmar_domain *domain,
1258 unsigned long start_pfn,
1259 unsigned long last_pfn)
1260 {
1261 struct page *freelist = NULL;
1262
1263 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1264 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1265 BUG_ON(start_pfn > last_pfn);
1266
1267 /* we don't need lock here; nobody else touches the iova range */
1268 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1269 domain->pgd, 0, start_pfn, last_pfn, NULL);
1270
1271 /* free pgd */
1272 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1273 struct page *pgd_page = virt_to_page(domain->pgd);
1274 pgd_page->freelist = freelist;
1275 freelist = pgd_page;
1276
1277 domain->pgd = NULL;
1278 }
1279
1280 return freelist;
1281 }
1282
1283 static void dma_free_pagelist(struct page *freelist)
1284 {
1285 struct page *pg;
1286
1287 while ((pg = freelist)) {
1288 freelist = pg->freelist;
1289 free_pgtable_page(page_address(pg));
1290 }
1291 }
1292
1293 /* iommu handling */
1294 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1295 {
1296 struct root_entry *root;
1297 unsigned long flags;
1298
1299 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1300 if (!root) {
1301 pr_err("Allocating root entry for %s failed\n",
1302 iommu->name);
1303 return -ENOMEM;
1304 }
1305
1306 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1307
1308 spin_lock_irqsave(&iommu->lock, flags);
1309 iommu->root_entry = root;
1310 spin_unlock_irqrestore(&iommu->lock, flags);
1311
1312 return 0;
1313 }
1314
1315 static void iommu_set_root_entry(struct intel_iommu *iommu)
1316 {
1317 u64 addr;
1318 u32 sts;
1319 unsigned long flag;
1320
1321 addr = virt_to_phys(iommu->root_entry);
1322 if (ecs_enabled(iommu))
1323 addr |= DMA_RTADDR_RTT;
1324
1325 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1326 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1327
1328 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1329
1330 /* Make sure hardware complete it */
1331 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1332 readl, (sts & DMA_GSTS_RTPS), sts);
1333
1334 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1335 }
1336
1337 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1338 {
1339 u32 val;
1340 unsigned long flag;
1341
1342 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1343 return;
1344
1345 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1346 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1347
1348 /* Make sure hardware complete it */
1349 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1350 readl, (!(val & DMA_GSTS_WBFS)), val);
1351
1352 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1353 }
1354
1355 /* return value determine if we need a write buffer flush */
1356 static void __iommu_flush_context(struct intel_iommu *iommu,
1357 u16 did, u16 source_id, u8 function_mask,
1358 u64 type)
1359 {
1360 u64 val = 0;
1361 unsigned long flag;
1362
1363 switch (type) {
1364 case DMA_CCMD_GLOBAL_INVL:
1365 val = DMA_CCMD_GLOBAL_INVL;
1366 break;
1367 case DMA_CCMD_DOMAIN_INVL:
1368 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1369 break;
1370 case DMA_CCMD_DEVICE_INVL:
1371 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1372 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1373 break;
1374 default:
1375 BUG();
1376 }
1377 val |= DMA_CCMD_ICC;
1378
1379 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1380 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1381
1382 /* Make sure hardware complete it */
1383 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1384 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1385
1386 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1387 }
1388
1389 /* return value determine if we need a write buffer flush */
1390 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1391 u64 addr, unsigned int size_order, u64 type)
1392 {
1393 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1394 u64 val = 0, val_iva = 0;
1395 unsigned long flag;
1396
1397 switch (type) {
1398 case DMA_TLB_GLOBAL_FLUSH:
1399 /* global flush doesn't need set IVA_REG */
1400 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1401 break;
1402 case DMA_TLB_DSI_FLUSH:
1403 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1404 break;
1405 case DMA_TLB_PSI_FLUSH:
1406 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1407 /* IH bit is passed in as part of address */
1408 val_iva = size_order | addr;
1409 break;
1410 default:
1411 BUG();
1412 }
1413 /* Note: set drain read/write */
1414 #if 0
1415 /*
1416 * This is probably to be super secure.. Looks like we can
1417 * ignore it without any impact.
1418 */
1419 if (cap_read_drain(iommu->cap))
1420 val |= DMA_TLB_READ_DRAIN;
1421 #endif
1422 if (cap_write_drain(iommu->cap))
1423 val |= DMA_TLB_WRITE_DRAIN;
1424
1425 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1426 /* Note: Only uses first TLB reg currently */
1427 if (val_iva)
1428 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1429 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1430
1431 /* Make sure hardware complete it */
1432 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1433 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1434
1435 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1436
1437 /* check IOTLB invalidation granularity */
1438 if (DMA_TLB_IAIG(val) == 0)
1439 pr_err("Flush IOTLB failed\n");
1440 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1441 pr_debug("TLB flush request %Lx, actual %Lx\n",
1442 (unsigned long long)DMA_TLB_IIRG(type),
1443 (unsigned long long)DMA_TLB_IAIG(val));
1444 }
1445
1446 static struct device_domain_info *
1447 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1448 u8 bus, u8 devfn)
1449 {
1450 struct device_domain_info *info;
1451
1452 assert_spin_locked(&device_domain_lock);
1453
1454 if (!iommu->qi)
1455 return NULL;
1456
1457 list_for_each_entry(info, &domain->devices, link)
1458 if (info->iommu == iommu && info->bus == bus &&
1459 info->devfn == devfn) {
1460 if (info->ats_supported && info->dev)
1461 return info;
1462 break;
1463 }
1464
1465 return NULL;
1466 }
1467
1468 static void domain_update_iotlb(struct dmar_domain *domain)
1469 {
1470 struct device_domain_info *info;
1471 bool has_iotlb_device = false;
1472
1473 assert_spin_locked(&device_domain_lock);
1474
1475 list_for_each_entry(info, &domain->devices, link) {
1476 struct pci_dev *pdev;
1477
1478 if (!info->dev || !dev_is_pci(info->dev))
1479 continue;
1480
1481 pdev = to_pci_dev(info->dev);
1482 if (pdev->ats_enabled) {
1483 has_iotlb_device = true;
1484 break;
1485 }
1486 }
1487
1488 domain->has_iotlb_device = has_iotlb_device;
1489 }
1490
1491 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1492 {
1493 struct pci_dev *pdev;
1494
1495 assert_spin_locked(&device_domain_lock);
1496
1497 if (!info || !dev_is_pci(info->dev))
1498 return;
1499
1500 pdev = to_pci_dev(info->dev);
1501
1502 #ifdef CONFIG_INTEL_IOMMU_SVM
1503 /* The PCIe spec, in its wisdom, declares that the behaviour of
1504 the device if you enable PASID support after ATS support is
1505 undefined. So always enable PASID support on devices which
1506 have it, even if we can't yet know if we're ever going to
1507 use it. */
1508 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1509 info->pasid_enabled = 1;
1510
1511 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1512 info->pri_enabled = 1;
1513 #endif
1514 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1515 info->ats_enabled = 1;
1516 domain_update_iotlb(info->domain);
1517 info->ats_qdep = pci_ats_queue_depth(pdev);
1518 }
1519 }
1520
1521 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1522 {
1523 struct pci_dev *pdev;
1524
1525 assert_spin_locked(&device_domain_lock);
1526
1527 if (!dev_is_pci(info->dev))
1528 return;
1529
1530 pdev = to_pci_dev(info->dev);
1531
1532 if (info->ats_enabled) {
1533 pci_disable_ats(pdev);
1534 info->ats_enabled = 0;
1535 domain_update_iotlb(info->domain);
1536 }
1537 #ifdef CONFIG_INTEL_IOMMU_SVM
1538 if (info->pri_enabled) {
1539 pci_disable_pri(pdev);
1540 info->pri_enabled = 0;
1541 }
1542 if (info->pasid_enabled) {
1543 pci_disable_pasid(pdev);
1544 info->pasid_enabled = 0;
1545 }
1546 #endif
1547 }
1548
1549 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1550 u64 addr, unsigned mask)
1551 {
1552 u16 sid, qdep;
1553 unsigned long flags;
1554 struct device_domain_info *info;
1555
1556 if (!domain->has_iotlb_device)
1557 return;
1558
1559 spin_lock_irqsave(&device_domain_lock, flags);
1560 list_for_each_entry(info, &domain->devices, link) {
1561 if (!info->ats_enabled)
1562 continue;
1563
1564 sid = info->bus << 8 | info->devfn;
1565 qdep = info->ats_qdep;
1566 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1567 }
1568 spin_unlock_irqrestore(&device_domain_lock, flags);
1569 }
1570
1571 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1572 struct dmar_domain *domain,
1573 unsigned long pfn, unsigned int pages,
1574 int ih, int map)
1575 {
1576 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1577 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1578 u16 did = domain->iommu_did[iommu->seq_id];
1579
1580 BUG_ON(pages == 0);
1581
1582 if (ih)
1583 ih = 1 << 6;
1584 /*
1585 * Fallback to domain selective flush if no PSI support or the size is
1586 * too big.
1587 * PSI requires page size to be 2 ^ x, and the base address is naturally
1588 * aligned to the size
1589 */
1590 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1591 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1592 DMA_TLB_DSI_FLUSH);
1593 else
1594 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1595 DMA_TLB_PSI_FLUSH);
1596
1597 /*
1598 * In caching mode, changes of pages from non-present to present require
1599 * flush. However, device IOTLB doesn't need to be flushed in this case.
1600 */
1601 if (!cap_caching_mode(iommu->cap) || !map)
1602 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1603 addr, mask);
1604 }
1605
1606 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1607 {
1608 u32 pmen;
1609 unsigned long flags;
1610
1611 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1612 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1613 pmen &= ~DMA_PMEN_EPM;
1614 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1615
1616 /* wait for the protected region status bit to clear */
1617 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1618 readl, !(pmen & DMA_PMEN_PRS), pmen);
1619
1620 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1621 }
1622
1623 static void iommu_enable_translation(struct intel_iommu *iommu)
1624 {
1625 u32 sts;
1626 unsigned long flags;
1627
1628 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1629 iommu->gcmd |= DMA_GCMD_TE;
1630 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1631
1632 /* Make sure hardware complete it */
1633 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1634 readl, (sts & DMA_GSTS_TES), sts);
1635
1636 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1637 }
1638
1639 static void iommu_disable_translation(struct intel_iommu *iommu)
1640 {
1641 u32 sts;
1642 unsigned long flag;
1643
1644 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1645 iommu->gcmd &= ~DMA_GCMD_TE;
1646 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1647
1648 /* Make sure hardware complete it */
1649 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1650 readl, (!(sts & DMA_GSTS_TES)), sts);
1651
1652 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1653 }
1654
1655
1656 static int iommu_init_domains(struct intel_iommu *iommu)
1657 {
1658 u32 ndomains, nlongs;
1659 size_t size;
1660
1661 ndomains = cap_ndoms(iommu->cap);
1662 pr_debug("%s: Number of Domains supported <%d>\n",
1663 iommu->name, ndomains);
1664 nlongs = BITS_TO_LONGS(ndomains);
1665
1666 spin_lock_init(&iommu->lock);
1667
1668 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1669 if (!iommu->domain_ids) {
1670 pr_err("%s: Allocating domain id array failed\n",
1671 iommu->name);
1672 return -ENOMEM;
1673 }
1674
1675 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1676 iommu->domains = kzalloc(size, GFP_KERNEL);
1677
1678 if (iommu->domains) {
1679 size = 256 * sizeof(struct dmar_domain *);
1680 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1681 }
1682
1683 if (!iommu->domains || !iommu->domains[0]) {
1684 pr_err("%s: Allocating domain array failed\n",
1685 iommu->name);
1686 kfree(iommu->domain_ids);
1687 kfree(iommu->domains);
1688 iommu->domain_ids = NULL;
1689 iommu->domains = NULL;
1690 return -ENOMEM;
1691 }
1692
1693
1694
1695 /*
1696 * If Caching mode is set, then invalid translations are tagged
1697 * with domain-id 0, hence we need to pre-allocate it. We also
1698 * use domain-id 0 as a marker for non-allocated domain-id, so
1699 * make sure it is not used for a real domain.
1700 */
1701 set_bit(0, iommu->domain_ids);
1702
1703 return 0;
1704 }
1705
1706 static void disable_dmar_iommu(struct intel_iommu *iommu)
1707 {
1708 struct device_domain_info *info, *tmp;
1709 unsigned long flags;
1710
1711 if (!iommu->domains || !iommu->domain_ids)
1712 return;
1713
1714 spin_lock_irqsave(&device_domain_lock, flags);
1715 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1716 struct dmar_domain *domain;
1717
1718 if (info->iommu != iommu)
1719 continue;
1720
1721 if (!info->dev || !info->domain)
1722 continue;
1723
1724 domain = info->domain;
1725
1726 dmar_remove_one_dev_info(domain, info->dev);
1727
1728 if (!domain_type_is_vm_or_si(domain))
1729 domain_exit(domain);
1730 }
1731 spin_unlock_irqrestore(&device_domain_lock, flags);
1732
1733 if (iommu->gcmd & DMA_GCMD_TE)
1734 iommu_disable_translation(iommu);
1735 }
1736
1737 static void free_dmar_iommu(struct intel_iommu *iommu)
1738 {
1739 if ((iommu->domains) && (iommu->domain_ids)) {
1740 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1741 int i;
1742
1743 for (i = 0; i < elems; i++)
1744 kfree(iommu->domains[i]);
1745 kfree(iommu->domains);
1746 kfree(iommu->domain_ids);
1747 iommu->domains = NULL;
1748 iommu->domain_ids = NULL;
1749 }
1750
1751 g_iommus[iommu->seq_id] = NULL;
1752
1753 /* free context mapping */
1754 free_context_table(iommu);
1755
1756 #ifdef CONFIG_INTEL_IOMMU_SVM
1757 if (pasid_enabled(iommu)) {
1758 if (ecap_prs(iommu->ecap))
1759 intel_svm_finish_prq(iommu);
1760 intel_svm_free_pasid_tables(iommu);
1761 }
1762 #endif
1763 }
1764
1765 static struct dmar_domain *alloc_domain(int flags)
1766 {
1767 struct dmar_domain *domain;
1768
1769 domain = alloc_domain_mem();
1770 if (!domain)
1771 return NULL;
1772
1773 memset(domain, 0, sizeof(*domain));
1774 domain->nid = -1;
1775 domain->flags = flags;
1776 domain->has_iotlb_device = false;
1777 INIT_LIST_HEAD(&domain->devices);
1778
1779 return domain;
1780 }
1781
1782 /* Must be called with iommu->lock */
1783 static int domain_attach_iommu(struct dmar_domain *domain,
1784 struct intel_iommu *iommu)
1785 {
1786 unsigned long ndomains;
1787 int num;
1788
1789 assert_spin_locked(&device_domain_lock);
1790 assert_spin_locked(&iommu->lock);
1791
1792 domain->iommu_refcnt[iommu->seq_id] += 1;
1793 domain->iommu_count += 1;
1794 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1795 ndomains = cap_ndoms(iommu->cap);
1796 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1797
1798 if (num >= ndomains) {
1799 pr_err("%s: No free domain ids\n", iommu->name);
1800 domain->iommu_refcnt[iommu->seq_id] -= 1;
1801 domain->iommu_count -= 1;
1802 return -ENOSPC;
1803 }
1804
1805 set_bit(num, iommu->domain_ids);
1806 set_iommu_domain(iommu, num, domain);
1807
1808 domain->iommu_did[iommu->seq_id] = num;
1809 domain->nid = iommu->node;
1810
1811 domain_update_iommu_cap(domain);
1812 }
1813
1814 return 0;
1815 }
1816
1817 static int domain_detach_iommu(struct dmar_domain *domain,
1818 struct intel_iommu *iommu)
1819 {
1820 int num, count = INT_MAX;
1821
1822 assert_spin_locked(&device_domain_lock);
1823 assert_spin_locked(&iommu->lock);
1824
1825 domain->iommu_refcnt[iommu->seq_id] -= 1;
1826 count = --domain->iommu_count;
1827 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1828 num = domain->iommu_did[iommu->seq_id];
1829 clear_bit(num, iommu->domain_ids);
1830 set_iommu_domain(iommu, num, NULL);
1831
1832 domain_update_iommu_cap(domain);
1833 domain->iommu_did[iommu->seq_id] = 0;
1834 }
1835
1836 return count;
1837 }
1838
1839 static struct iova_domain reserved_iova_list;
1840 static struct lock_class_key reserved_rbtree_key;
1841
1842 static int dmar_init_reserved_ranges(void)
1843 {
1844 struct pci_dev *pdev = NULL;
1845 struct iova *iova;
1846 int i;
1847
1848 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1849 DMA_32BIT_PFN);
1850
1851 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1852 &reserved_rbtree_key);
1853
1854 /* IOAPIC ranges shouldn't be accessed by DMA */
1855 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1856 IOVA_PFN(IOAPIC_RANGE_END));
1857 if (!iova) {
1858 pr_err("Reserve IOAPIC range failed\n");
1859 return -ENODEV;
1860 }
1861
1862 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1863 for_each_pci_dev(pdev) {
1864 struct resource *r;
1865
1866 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1867 r = &pdev->resource[i];
1868 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1869 continue;
1870 iova = reserve_iova(&reserved_iova_list,
1871 IOVA_PFN(r->start),
1872 IOVA_PFN(r->end));
1873 if (!iova) {
1874 pr_err("Reserve iova failed\n");
1875 return -ENODEV;
1876 }
1877 }
1878 }
1879 return 0;
1880 }
1881
1882 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1883 {
1884 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1885 }
1886
1887 static inline int guestwidth_to_adjustwidth(int gaw)
1888 {
1889 int agaw;
1890 int r = (gaw - 12) % 9;
1891
1892 if (r == 0)
1893 agaw = gaw;
1894 else
1895 agaw = gaw + 9 - r;
1896 if (agaw > 64)
1897 agaw = 64;
1898 return agaw;
1899 }
1900
1901 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1902 int guest_width)
1903 {
1904 int adjust_width, agaw;
1905 unsigned long sagaw;
1906
1907 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1908 DMA_32BIT_PFN);
1909 domain_reserve_special_ranges(domain);
1910
1911 /* calculate AGAW */
1912 if (guest_width > cap_mgaw(iommu->cap))
1913 guest_width = cap_mgaw(iommu->cap);
1914 domain->gaw = guest_width;
1915 adjust_width = guestwidth_to_adjustwidth(guest_width);
1916 agaw = width_to_agaw(adjust_width);
1917 sagaw = cap_sagaw(iommu->cap);
1918 if (!test_bit(agaw, &sagaw)) {
1919 /* hardware doesn't support it, choose a bigger one */
1920 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1921 agaw = find_next_bit(&sagaw, 5, agaw);
1922 if (agaw >= 5)
1923 return -ENODEV;
1924 }
1925 domain->agaw = agaw;
1926
1927 if (ecap_coherent(iommu->ecap))
1928 domain->iommu_coherency = 1;
1929 else
1930 domain->iommu_coherency = 0;
1931
1932 if (ecap_sc_support(iommu->ecap))
1933 domain->iommu_snooping = 1;
1934 else
1935 domain->iommu_snooping = 0;
1936
1937 if (intel_iommu_superpage)
1938 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1939 else
1940 domain->iommu_superpage = 0;
1941
1942 domain->nid = iommu->node;
1943
1944 /* always allocate the top pgd */
1945 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1946 if (!domain->pgd)
1947 return -ENOMEM;
1948 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1949 return 0;
1950 }
1951
1952 static void domain_exit(struct dmar_domain *domain)
1953 {
1954 struct page *freelist = NULL;
1955
1956 /* Domain 0 is reserved, so dont process it */
1957 if (!domain)
1958 return;
1959
1960 /* Flush any lazy unmaps that may reference this domain */
1961 if (!intel_iommu_strict) {
1962 int cpu;
1963
1964 for_each_possible_cpu(cpu)
1965 flush_unmaps_timeout(cpu);
1966 }
1967
1968 /* Remove associated devices and clear attached or cached domains */
1969 rcu_read_lock();
1970 domain_remove_dev_info(domain);
1971 rcu_read_unlock();
1972
1973 /* destroy iovas */
1974 put_iova_domain(&domain->iovad);
1975
1976 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1977
1978 dma_free_pagelist(freelist);
1979
1980 free_domain_mem(domain);
1981 }
1982
1983 static int domain_context_mapping_one(struct dmar_domain *domain,
1984 struct intel_iommu *iommu,
1985 u8 bus, u8 devfn)
1986 {
1987 u16 did = domain->iommu_did[iommu->seq_id];
1988 int translation = CONTEXT_TT_MULTI_LEVEL;
1989 struct device_domain_info *info = NULL;
1990 struct context_entry *context;
1991 unsigned long flags;
1992 struct dma_pte *pgd;
1993 int ret, agaw;
1994
1995 WARN_ON(did == 0);
1996
1997 if (hw_pass_through && domain_type_is_si(domain))
1998 translation = CONTEXT_TT_PASS_THROUGH;
1999
2000 pr_debug("Set context mapping for %02x:%02x.%d\n",
2001 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2002
2003 BUG_ON(!domain->pgd);
2004
2005 spin_lock_irqsave(&device_domain_lock, flags);
2006 spin_lock(&iommu->lock);
2007
2008 ret = -ENOMEM;
2009 context = iommu_context_addr(iommu, bus, devfn, 1);
2010 if (!context)
2011 goto out_unlock;
2012
2013 ret = 0;
2014 if (context_present(context))
2015 goto out_unlock;
2016
2017 pgd = domain->pgd;
2018
2019 context_clear_entry(context);
2020 context_set_domain_id(context, did);
2021
2022 /*
2023 * Skip top levels of page tables for iommu which has less agaw
2024 * than default. Unnecessary for PT mode.
2025 */
2026 if (translation != CONTEXT_TT_PASS_THROUGH) {
2027 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
2028 ret = -ENOMEM;
2029 pgd = phys_to_virt(dma_pte_addr(pgd));
2030 if (!dma_pte_present(pgd))
2031 goto out_unlock;
2032 }
2033
2034 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2035 if (info && info->ats_supported)
2036 translation = CONTEXT_TT_DEV_IOTLB;
2037 else
2038 translation = CONTEXT_TT_MULTI_LEVEL;
2039
2040 context_set_address_root(context, virt_to_phys(pgd));
2041 context_set_address_width(context, iommu->agaw);
2042 } else {
2043 /*
2044 * In pass through mode, AW must be programmed to
2045 * indicate the largest AGAW value supported by
2046 * hardware. And ASR is ignored by hardware.
2047 */
2048 context_set_address_width(context, iommu->msagaw);
2049 }
2050
2051 context_set_translation_type(context, translation);
2052 context_set_fault_enable(context);
2053 context_set_present(context);
2054 domain_flush_cache(domain, context, sizeof(*context));
2055
2056 /*
2057 * It's a non-present to present mapping. If hardware doesn't cache
2058 * non-present entry we only need to flush the write-buffer. If the
2059 * _does_ cache non-present entries, then it does so in the special
2060 * domain #0, which we have to flush:
2061 */
2062 if (cap_caching_mode(iommu->cap)) {
2063 iommu->flush.flush_context(iommu, 0,
2064 (((u16)bus) << 8) | devfn,
2065 DMA_CCMD_MASK_NOBIT,
2066 DMA_CCMD_DEVICE_INVL);
2067 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2068 } else {
2069 iommu_flush_write_buffer(iommu);
2070 }
2071 iommu_enable_dev_iotlb(info);
2072
2073 ret = 0;
2074
2075 out_unlock:
2076 spin_unlock(&iommu->lock);
2077 spin_unlock_irqrestore(&device_domain_lock, flags);
2078
2079 return ret;
2080 }
2081
2082 struct domain_context_mapping_data {
2083 struct dmar_domain *domain;
2084 struct intel_iommu *iommu;
2085 };
2086
2087 static int domain_context_mapping_cb(struct pci_dev *pdev,
2088 u16 alias, void *opaque)
2089 {
2090 struct domain_context_mapping_data *data = opaque;
2091
2092 return domain_context_mapping_one(data->domain, data->iommu,
2093 PCI_BUS_NUM(alias), alias & 0xff);
2094 }
2095
2096 static int
2097 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2098 {
2099 struct intel_iommu *iommu;
2100 u8 bus, devfn;
2101 struct domain_context_mapping_data data;
2102
2103 iommu = device_to_iommu(dev, &bus, &devfn);
2104 if (!iommu)
2105 return -ENODEV;
2106
2107 if (!dev_is_pci(dev))
2108 return domain_context_mapping_one(domain, iommu, bus, devfn);
2109
2110 data.domain = domain;
2111 data.iommu = iommu;
2112
2113 return pci_for_each_dma_alias(to_pci_dev(dev),
2114 &domain_context_mapping_cb, &data);
2115 }
2116
2117 static int domain_context_mapped_cb(struct pci_dev *pdev,
2118 u16 alias, void *opaque)
2119 {
2120 struct intel_iommu *iommu = opaque;
2121
2122 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2123 }
2124
2125 static int domain_context_mapped(struct device *dev)
2126 {
2127 struct intel_iommu *iommu;
2128 u8 bus, devfn;
2129
2130 iommu = device_to_iommu(dev, &bus, &devfn);
2131 if (!iommu)
2132 return -ENODEV;
2133
2134 if (!dev_is_pci(dev))
2135 return device_context_mapped(iommu, bus, devfn);
2136
2137 return !pci_for_each_dma_alias(to_pci_dev(dev),
2138 domain_context_mapped_cb, iommu);
2139 }
2140
2141 /* Returns a number of VTD pages, but aligned to MM page size */
2142 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2143 size_t size)
2144 {
2145 host_addr &= ~PAGE_MASK;
2146 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2147 }
2148
2149 /* Return largest possible superpage level for a given mapping */
2150 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2151 unsigned long iov_pfn,
2152 unsigned long phy_pfn,
2153 unsigned long pages)
2154 {
2155 int support, level = 1;
2156 unsigned long pfnmerge;
2157
2158 support = domain->iommu_superpage;
2159
2160 /* To use a large page, the virtual *and* physical addresses
2161 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2162 of them will mean we have to use smaller pages. So just
2163 merge them and check both at once. */
2164 pfnmerge = iov_pfn | phy_pfn;
2165
2166 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2167 pages >>= VTD_STRIDE_SHIFT;
2168 if (!pages)
2169 break;
2170 pfnmerge >>= VTD_STRIDE_SHIFT;
2171 level++;
2172 support--;
2173 }
2174 return level;
2175 }
2176
2177 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2178 struct scatterlist *sg, unsigned long phys_pfn,
2179 unsigned long nr_pages, int prot)
2180 {
2181 struct dma_pte *first_pte = NULL, *pte = NULL;
2182 phys_addr_t uninitialized_var(pteval);
2183 unsigned long sg_res = 0;
2184 unsigned int largepage_lvl = 0;
2185 unsigned long lvl_pages = 0;
2186
2187 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2188
2189 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2190 return -EINVAL;
2191
2192 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2193
2194 if (!sg) {
2195 sg_res = nr_pages;
2196 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2197 }
2198
2199 while (nr_pages > 0) {
2200 uint64_t tmp;
2201
2202 if (!sg_res) {
2203 sg_res = aligned_nrpages(sg->offset, sg->length);
2204 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2205 sg->dma_length = sg->length;
2206 pteval = page_to_phys(sg_page(sg)) | prot;
2207 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2208 }
2209
2210 if (!pte) {
2211 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2212
2213 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2214 if (!pte)
2215 return -ENOMEM;
2216 /* It is large page*/
2217 if (largepage_lvl > 1) {
2218 unsigned long nr_superpages, end_pfn;
2219
2220 pteval |= DMA_PTE_LARGE_PAGE;
2221 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2222
2223 nr_superpages = sg_res / lvl_pages;
2224 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2225
2226 /*
2227 * Ensure that old small page tables are
2228 * removed to make room for superpage(s).
2229 */
2230 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
2231 } else {
2232 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2233 }
2234
2235 }
2236 /* We don't need lock here, nobody else
2237 * touches the iova range
2238 */
2239 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2240 if (tmp) {
2241 static int dumps = 5;
2242 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2243 iov_pfn, tmp, (unsigned long long)pteval);
2244 if (dumps) {
2245 dumps--;
2246 debug_dma_dump_mappings(NULL);
2247 }
2248 WARN_ON(1);
2249 }
2250
2251 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2252
2253 BUG_ON(nr_pages < lvl_pages);
2254 BUG_ON(sg_res < lvl_pages);
2255
2256 nr_pages -= lvl_pages;
2257 iov_pfn += lvl_pages;
2258 phys_pfn += lvl_pages;
2259 pteval += lvl_pages * VTD_PAGE_SIZE;
2260 sg_res -= lvl_pages;
2261
2262 /* If the next PTE would be the first in a new page, then we
2263 need to flush the cache on the entries we've just written.
2264 And then we'll need to recalculate 'pte', so clear it and
2265 let it get set again in the if (!pte) block above.
2266
2267 If we're done (!nr_pages) we need to flush the cache too.
2268
2269 Also if we've been setting superpages, we may need to
2270 recalculate 'pte' and switch back to smaller pages for the
2271 end of the mapping, if the trailing size is not enough to
2272 use another superpage (i.e. sg_res < lvl_pages). */
2273 pte++;
2274 if (!nr_pages || first_pte_in_page(pte) ||
2275 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2276 domain_flush_cache(domain, first_pte,
2277 (void *)pte - (void *)first_pte);
2278 pte = NULL;
2279 }
2280
2281 if (!sg_res && nr_pages)
2282 sg = sg_next(sg);
2283 }
2284 return 0;
2285 }
2286
2287 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2288 struct scatterlist *sg, unsigned long nr_pages,
2289 int prot)
2290 {
2291 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2292 }
2293
2294 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2295 unsigned long phys_pfn, unsigned long nr_pages,
2296 int prot)
2297 {
2298 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2299 }
2300
2301 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2302 {
2303 if (!iommu)
2304 return;
2305
2306 clear_context_table(iommu, bus, devfn);
2307 iommu->flush.flush_context(iommu, 0, 0, 0,
2308 DMA_CCMD_GLOBAL_INVL);
2309 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2310 }
2311
2312 static inline void unlink_domain_info(struct device_domain_info *info)
2313 {
2314 assert_spin_locked(&device_domain_lock);
2315 list_del(&info->link);
2316 list_del(&info->global);
2317 if (info->dev)
2318 info->dev->archdata.iommu = NULL;
2319 }
2320
2321 static void domain_remove_dev_info(struct dmar_domain *domain)
2322 {
2323 struct device_domain_info *info, *tmp;
2324 unsigned long flags;
2325
2326 spin_lock_irqsave(&device_domain_lock, flags);
2327 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2328 __dmar_remove_one_dev_info(info);
2329 spin_unlock_irqrestore(&device_domain_lock, flags);
2330 }
2331
2332 /*
2333 * find_domain
2334 * Note: we use struct device->archdata.iommu stores the info
2335 */
2336 static struct dmar_domain *find_domain(struct device *dev)
2337 {
2338 struct device_domain_info *info;
2339
2340 /* No lock here, assumes no domain exit in normal case */
2341 info = dev->archdata.iommu;
2342 if (info)
2343 return info->domain;
2344 return NULL;
2345 }
2346
2347 static inline struct device_domain_info *
2348 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2349 {
2350 struct device_domain_info *info;
2351
2352 list_for_each_entry(info, &device_domain_list, global)
2353 if (info->iommu->segment == segment && info->bus == bus &&
2354 info->devfn == devfn)
2355 return info;
2356
2357 return NULL;
2358 }
2359
2360 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2361 int bus, int devfn,
2362 struct device *dev,
2363 struct dmar_domain *domain)
2364 {
2365 struct dmar_domain *found = NULL;
2366 struct device_domain_info *info;
2367 unsigned long flags;
2368 int ret;
2369
2370 info = alloc_devinfo_mem();
2371 if (!info)
2372 return NULL;
2373
2374 info->bus = bus;
2375 info->devfn = devfn;
2376 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2377 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2378 info->ats_qdep = 0;
2379 info->dev = dev;
2380 info->domain = domain;
2381 info->iommu = iommu;
2382
2383 if (dev && dev_is_pci(dev)) {
2384 struct pci_dev *pdev = to_pci_dev(info->dev);
2385
2386 if (ecap_dev_iotlb_support(iommu->ecap) &&
2387 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2388 dmar_find_matched_atsr_unit(pdev))
2389 info->ats_supported = 1;
2390
2391 if (ecs_enabled(iommu)) {
2392 if (pasid_enabled(iommu)) {
2393 int features = pci_pasid_features(pdev);
2394 if (features >= 0)
2395 info->pasid_supported = features | 1;
2396 }
2397
2398 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2399 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2400 info->pri_supported = 1;
2401 }
2402 }
2403
2404 spin_lock_irqsave(&device_domain_lock, flags);
2405 if (dev)
2406 found = find_domain(dev);
2407
2408 if (!found) {
2409 struct device_domain_info *info2;
2410 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2411 if (info2) {
2412 found = info2->domain;
2413 info2->dev = dev;
2414 }
2415 }
2416
2417 if (found) {
2418 spin_unlock_irqrestore(&device_domain_lock, flags);
2419 free_devinfo_mem(info);
2420 /* Caller must free the original domain */
2421 return found;
2422 }
2423
2424 spin_lock(&iommu->lock);
2425 ret = domain_attach_iommu(domain, iommu);
2426 spin_unlock(&iommu->lock);
2427
2428 if (ret) {
2429 spin_unlock_irqrestore(&device_domain_lock, flags);
2430 free_devinfo_mem(info);
2431 return NULL;
2432 }
2433
2434 list_add(&info->link, &domain->devices);
2435 list_add(&info->global, &device_domain_list);
2436 if (dev)
2437 dev->archdata.iommu = info;
2438 spin_unlock_irqrestore(&device_domain_lock, flags);
2439
2440 if (dev && domain_context_mapping(domain, dev)) {
2441 pr_err("Domain context map for %s failed\n", dev_name(dev));
2442 dmar_remove_one_dev_info(domain, dev);
2443 return NULL;
2444 }
2445
2446 return domain;
2447 }
2448
2449 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2450 {
2451 *(u16 *)opaque = alias;
2452 return 0;
2453 }
2454
2455 /* domain is initialized */
2456 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2457 {
2458 struct device_domain_info *info = NULL;
2459 struct dmar_domain *domain, *tmp;
2460 struct intel_iommu *iommu;
2461 u16 req_id, dma_alias;
2462 unsigned long flags;
2463 u8 bus, devfn;
2464
2465 domain = find_domain(dev);
2466 if (domain)
2467 return domain;
2468
2469 iommu = device_to_iommu(dev, &bus, &devfn);
2470 if (!iommu)
2471 return NULL;
2472
2473 req_id = ((u16)bus << 8) | devfn;
2474
2475 if (dev_is_pci(dev)) {
2476 struct pci_dev *pdev = to_pci_dev(dev);
2477
2478 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2479
2480 spin_lock_irqsave(&device_domain_lock, flags);
2481 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2482 PCI_BUS_NUM(dma_alias),
2483 dma_alias & 0xff);
2484 if (info) {
2485 iommu = info->iommu;
2486 domain = info->domain;
2487 }
2488 spin_unlock_irqrestore(&device_domain_lock, flags);
2489
2490 /* DMA alias already has a domain, uses it */
2491 if (info)
2492 goto found_domain;
2493 }
2494
2495 /* Allocate and initialize new domain for the device */
2496 domain = alloc_domain(0);
2497 if (!domain)
2498 return NULL;
2499 if (domain_init(domain, iommu, gaw)) {
2500 domain_exit(domain);
2501 return NULL;
2502 }
2503
2504 /* register PCI DMA alias device */
2505 if (dev_is_pci(dev) && req_id != dma_alias) {
2506 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2507 dma_alias & 0xff, NULL, domain);
2508
2509 if (!tmp || tmp != domain) {
2510 domain_exit(domain);
2511 domain = tmp;
2512 }
2513
2514 if (!domain)
2515 return NULL;
2516 }
2517
2518 found_domain:
2519 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2520
2521 if (!tmp || tmp != domain) {
2522 domain_exit(domain);
2523 domain = tmp;
2524 }
2525
2526 return domain;
2527 }
2528
2529 static int iommu_domain_identity_map(struct dmar_domain *domain,
2530 unsigned long long start,
2531 unsigned long long end)
2532 {
2533 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2534 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2535
2536 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2537 dma_to_mm_pfn(last_vpfn))) {
2538 pr_err("Reserving iova failed\n");
2539 return -ENOMEM;
2540 }
2541
2542 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2543 /*
2544 * RMRR range might have overlap with physical memory range,
2545 * clear it first
2546 */
2547 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2548
2549 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2550 last_vpfn - first_vpfn + 1,
2551 DMA_PTE_READ|DMA_PTE_WRITE);
2552 }
2553
2554 static int domain_prepare_identity_map(struct device *dev,
2555 struct dmar_domain *domain,
2556 unsigned long long start,
2557 unsigned long long end)
2558 {
2559 /* For _hardware_ passthrough, don't bother. But for software
2560 passthrough, we do it anyway -- it may indicate a memory
2561 range which is reserved in E820, so which didn't get set
2562 up to start with in si_domain */
2563 if (domain == si_domain && hw_pass_through) {
2564 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2565 dev_name(dev), start, end);
2566 return 0;
2567 }
2568
2569 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2570 dev_name(dev), start, end);
2571
2572 if (end < start) {
2573 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2574 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2575 dmi_get_system_info(DMI_BIOS_VENDOR),
2576 dmi_get_system_info(DMI_BIOS_VERSION),
2577 dmi_get_system_info(DMI_PRODUCT_VERSION));
2578 return -EIO;
2579 }
2580
2581 if (end >> agaw_to_width(domain->agaw)) {
2582 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2583 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2584 agaw_to_width(domain->agaw),
2585 dmi_get_system_info(DMI_BIOS_VENDOR),
2586 dmi_get_system_info(DMI_BIOS_VERSION),
2587 dmi_get_system_info(DMI_PRODUCT_VERSION));
2588 return -EIO;
2589 }
2590
2591 return iommu_domain_identity_map(domain, start, end);
2592 }
2593
2594 static int iommu_prepare_identity_map(struct device *dev,
2595 unsigned long long start,
2596 unsigned long long end)
2597 {
2598 struct dmar_domain *domain;
2599 int ret;
2600
2601 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2602 if (!domain)
2603 return -ENOMEM;
2604
2605 ret = domain_prepare_identity_map(dev, domain, start, end);
2606 if (ret)
2607 domain_exit(domain);
2608
2609 return ret;
2610 }
2611
2612 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2613 struct device *dev)
2614 {
2615 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2616 return 0;
2617 return iommu_prepare_identity_map(dev, rmrr->base_address,
2618 rmrr->end_address);
2619 }
2620
2621 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2622 static inline void iommu_prepare_isa(void)
2623 {
2624 struct pci_dev *pdev;
2625 int ret;
2626
2627 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2628 if (!pdev)
2629 return;
2630
2631 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2632 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2633
2634 if (ret)
2635 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2636
2637 pci_dev_put(pdev);
2638 }
2639 #else
2640 static inline void iommu_prepare_isa(void)
2641 {
2642 return;
2643 }
2644 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2645
2646 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2647
2648 static int __init si_domain_init(int hw)
2649 {
2650 int nid, ret = 0;
2651
2652 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2653 if (!si_domain)
2654 return -EFAULT;
2655
2656 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2657 domain_exit(si_domain);
2658 return -EFAULT;
2659 }
2660
2661 pr_debug("Identity mapping domain allocated\n");
2662
2663 if (hw)
2664 return 0;
2665
2666 for_each_online_node(nid) {
2667 unsigned long start_pfn, end_pfn;
2668 int i;
2669
2670 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2671 ret = iommu_domain_identity_map(si_domain,
2672 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2673 if (ret)
2674 return ret;
2675 }
2676 }
2677
2678 return 0;
2679 }
2680
2681 static int identity_mapping(struct device *dev)
2682 {
2683 struct device_domain_info *info;
2684
2685 if (likely(!iommu_identity_mapping))
2686 return 0;
2687
2688 info = dev->archdata.iommu;
2689 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2690 return (info->domain == si_domain);
2691
2692 return 0;
2693 }
2694
2695 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2696 {
2697 struct dmar_domain *ndomain;
2698 struct intel_iommu *iommu;
2699 u8 bus, devfn;
2700
2701 iommu = device_to_iommu(dev, &bus, &devfn);
2702 if (!iommu)
2703 return -ENODEV;
2704
2705 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2706 if (ndomain != domain)
2707 return -EBUSY;
2708
2709 return 0;
2710 }
2711
2712 static bool device_has_rmrr(struct device *dev)
2713 {
2714 struct dmar_rmrr_unit *rmrr;
2715 struct device *tmp;
2716 int i;
2717
2718 rcu_read_lock();
2719 for_each_rmrr_units(rmrr) {
2720 /*
2721 * Return TRUE if this RMRR contains the device that
2722 * is passed in.
2723 */
2724 for_each_active_dev_scope(rmrr->devices,
2725 rmrr->devices_cnt, i, tmp)
2726 if (tmp == dev) {
2727 rcu_read_unlock();
2728 return true;
2729 }
2730 }
2731 rcu_read_unlock();
2732 return false;
2733 }
2734
2735 /*
2736 * There are a couple cases where we need to restrict the functionality of
2737 * devices associated with RMRRs. The first is when evaluating a device for
2738 * identity mapping because problems exist when devices are moved in and out
2739 * of domains and their respective RMRR information is lost. This means that
2740 * a device with associated RMRRs will never be in a "passthrough" domain.
2741 * The second is use of the device through the IOMMU API. This interface
2742 * expects to have full control of the IOVA space for the device. We cannot
2743 * satisfy both the requirement that RMRR access is maintained and have an
2744 * unencumbered IOVA space. We also have no ability to quiesce the device's
2745 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2746 * We therefore prevent devices associated with an RMRR from participating in
2747 * the IOMMU API, which eliminates them from device assignment.
2748 *
2749 * In both cases we assume that PCI USB devices with RMRRs have them largely
2750 * for historical reasons and that the RMRR space is not actively used post
2751 * boot. This exclusion may change if vendors begin to abuse it.
2752 *
2753 * The same exception is made for graphics devices, with the requirement that
2754 * any use of the RMRR regions will be torn down before assigning the device
2755 * to a guest.
2756 */
2757 static bool device_is_rmrr_locked(struct device *dev)
2758 {
2759 if (!device_has_rmrr(dev))
2760 return false;
2761
2762 if (dev_is_pci(dev)) {
2763 struct pci_dev *pdev = to_pci_dev(dev);
2764
2765 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2766 return false;
2767 }
2768
2769 return true;
2770 }
2771
2772 static int iommu_should_identity_map(struct device *dev, int startup)
2773 {
2774
2775 if (dev_is_pci(dev)) {
2776 struct pci_dev *pdev = to_pci_dev(dev);
2777
2778 if (device_is_rmrr_locked(dev))
2779 return 0;
2780
2781 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2782 return 1;
2783
2784 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2785 return 1;
2786
2787 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2788 return 0;
2789
2790 /*
2791 * We want to start off with all devices in the 1:1 domain, and
2792 * take them out later if we find they can't access all of memory.
2793 *
2794 * However, we can't do this for PCI devices behind bridges,
2795 * because all PCI devices behind the same bridge will end up
2796 * with the same source-id on their transactions.
2797 *
2798 * Practically speaking, we can't change things around for these
2799 * devices at run-time, because we can't be sure there'll be no
2800 * DMA transactions in flight for any of their siblings.
2801 *
2802 * So PCI devices (unless they're on the root bus) as well as
2803 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2804 * the 1:1 domain, just in _case_ one of their siblings turns out
2805 * not to be able to map all of memory.
2806 */
2807 if (!pci_is_pcie(pdev)) {
2808 if (!pci_is_root_bus(pdev->bus))
2809 return 0;
2810 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2811 return 0;
2812 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2813 return 0;
2814 } else {
2815 if (device_has_rmrr(dev))
2816 return 0;
2817 }
2818
2819 /*
2820 * At boot time, we don't yet know if devices will be 64-bit capable.
2821 * Assume that they will — if they turn out not to be, then we can
2822 * take them out of the 1:1 domain later.
2823 */
2824 if (!startup) {
2825 /*
2826 * If the device's dma_mask is less than the system's memory
2827 * size then this is not a candidate for identity mapping.
2828 */
2829 u64 dma_mask = *dev->dma_mask;
2830
2831 if (dev->coherent_dma_mask &&
2832 dev->coherent_dma_mask < dma_mask)
2833 dma_mask = dev->coherent_dma_mask;
2834
2835 return dma_mask >= dma_get_required_mask(dev);
2836 }
2837
2838 return 1;
2839 }
2840
2841 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2842 {
2843 int ret;
2844
2845 if (!iommu_should_identity_map(dev, 1))
2846 return 0;
2847
2848 ret = domain_add_dev_info(si_domain, dev);
2849 if (!ret)
2850 pr_info("%s identity mapping for device %s\n",
2851 hw ? "Hardware" : "Software", dev_name(dev));
2852 else if (ret == -ENODEV)
2853 /* device not associated with an iommu */
2854 ret = 0;
2855
2856 return ret;
2857 }
2858
2859
2860 static int __init iommu_prepare_static_identity_mapping(int hw)
2861 {
2862 struct pci_dev *pdev = NULL;
2863 struct dmar_drhd_unit *drhd;
2864 struct intel_iommu *iommu;
2865 struct device *dev;
2866 int i;
2867 int ret = 0;
2868
2869 for_each_pci_dev(pdev) {
2870 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2871 if (ret)
2872 return ret;
2873 }
2874
2875 for_each_active_iommu(iommu, drhd)
2876 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2877 struct acpi_device_physical_node *pn;
2878 struct acpi_device *adev;
2879
2880 if (dev->bus != &acpi_bus_type)
2881 continue;
2882
2883 adev= to_acpi_device(dev);
2884 mutex_lock(&adev->physical_node_lock);
2885 list_for_each_entry(pn, &adev->physical_node_list, node) {
2886 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2887 if (ret)
2888 break;
2889 }
2890 mutex_unlock(&adev->physical_node_lock);
2891 if (ret)
2892 return ret;
2893 }
2894
2895 return 0;
2896 }
2897
2898 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2899 {
2900 /*
2901 * Start from the sane iommu hardware state.
2902 * If the queued invalidation is already initialized by us
2903 * (for example, while enabling interrupt-remapping) then
2904 * we got the things already rolling from a sane state.
2905 */
2906 if (!iommu->qi) {
2907 /*
2908 * Clear any previous faults.
2909 */
2910 dmar_fault(-1, iommu);
2911 /*
2912 * Disable queued invalidation if supported and already enabled
2913 * before OS handover.
2914 */
2915 dmar_disable_qi(iommu);
2916 }
2917
2918 if (dmar_enable_qi(iommu)) {
2919 /*
2920 * Queued Invalidate not enabled, use Register Based Invalidate
2921 */
2922 iommu->flush.flush_context = __iommu_flush_context;
2923 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2924 pr_info("%s: Using Register based invalidation\n",
2925 iommu->name);
2926 } else {
2927 iommu->flush.flush_context = qi_flush_context;
2928 iommu->flush.flush_iotlb = qi_flush_iotlb;
2929 pr_info("%s: Using Queued invalidation\n", iommu->name);
2930 }
2931 }
2932
2933 static int copy_context_table(struct intel_iommu *iommu,
2934 struct root_entry *old_re,
2935 struct context_entry **tbl,
2936 int bus, bool ext)
2937 {
2938 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2939 struct context_entry *new_ce = NULL, ce;
2940 struct context_entry *old_ce = NULL;
2941 struct root_entry re;
2942 phys_addr_t old_ce_phys;
2943
2944 tbl_idx = ext ? bus * 2 : bus;
2945 memcpy(&re, old_re, sizeof(re));
2946
2947 for (devfn = 0; devfn < 256; devfn++) {
2948 /* First calculate the correct index */
2949 idx = (ext ? devfn * 2 : devfn) % 256;
2950
2951 if (idx == 0) {
2952 /* First save what we may have and clean up */
2953 if (new_ce) {
2954 tbl[tbl_idx] = new_ce;
2955 __iommu_flush_cache(iommu, new_ce,
2956 VTD_PAGE_SIZE);
2957 pos = 1;
2958 }
2959
2960 if (old_ce)
2961 iounmap(old_ce);
2962
2963 ret = 0;
2964 if (devfn < 0x80)
2965 old_ce_phys = root_entry_lctp(&re);
2966 else
2967 old_ce_phys = root_entry_uctp(&re);
2968
2969 if (!old_ce_phys) {
2970 if (ext && devfn == 0) {
2971 /* No LCTP, try UCTP */
2972 devfn = 0x7f;
2973 continue;
2974 } else {
2975 goto out;
2976 }
2977 }
2978
2979 ret = -ENOMEM;
2980 old_ce = memremap(old_ce_phys, PAGE_SIZE,
2981 MEMREMAP_WB);
2982 if (!old_ce)
2983 goto out;
2984
2985 new_ce = alloc_pgtable_page(iommu->node);
2986 if (!new_ce)
2987 goto out_unmap;
2988
2989 ret = 0;
2990 }
2991
2992 /* Now copy the context entry */
2993 memcpy(&ce, old_ce + idx, sizeof(ce));
2994
2995 if (!__context_present(&ce))
2996 continue;
2997
2998 did = context_domain_id(&ce);
2999 if (did >= 0 && did < cap_ndoms(iommu->cap))
3000 set_bit(did, iommu->domain_ids);
3001
3002 /*
3003 * We need a marker for copied context entries. This
3004 * marker needs to work for the old format as well as
3005 * for extended context entries.
3006 *
3007 * Bit 67 of the context entry is used. In the old
3008 * format this bit is available to software, in the
3009 * extended format it is the PGE bit, but PGE is ignored
3010 * by HW if PASIDs are disabled (and thus still
3011 * available).
3012 *
3013 * So disable PASIDs first and then mark the entry
3014 * copied. This means that we don't copy PASID
3015 * translations from the old kernel, but this is fine as
3016 * faults there are not fatal.
3017 */
3018 context_clear_pasid_enable(&ce);
3019 context_set_copied(&ce);
3020
3021 new_ce[idx] = ce;
3022 }
3023
3024 tbl[tbl_idx + pos] = new_ce;
3025
3026 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3027
3028 out_unmap:
3029 memunmap(old_ce);
3030
3031 out:
3032 return ret;
3033 }
3034
3035 static int copy_translation_tables(struct intel_iommu *iommu)
3036 {
3037 struct context_entry **ctxt_tbls;
3038 struct root_entry *old_rt;
3039 phys_addr_t old_rt_phys;
3040 int ctxt_table_entries;
3041 unsigned long flags;
3042 u64 rtaddr_reg;
3043 int bus, ret;
3044 bool new_ext, ext;
3045
3046 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3047 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3048 new_ext = !!ecap_ecs(iommu->ecap);
3049
3050 /*
3051 * The RTT bit can only be changed when translation is disabled,
3052 * but disabling translation means to open a window for data
3053 * corruption. So bail out and don't copy anything if we would
3054 * have to change the bit.
3055 */
3056 if (new_ext != ext)
3057 return -EINVAL;
3058
3059 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3060 if (!old_rt_phys)
3061 return -EINVAL;
3062
3063 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3064 if (!old_rt)
3065 return -ENOMEM;
3066
3067 /* This is too big for the stack - allocate it from slab */
3068 ctxt_table_entries = ext ? 512 : 256;
3069 ret = -ENOMEM;
3070 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3071 if (!ctxt_tbls)
3072 goto out_unmap;
3073
3074 for (bus = 0; bus < 256; bus++) {
3075 ret = copy_context_table(iommu, &old_rt[bus],
3076 ctxt_tbls, bus, ext);
3077 if (ret) {
3078 pr_err("%s: Failed to copy context table for bus %d\n",
3079 iommu->name, bus);
3080 continue;
3081 }
3082 }
3083
3084 spin_lock_irqsave(&iommu->lock, flags);
3085
3086 /* Context tables are copied, now write them to the root_entry table */
3087 for (bus = 0; bus < 256; bus++) {
3088 int idx = ext ? bus * 2 : bus;
3089 u64 val;
3090
3091 if (ctxt_tbls[idx]) {
3092 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3093 iommu->root_entry[bus].lo = val;
3094 }
3095
3096 if (!ext || !ctxt_tbls[idx + 1])
3097 continue;
3098
3099 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3100 iommu->root_entry[bus].hi = val;
3101 }
3102
3103 spin_unlock_irqrestore(&iommu->lock, flags);
3104
3105 kfree(ctxt_tbls);
3106
3107 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3108
3109 ret = 0;
3110
3111 out_unmap:
3112 memunmap(old_rt);
3113
3114 return ret;
3115 }
3116
3117 static int __init init_dmars(void)
3118 {
3119 struct dmar_drhd_unit *drhd;
3120 struct dmar_rmrr_unit *rmrr;
3121 bool copied_tables = false;
3122 struct device *dev;
3123 struct intel_iommu *iommu;
3124 int i, ret, cpu;
3125
3126 /*
3127 * for each drhd
3128 * allocate root
3129 * initialize and program root entry to not present
3130 * endfor
3131 */
3132 for_each_drhd_unit(drhd) {
3133 /*
3134 * lock not needed as this is only incremented in the single
3135 * threaded kernel __init code path all other access are read
3136 * only
3137 */
3138 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3139 g_num_of_iommus++;
3140 continue;
3141 }
3142 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3143 }
3144
3145 /* Preallocate enough resources for IOMMU hot-addition */
3146 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3147 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3148
3149 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3150 GFP_KERNEL);
3151 if (!g_iommus) {
3152 pr_err("Allocating global iommu array failed\n");
3153 ret = -ENOMEM;
3154 goto error;
3155 }
3156
3157 for_each_possible_cpu(cpu) {
3158 struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
3159 cpu);
3160
3161 dfd->tables = kzalloc(g_num_of_iommus *
3162 sizeof(struct deferred_flush_table),
3163 GFP_KERNEL);
3164 if (!dfd->tables) {
3165 ret = -ENOMEM;
3166 goto free_g_iommus;
3167 }
3168
3169 spin_lock_init(&dfd->lock);
3170 setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
3171 }
3172
3173 for_each_active_iommu(iommu, drhd) {
3174 g_iommus[iommu->seq_id] = iommu;
3175
3176 intel_iommu_init_qi(iommu);
3177
3178 ret = iommu_init_domains(iommu);
3179 if (ret)
3180 goto free_iommu;
3181
3182 init_translation_status(iommu);
3183
3184 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3185 iommu_disable_translation(iommu);
3186 clear_translation_pre_enabled(iommu);
3187 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3188 iommu->name);
3189 }
3190
3191 /*
3192 * TBD:
3193 * we could share the same root & context tables
3194 * among all IOMMU's. Need to Split it later.
3195 */
3196 ret = iommu_alloc_root_entry(iommu);
3197 if (ret)
3198 goto free_iommu;
3199
3200 if (translation_pre_enabled(iommu)) {
3201 pr_info("Translation already enabled - trying to copy translation structures\n");
3202
3203 ret = copy_translation_tables(iommu);
3204 if (ret) {
3205 /*
3206 * We found the IOMMU with translation
3207 * enabled - but failed to copy over the
3208 * old root-entry table. Try to proceed
3209 * by disabling translation now and
3210 * allocating a clean root-entry table.
3211 * This might cause DMAR faults, but
3212 * probably the dump will still succeed.
3213 */
3214 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3215 iommu->name);
3216 iommu_disable_translation(iommu);
3217 clear_translation_pre_enabled(iommu);
3218 } else {
3219 pr_info("Copied translation tables from previous kernel for %s\n",
3220 iommu->name);
3221 copied_tables = true;
3222 }
3223 }
3224
3225 if (!ecap_pass_through(iommu->ecap))
3226 hw_pass_through = 0;
3227 #ifdef CONFIG_INTEL_IOMMU_SVM
3228 if (pasid_enabled(iommu))
3229 intel_svm_alloc_pasid_tables(iommu);
3230 #endif
3231 }
3232
3233 /*
3234 * Now that qi is enabled on all iommus, set the root entry and flush
3235 * caches. This is required on some Intel X58 chipsets, otherwise the
3236 * flush_context function will loop forever and the boot hangs.
3237 */
3238 for_each_active_iommu(iommu, drhd) {
3239 iommu_flush_write_buffer(iommu);
3240 iommu_set_root_entry(iommu);
3241 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3242 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3243 }
3244
3245 if (iommu_pass_through)
3246 iommu_identity_mapping |= IDENTMAP_ALL;
3247
3248 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3249 iommu_identity_mapping |= IDENTMAP_GFX;
3250 #endif
3251
3252 if (iommu_identity_mapping) {
3253 ret = si_domain_init(hw_pass_through);
3254 if (ret)
3255 goto free_iommu;
3256 }
3257
3258 check_tylersburg_isoch();
3259
3260 /*
3261 * If we copied translations from a previous kernel in the kdump
3262 * case, we can not assign the devices to domains now, as that
3263 * would eliminate the old mappings. So skip this part and defer
3264 * the assignment to device driver initialization time.
3265 */
3266 if (copied_tables)
3267 goto domains_done;
3268
3269 /*
3270 * If pass through is not set or not enabled, setup context entries for
3271 * identity mappings for rmrr, gfx, and isa and may fall back to static
3272 * identity mapping if iommu_identity_mapping is set.
3273 */
3274 if (iommu_identity_mapping) {
3275 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3276 if (ret) {
3277 pr_crit("Failed to setup IOMMU pass-through\n");
3278 goto free_iommu;
3279 }
3280 }
3281 /*
3282 * For each rmrr
3283 * for each dev attached to rmrr
3284 * do
3285 * locate drhd for dev, alloc domain for dev
3286 * allocate free domain
3287 * allocate page table entries for rmrr
3288 * if context not allocated for bus
3289 * allocate and init context
3290 * set present in root table for this bus
3291 * init context with domain, translation etc
3292 * endfor
3293 * endfor
3294 */
3295 pr_info("Setting RMRR:\n");
3296 for_each_rmrr_units(rmrr) {
3297 /* some BIOS lists non-exist devices in DMAR table. */
3298 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3299 i, dev) {
3300 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3301 if (ret)
3302 pr_err("Mapping reserved region failed\n");
3303 }
3304 }
3305
3306 iommu_prepare_isa();
3307
3308 domains_done:
3309
3310 /*
3311 * for each drhd
3312 * enable fault log
3313 * global invalidate context cache
3314 * global invalidate iotlb
3315 * enable translation
3316 */
3317 for_each_iommu(iommu, drhd) {
3318 if (drhd->ignored) {
3319 /*
3320 * we always have to disable PMRs or DMA may fail on
3321 * this device
3322 */
3323 if (force_on)
3324 iommu_disable_protect_mem_regions(iommu);
3325 continue;
3326 }
3327
3328 iommu_flush_write_buffer(iommu);
3329
3330 #ifdef CONFIG_INTEL_IOMMU_SVM
3331 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3332 ret = intel_svm_enable_prq(iommu);
3333 if (ret)
3334 goto free_iommu;
3335 }
3336 #endif
3337 ret = dmar_set_interrupt(iommu);
3338 if (ret)
3339 goto free_iommu;
3340
3341 if (!translation_pre_enabled(iommu))
3342 iommu_enable_translation(iommu);
3343
3344 iommu_disable_protect_mem_regions(iommu);
3345 }
3346
3347 return 0;
3348
3349 free_iommu:
3350 for_each_active_iommu(iommu, drhd) {
3351 disable_dmar_iommu(iommu);
3352 free_dmar_iommu(iommu);
3353 }
3354 free_g_iommus:
3355 for_each_possible_cpu(cpu)
3356 kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
3357 kfree(g_iommus);
3358 error:
3359 return ret;
3360 }
3361
3362 /* This takes a number of _MM_ pages, not VTD pages */
3363 static unsigned long intel_alloc_iova(struct device *dev,
3364 struct dmar_domain *domain,
3365 unsigned long nrpages, uint64_t dma_mask)
3366 {
3367 unsigned long iova_pfn = 0;
3368
3369 /* Restrict dma_mask to the width that the iommu can handle */
3370 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3371 /* Ensure we reserve the whole size-aligned region */
3372 nrpages = __roundup_pow_of_two(nrpages);
3373
3374 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3375 /*
3376 * First try to allocate an io virtual address in
3377 * DMA_BIT_MASK(32) and if that fails then try allocating
3378 * from higher range
3379 */
3380 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3381 IOVA_PFN(DMA_BIT_MASK(32)));
3382 if (iova_pfn)
3383 return iova_pfn;
3384 }
3385 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask));
3386 if (unlikely(!iova_pfn)) {
3387 pr_err("Allocating %ld-page iova for %s failed",
3388 nrpages, dev_name(dev));
3389 return 0;
3390 }
3391
3392 return iova_pfn;
3393 }
3394
3395 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3396 {
3397 struct dmar_rmrr_unit *rmrr;
3398 struct dmar_domain *domain;
3399 struct device *i_dev;
3400 int i, ret;
3401
3402 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3403 if (!domain) {
3404 pr_err("Allocating domain for %s failed\n",
3405 dev_name(dev));
3406 return NULL;
3407 }
3408
3409 /* We have a new domain - setup possible RMRRs for the device */
3410 rcu_read_lock();
3411 for_each_rmrr_units(rmrr) {
3412 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3413 i, i_dev) {
3414 if (i_dev != dev)
3415 continue;
3416
3417 ret = domain_prepare_identity_map(dev, domain,
3418 rmrr->base_address,
3419 rmrr->end_address);
3420 if (ret)
3421 dev_err(dev, "Mapping reserved region failed\n");
3422 }
3423 }
3424 rcu_read_unlock();
3425
3426 return domain;
3427 }
3428
3429 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3430 {
3431 struct device_domain_info *info;
3432
3433 /* No lock here, assumes no domain exit in normal case */
3434 info = dev->archdata.iommu;
3435 if (likely(info))
3436 return info->domain;
3437
3438 return __get_valid_domain_for_dev(dev);
3439 }
3440
3441 /* Check if the dev needs to go through non-identity map and unmap process.*/
3442 static int iommu_no_mapping(struct device *dev)
3443 {
3444 int found;
3445
3446 if (iommu_dummy(dev))
3447 return 1;
3448
3449 if (!iommu_identity_mapping)
3450 return 0;
3451
3452 found = identity_mapping(dev);
3453 if (found) {
3454 if (iommu_should_identity_map(dev, 0))
3455 return 1;
3456 else {
3457 /*
3458 * 32 bit DMA is removed from si_domain and fall back
3459 * to non-identity mapping.
3460 */
3461 dmar_remove_one_dev_info(si_domain, dev);
3462 pr_info("32bit %s uses non-identity mapping\n",
3463 dev_name(dev));
3464 return 0;
3465 }
3466 } else {
3467 /*
3468 * In case of a detached 64 bit DMA device from vm, the device
3469 * is put into si_domain for identity mapping.
3470 */
3471 if (iommu_should_identity_map(dev, 0)) {
3472 int ret;
3473 ret = domain_add_dev_info(si_domain, dev);
3474 if (!ret) {
3475 pr_info("64bit %s uses identity mapping\n",
3476 dev_name(dev));
3477 return 1;
3478 }
3479 }
3480 }
3481
3482 return 0;
3483 }
3484
3485 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3486 size_t size, int dir, u64 dma_mask)
3487 {
3488 struct dmar_domain *domain;
3489 phys_addr_t start_paddr;
3490 unsigned long iova_pfn;
3491 int prot = 0;
3492 int ret;
3493 struct intel_iommu *iommu;
3494 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3495
3496 BUG_ON(dir == DMA_NONE);
3497
3498 if (iommu_no_mapping(dev))
3499 return paddr;
3500
3501 domain = get_valid_domain_for_dev(dev);
3502 if (!domain)
3503 return 0;
3504
3505 iommu = domain_get_iommu(domain);
3506 size = aligned_nrpages(paddr, size);
3507
3508 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3509 if (!iova_pfn)
3510 goto error;
3511
3512 /*
3513 * Check if DMAR supports zero-length reads on write only
3514 * mappings..
3515 */
3516 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3517 !cap_zlr(iommu->cap))
3518 prot |= DMA_PTE_READ;
3519 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3520 prot |= DMA_PTE_WRITE;
3521 /*
3522 * paddr - (paddr + size) might be partial page, we should map the whole
3523 * page. Note: if two part of one page are separately mapped, we
3524 * might have two guest_addr mapping to the same host paddr, but this
3525 * is not a big problem
3526 */
3527 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3528 mm_to_dma_pfn(paddr_pfn), size, prot);
3529 if (ret)
3530 goto error;
3531
3532 /* it's a non-present to present mapping. Only flush if caching mode */
3533 if (cap_caching_mode(iommu->cap))
3534 iommu_flush_iotlb_psi(iommu, domain,
3535 mm_to_dma_pfn(iova_pfn),
3536 size, 0, 1);
3537 else
3538 iommu_flush_write_buffer(iommu);
3539
3540 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3541 start_paddr += paddr & ~PAGE_MASK;
3542 return start_paddr;
3543
3544 error:
3545 if (iova_pfn)
3546 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3547 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3548 dev_name(dev), size, (unsigned long long)paddr, dir);
3549 return 0;
3550 }
3551
3552 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3553 unsigned long offset, size_t size,
3554 enum dma_data_direction dir,
3555 unsigned long attrs)
3556 {
3557 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3558 dir, *dev->dma_mask);
3559 }
3560
3561 static void flush_unmaps(struct deferred_flush_data *flush_data)
3562 {
3563 int i, j;
3564
3565 flush_data->timer_on = 0;
3566
3567 /* just flush them all */
3568 for (i = 0; i < g_num_of_iommus; i++) {
3569 struct intel_iommu *iommu = g_iommus[i];
3570 struct deferred_flush_table *flush_table =
3571 &flush_data->tables[i];
3572 if (!iommu)
3573 continue;
3574
3575 if (!flush_table->next)
3576 continue;
3577
3578 /* In caching mode, global flushes turn emulation expensive */
3579 if (!cap_caching_mode(iommu->cap))
3580 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3581 DMA_TLB_GLOBAL_FLUSH);
3582 for (j = 0; j < flush_table->next; j++) {
3583 unsigned long mask;
3584 struct deferred_flush_entry *entry =
3585 &flush_table->entries[j];
3586 unsigned long iova_pfn = entry->iova_pfn;
3587 unsigned long nrpages = entry->nrpages;
3588 struct dmar_domain *domain = entry->domain;
3589 struct page *freelist = entry->freelist;
3590
3591 /* On real hardware multiple invalidations are expensive */
3592 if (cap_caching_mode(iommu->cap))
3593 iommu_flush_iotlb_psi(iommu, domain,
3594 mm_to_dma_pfn(iova_pfn),
3595 nrpages, !freelist, 0);
3596 else {
3597 mask = ilog2(nrpages);
3598 iommu_flush_dev_iotlb(domain,
3599 (uint64_t)iova_pfn << PAGE_SHIFT, mask);
3600 }
3601 free_iova_fast(&domain->iovad, iova_pfn, nrpages);
3602 if (freelist)
3603 dma_free_pagelist(freelist);
3604 }
3605 flush_table->next = 0;
3606 }
3607
3608 flush_data->size = 0;
3609 }
3610
3611 static void flush_unmaps_timeout(unsigned long cpuid)
3612 {
3613 struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3614 unsigned long flags;
3615
3616 spin_lock_irqsave(&flush_data->lock, flags);
3617 flush_unmaps(flush_data);
3618 spin_unlock_irqrestore(&flush_data->lock, flags);
3619 }
3620
3621 static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
3622 unsigned long nrpages, struct page *freelist)
3623 {
3624 unsigned long flags;
3625 int entry_id, iommu_id;
3626 struct intel_iommu *iommu;
3627 struct deferred_flush_entry *entry;
3628 struct deferred_flush_data *flush_data;
3629 unsigned int cpuid;
3630
3631 cpuid = get_cpu();
3632 flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3633
3634 /* Flush all CPUs' entries to avoid deferring too much. If
3635 * this becomes a bottleneck, can just flush us, and rely on
3636 * flush timer for the rest.
3637 */
3638 if (flush_data->size == HIGH_WATER_MARK) {
3639 int cpu;
3640
3641 for_each_online_cpu(cpu)
3642 flush_unmaps_timeout(cpu);
3643 }
3644
3645 spin_lock_irqsave(&flush_data->lock, flags);
3646
3647 iommu = domain_get_iommu(dom);
3648 iommu_id = iommu->seq_id;
3649
3650 entry_id = flush_data->tables[iommu_id].next;
3651 ++(flush_data->tables[iommu_id].next);
3652
3653 entry = &flush_data->tables[iommu_id].entries[entry_id];
3654 entry->domain = dom;
3655 entry->iova_pfn = iova_pfn;
3656 entry->nrpages = nrpages;
3657 entry->freelist = freelist;
3658
3659 if (!flush_data->timer_on) {
3660 mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
3661 flush_data->timer_on = 1;
3662 }
3663 flush_data->size++;
3664 spin_unlock_irqrestore(&flush_data->lock, flags);
3665
3666 put_cpu();
3667 }
3668
3669 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3670 {
3671 struct dmar_domain *domain;
3672 unsigned long start_pfn, last_pfn;
3673 unsigned long nrpages;
3674 unsigned long iova_pfn;
3675 struct intel_iommu *iommu;
3676 struct page *freelist;
3677
3678 if (iommu_no_mapping(dev))
3679 return;
3680
3681 domain = find_domain(dev);
3682 BUG_ON(!domain);
3683
3684 iommu = domain_get_iommu(domain);
3685
3686 iova_pfn = IOVA_PFN(dev_addr);
3687
3688 nrpages = aligned_nrpages(dev_addr, size);
3689 start_pfn = mm_to_dma_pfn(iova_pfn);
3690 last_pfn = start_pfn + nrpages - 1;
3691
3692 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3693 dev_name(dev), start_pfn, last_pfn);
3694
3695 freelist = domain_unmap(domain, start_pfn, last_pfn);
3696
3697 if (intel_iommu_strict) {
3698 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3699 nrpages, !freelist, 0);
3700 /* free iova */
3701 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3702 dma_free_pagelist(freelist);
3703 } else {
3704 add_unmap(domain, iova_pfn, nrpages, freelist);
3705 /*
3706 * queue up the release of the unmap to save the 1/6th of the
3707 * cpu used up by the iotlb flush operation...
3708 */
3709 }
3710 }
3711
3712 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3713 size_t size, enum dma_data_direction dir,
3714 unsigned long attrs)
3715 {
3716 intel_unmap(dev, dev_addr, size);
3717 }
3718
3719 static void *intel_alloc_coherent(struct device *dev, size_t size,
3720 dma_addr_t *dma_handle, gfp_t flags,
3721 unsigned long attrs)
3722 {
3723 struct page *page = NULL;
3724 int order;
3725
3726 size = PAGE_ALIGN(size);
3727 order = get_order(size);
3728
3729 if (!iommu_no_mapping(dev))
3730 flags &= ~(GFP_DMA | GFP_DMA32);
3731 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3732 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3733 flags |= GFP_DMA;
3734 else
3735 flags |= GFP_DMA32;
3736 }
3737
3738 if (gfpflags_allow_blocking(flags)) {
3739 unsigned int count = size >> PAGE_SHIFT;
3740
3741 page = dma_alloc_from_contiguous(dev, count, order);
3742 if (page && iommu_no_mapping(dev) &&
3743 page_to_phys(page) + size > dev->coherent_dma_mask) {
3744 dma_release_from_contiguous(dev, page, count);
3745 page = NULL;
3746 }
3747 }
3748
3749 if (!page)
3750 page = alloc_pages(flags, order);
3751 if (!page)
3752 return NULL;
3753 memset(page_address(page), 0, size);
3754
3755 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3756 DMA_BIDIRECTIONAL,
3757 dev->coherent_dma_mask);
3758 if (*dma_handle)
3759 return page_address(page);
3760 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3761 __free_pages(page, order);
3762
3763 return NULL;
3764 }
3765
3766 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3767 dma_addr_t dma_handle, unsigned long attrs)
3768 {
3769 int order;
3770 struct page *page = virt_to_page(vaddr);
3771
3772 size = PAGE_ALIGN(size);
3773 order = get_order(size);
3774
3775 intel_unmap(dev, dma_handle, size);
3776 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3777 __free_pages(page, order);
3778 }
3779
3780 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3781 int nelems, enum dma_data_direction dir,
3782 unsigned long attrs)
3783 {
3784 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3785 unsigned long nrpages = 0;
3786 struct scatterlist *sg;
3787 int i;
3788
3789 for_each_sg(sglist, sg, nelems, i) {
3790 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3791 }
3792
3793 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3794 }
3795
3796 static int intel_nontranslate_map_sg(struct device *hddev,
3797 struct scatterlist *sglist, int nelems, int dir)
3798 {
3799 int i;
3800 struct scatterlist *sg;
3801
3802 for_each_sg(sglist, sg, nelems, i) {
3803 BUG_ON(!sg_page(sg));
3804 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3805 sg->dma_length = sg->length;
3806 }
3807 return nelems;
3808 }
3809
3810 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3811 enum dma_data_direction dir, unsigned long attrs)
3812 {
3813 int i;
3814 struct dmar_domain *domain;
3815 size_t size = 0;
3816 int prot = 0;
3817 unsigned long iova_pfn;
3818 int ret;
3819 struct scatterlist *sg;
3820 unsigned long start_vpfn;
3821 struct intel_iommu *iommu;
3822
3823 BUG_ON(dir == DMA_NONE);
3824 if (iommu_no_mapping(dev))
3825 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3826
3827 domain = get_valid_domain_for_dev(dev);
3828 if (!domain)
3829 return 0;
3830
3831 iommu = domain_get_iommu(domain);
3832
3833 for_each_sg(sglist, sg, nelems, i)
3834 size += aligned_nrpages(sg->offset, sg->length);
3835
3836 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3837 *dev->dma_mask);
3838 if (!iova_pfn) {
3839 sglist->dma_length = 0;
3840 return 0;
3841 }
3842
3843 /*
3844 * Check if DMAR supports zero-length reads on write only
3845 * mappings..
3846 */
3847 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3848 !cap_zlr(iommu->cap))
3849 prot |= DMA_PTE_READ;
3850 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3851 prot |= DMA_PTE_WRITE;
3852
3853 start_vpfn = mm_to_dma_pfn(iova_pfn);
3854
3855 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3856 if (unlikely(ret)) {
3857 dma_pte_free_pagetable(domain, start_vpfn,
3858 start_vpfn + size - 1);
3859 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3860 return 0;
3861 }
3862
3863 /* it's a non-present to present mapping. Only flush if caching mode */
3864 if (cap_caching_mode(iommu->cap))
3865 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3866 else
3867 iommu_flush_write_buffer(iommu);
3868
3869 return nelems;
3870 }
3871
3872 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3873 {
3874 return !dma_addr;
3875 }
3876
3877 struct dma_map_ops intel_dma_ops = {
3878 .alloc = intel_alloc_coherent,
3879 .free = intel_free_coherent,
3880 .map_sg = intel_map_sg,
3881 .unmap_sg = intel_unmap_sg,
3882 .map_page = intel_map_page,
3883 .unmap_page = intel_unmap_page,
3884 .mapping_error = intel_mapping_error,
3885 };
3886
3887 static inline int iommu_domain_cache_init(void)
3888 {
3889 int ret = 0;
3890
3891 iommu_domain_cache = kmem_cache_create("iommu_domain",
3892 sizeof(struct dmar_domain),
3893 0,
3894 SLAB_HWCACHE_ALIGN,
3895
3896 NULL);
3897 if (!iommu_domain_cache) {
3898 pr_err("Couldn't create iommu_domain cache\n");
3899 ret = -ENOMEM;
3900 }
3901
3902 return ret;
3903 }
3904
3905 static inline int iommu_devinfo_cache_init(void)
3906 {
3907 int ret = 0;
3908
3909 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3910 sizeof(struct device_domain_info),
3911 0,
3912 SLAB_HWCACHE_ALIGN,
3913 NULL);
3914 if (!iommu_devinfo_cache) {
3915 pr_err("Couldn't create devinfo cache\n");
3916 ret = -ENOMEM;
3917 }
3918
3919 return ret;
3920 }
3921
3922 static int __init iommu_init_mempool(void)
3923 {
3924 int ret;
3925 ret = iova_cache_get();
3926 if (ret)
3927 return ret;
3928
3929 ret = iommu_domain_cache_init();
3930 if (ret)
3931 goto domain_error;
3932
3933 ret = iommu_devinfo_cache_init();
3934 if (!ret)
3935 return ret;
3936
3937 kmem_cache_destroy(iommu_domain_cache);
3938 domain_error:
3939 iova_cache_put();
3940
3941 return -ENOMEM;
3942 }
3943
3944 static void __init iommu_exit_mempool(void)
3945 {
3946 kmem_cache_destroy(iommu_devinfo_cache);
3947 kmem_cache_destroy(iommu_domain_cache);
3948 iova_cache_put();
3949 }
3950
3951 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3952 {
3953 struct dmar_drhd_unit *drhd;
3954 u32 vtbar;
3955 int rc;
3956
3957 /* We know that this device on this chipset has its own IOMMU.
3958 * If we find it under a different IOMMU, then the BIOS is lying
3959 * to us. Hope that the IOMMU for this device is actually
3960 * disabled, and it needs no translation...
3961 */
3962 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3963 if (rc) {
3964 /* "can't" happen */
3965 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3966 return;
3967 }
3968 vtbar &= 0xffff0000;
3969
3970 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3971 drhd = dmar_find_matched_drhd_unit(pdev);
3972 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3973 TAINT_FIRMWARE_WORKAROUND,
3974 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3975 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3976 }
3977 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3978
3979 static void __init init_no_remapping_devices(void)
3980 {
3981 struct dmar_drhd_unit *drhd;
3982 struct device *dev;
3983 int i;
3984
3985 for_each_drhd_unit(drhd) {
3986 if (!drhd->include_all) {
3987 for_each_active_dev_scope(drhd->devices,
3988 drhd->devices_cnt, i, dev)
3989 break;
3990 /* ignore DMAR unit if no devices exist */
3991 if (i == drhd->devices_cnt)
3992 drhd->ignored = 1;
3993 }
3994 }
3995
3996 for_each_active_drhd_unit(drhd) {
3997 if (drhd->include_all)
3998 continue;
3999
4000 for_each_active_dev_scope(drhd->devices,
4001 drhd->devices_cnt, i, dev)
4002 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4003 break;
4004 if (i < drhd->devices_cnt)
4005 continue;
4006
4007 /* This IOMMU has *only* gfx devices. Either bypass it or
4008 set the gfx_mapped flag, as appropriate */
4009 if (dmar_map_gfx) {
4010 intel_iommu_gfx_mapped = 1;
4011 } else {
4012 drhd->ignored = 1;
4013 for_each_active_dev_scope(drhd->devices,
4014 drhd->devices_cnt, i, dev)
4015 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4016 }
4017 }
4018 }
4019
4020 #ifdef CONFIG_SUSPEND
4021 static int init_iommu_hw(void)
4022 {
4023 struct dmar_drhd_unit *drhd;
4024 struct intel_iommu *iommu = NULL;
4025
4026 for_each_active_iommu(iommu, drhd)
4027 if (iommu->qi)
4028 dmar_reenable_qi(iommu);
4029
4030 for_each_iommu(iommu, drhd) {
4031 if (drhd->ignored) {
4032 /*
4033 * we always have to disable PMRs or DMA may fail on
4034 * this device
4035 */
4036 if (force_on)
4037 iommu_disable_protect_mem_regions(iommu);
4038 continue;
4039 }
4040
4041 iommu_flush_write_buffer(iommu);
4042
4043 iommu_set_root_entry(iommu);
4044
4045 iommu->flush.flush_context(iommu, 0, 0, 0,
4046 DMA_CCMD_GLOBAL_INVL);
4047 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4048 iommu_enable_translation(iommu);
4049 iommu_disable_protect_mem_regions(iommu);
4050 }
4051
4052 return 0;
4053 }
4054
4055 static void iommu_flush_all(void)
4056 {
4057 struct dmar_drhd_unit *drhd;
4058 struct intel_iommu *iommu;
4059
4060 for_each_active_iommu(iommu, drhd) {
4061 iommu->flush.flush_context(iommu, 0, 0, 0,
4062 DMA_CCMD_GLOBAL_INVL);
4063 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4064 DMA_TLB_GLOBAL_FLUSH);
4065 }
4066 }
4067
4068 static int iommu_suspend(void)
4069 {
4070 struct dmar_drhd_unit *drhd;
4071 struct intel_iommu *iommu = NULL;
4072 unsigned long flag;
4073
4074 for_each_active_iommu(iommu, drhd) {
4075 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4076 GFP_ATOMIC);
4077 if (!iommu->iommu_state)
4078 goto nomem;
4079 }
4080
4081 iommu_flush_all();
4082
4083 for_each_active_iommu(iommu, drhd) {
4084 iommu_disable_translation(iommu);
4085
4086 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4087
4088 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4089 readl(iommu->reg + DMAR_FECTL_REG);
4090 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4091 readl(iommu->reg + DMAR_FEDATA_REG);
4092 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4093 readl(iommu->reg + DMAR_FEADDR_REG);
4094 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4095 readl(iommu->reg + DMAR_FEUADDR_REG);
4096
4097 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4098 }
4099 return 0;
4100
4101 nomem:
4102 for_each_active_iommu(iommu, drhd)
4103 kfree(iommu->iommu_state);
4104
4105 return -ENOMEM;
4106 }
4107
4108 static void iommu_resume(void)
4109 {
4110 struct dmar_drhd_unit *drhd;
4111 struct intel_iommu *iommu = NULL;
4112 unsigned long flag;
4113
4114 if (init_iommu_hw()) {
4115 if (force_on)
4116 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4117 else
4118 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4119 return;
4120 }
4121
4122 for_each_active_iommu(iommu, drhd) {
4123
4124 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4125
4126 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4127 iommu->reg + DMAR_FECTL_REG);
4128 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4129 iommu->reg + DMAR_FEDATA_REG);
4130 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4131 iommu->reg + DMAR_FEADDR_REG);
4132 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4133 iommu->reg + DMAR_FEUADDR_REG);
4134
4135 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4136 }
4137
4138 for_each_active_iommu(iommu, drhd)
4139 kfree(iommu->iommu_state);
4140 }
4141
4142 static struct syscore_ops iommu_syscore_ops = {
4143 .resume = iommu_resume,
4144 .suspend = iommu_suspend,
4145 };
4146
4147 static void __init init_iommu_pm_ops(void)
4148 {
4149 register_syscore_ops(&iommu_syscore_ops);
4150 }
4151
4152 #else
4153 static inline void init_iommu_pm_ops(void) {}
4154 #endif /* CONFIG_PM */
4155
4156
4157 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4158 {
4159 struct acpi_dmar_reserved_memory *rmrr;
4160 struct dmar_rmrr_unit *rmrru;
4161
4162 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4163 if (!rmrru)
4164 return -ENOMEM;
4165
4166 rmrru->hdr = header;
4167 rmrr = (struct acpi_dmar_reserved_memory *)header;
4168 rmrru->base_address = rmrr->base_address;
4169 rmrru->end_address = rmrr->end_address;
4170 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4171 ((void *)rmrr) + rmrr->header.length,
4172 &rmrru->devices_cnt);
4173 if (rmrru->devices_cnt && rmrru->devices == NULL) {
4174 kfree(rmrru);
4175 return -ENOMEM;
4176 }
4177
4178 list_add(&rmrru->list, &dmar_rmrr_units);
4179
4180 return 0;
4181 }
4182
4183 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4184 {
4185 struct dmar_atsr_unit *atsru;
4186 struct acpi_dmar_atsr *tmp;
4187
4188 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4189 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4190 if (atsr->segment != tmp->segment)
4191 continue;
4192 if (atsr->header.length != tmp->header.length)
4193 continue;
4194 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4195 return atsru;
4196 }
4197
4198 return NULL;
4199 }
4200
4201 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4202 {
4203 struct acpi_dmar_atsr *atsr;
4204 struct dmar_atsr_unit *atsru;
4205
4206 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4207 return 0;
4208
4209 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4210 atsru = dmar_find_atsr(atsr);
4211 if (atsru)
4212 return 0;
4213
4214 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4215 if (!atsru)
4216 return -ENOMEM;
4217
4218 /*
4219 * If memory is allocated from slab by ACPI _DSM method, we need to
4220 * copy the memory content because the memory buffer will be freed
4221 * on return.
4222 */
4223 atsru->hdr = (void *)(atsru + 1);
4224 memcpy(atsru->hdr, hdr, hdr->length);
4225 atsru->include_all = atsr->flags & 0x1;
4226 if (!atsru->include_all) {
4227 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4228 (void *)atsr + atsr->header.length,
4229 &atsru->devices_cnt);
4230 if (atsru->devices_cnt && atsru->devices == NULL) {
4231 kfree(atsru);
4232 return -ENOMEM;
4233 }
4234 }
4235
4236 list_add_rcu(&atsru->list, &dmar_atsr_units);
4237
4238 return 0;
4239 }
4240
4241 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4242 {
4243 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4244 kfree(atsru);
4245 }
4246
4247 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4248 {
4249 struct acpi_dmar_atsr *atsr;
4250 struct dmar_atsr_unit *atsru;
4251
4252 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4253 atsru = dmar_find_atsr(atsr);
4254 if (atsru) {
4255 list_del_rcu(&atsru->list);
4256 synchronize_rcu();
4257 intel_iommu_free_atsr(atsru);
4258 }
4259
4260 return 0;
4261 }
4262
4263 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4264 {
4265 int i;
4266 struct device *dev;
4267 struct acpi_dmar_atsr *atsr;
4268 struct dmar_atsr_unit *atsru;
4269
4270 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4271 atsru = dmar_find_atsr(atsr);
4272 if (!atsru)
4273 return 0;
4274
4275 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4276 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4277 i, dev)
4278 return -EBUSY;
4279 }
4280
4281 return 0;
4282 }
4283
4284 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4285 {
4286 int sp, ret = 0;
4287 struct intel_iommu *iommu = dmaru->iommu;
4288
4289 if (g_iommus[iommu->seq_id])
4290 return 0;
4291
4292 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4293 pr_warn("%s: Doesn't support hardware pass through.\n",
4294 iommu->name);
4295 return -ENXIO;
4296 }
4297 if (!ecap_sc_support(iommu->ecap) &&
4298 domain_update_iommu_snooping(iommu)) {
4299 pr_warn("%s: Doesn't support snooping.\n",
4300 iommu->name);
4301 return -ENXIO;
4302 }
4303 sp = domain_update_iommu_superpage(iommu) - 1;
4304 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4305 pr_warn("%s: Doesn't support large page.\n",
4306 iommu->name);
4307 return -ENXIO;
4308 }
4309
4310 /*
4311 * Disable translation if already enabled prior to OS handover.
4312 */
4313 if (iommu->gcmd & DMA_GCMD_TE)
4314 iommu_disable_translation(iommu);
4315
4316 g_iommus[iommu->seq_id] = iommu;
4317 ret = iommu_init_domains(iommu);
4318 if (ret == 0)
4319 ret = iommu_alloc_root_entry(iommu);
4320 if (ret)
4321 goto out;
4322
4323 #ifdef CONFIG_INTEL_IOMMU_SVM
4324 if (pasid_enabled(iommu))
4325 intel_svm_alloc_pasid_tables(iommu);
4326 #endif
4327
4328 if (dmaru->ignored) {
4329 /*
4330 * we always have to disable PMRs or DMA may fail on this device
4331 */
4332 if (force_on)
4333 iommu_disable_protect_mem_regions(iommu);
4334 return 0;
4335 }
4336
4337 intel_iommu_init_qi(iommu);
4338 iommu_flush_write_buffer(iommu);
4339
4340 #ifdef CONFIG_INTEL_IOMMU_SVM
4341 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4342 ret = intel_svm_enable_prq(iommu);
4343 if (ret)
4344 goto disable_iommu;
4345 }
4346 #endif
4347 ret = dmar_set_interrupt(iommu);
4348 if (ret)
4349 goto disable_iommu;
4350
4351 iommu_set_root_entry(iommu);
4352 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4353 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4354 iommu_enable_translation(iommu);
4355
4356 iommu_disable_protect_mem_regions(iommu);
4357 return 0;
4358
4359 disable_iommu:
4360 disable_dmar_iommu(iommu);
4361 out:
4362 free_dmar_iommu(iommu);
4363 return ret;
4364 }
4365
4366 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4367 {
4368 int ret = 0;
4369 struct intel_iommu *iommu = dmaru->iommu;
4370
4371 if (!intel_iommu_enabled)
4372 return 0;
4373 if (iommu == NULL)
4374 return -EINVAL;
4375
4376 if (insert) {
4377 ret = intel_iommu_add(dmaru);
4378 } else {
4379 disable_dmar_iommu(iommu);
4380 free_dmar_iommu(iommu);
4381 }
4382
4383 return ret;
4384 }
4385
4386 static void intel_iommu_free_dmars(void)
4387 {
4388 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4389 struct dmar_atsr_unit *atsru, *atsr_n;
4390
4391 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4392 list_del(&rmrru->list);
4393 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4394 kfree(rmrru);
4395 }
4396
4397 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4398 list_del(&atsru->list);
4399 intel_iommu_free_atsr(atsru);
4400 }
4401 }
4402
4403 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4404 {
4405 int i, ret = 1;
4406 struct pci_bus *bus;
4407 struct pci_dev *bridge = NULL;
4408 struct device *tmp;
4409 struct acpi_dmar_atsr *atsr;
4410 struct dmar_atsr_unit *atsru;
4411
4412 dev = pci_physfn(dev);
4413 for (bus = dev->bus; bus; bus = bus->parent) {
4414 bridge = bus->self;
4415 /* If it's an integrated device, allow ATS */
4416 if (!bridge)
4417 return 1;
4418 /* Connected via non-PCIe: no ATS */
4419 if (!pci_is_pcie(bridge) ||
4420 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4421 return 0;
4422 /* If we found the root port, look it up in the ATSR */
4423 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4424 break;
4425 }
4426
4427 rcu_read_lock();
4428 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4429 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4430 if (atsr->segment != pci_domain_nr(dev->bus))
4431 continue;
4432
4433 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4434 if (tmp == &bridge->dev)
4435 goto out;
4436
4437 if (atsru->include_all)
4438 goto out;
4439 }
4440 ret = 0;
4441 out:
4442 rcu_read_unlock();
4443
4444 return ret;
4445 }
4446
4447 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4448 {
4449 int ret = 0;
4450 struct dmar_rmrr_unit *rmrru;
4451 struct dmar_atsr_unit *atsru;
4452 struct acpi_dmar_atsr *atsr;
4453 struct acpi_dmar_reserved_memory *rmrr;
4454
4455 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4456 return 0;
4457
4458 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4459 rmrr = container_of(rmrru->hdr,
4460 struct acpi_dmar_reserved_memory, header);
4461 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4462 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4463 ((void *)rmrr) + rmrr->header.length,
4464 rmrr->segment, rmrru->devices,
4465 rmrru->devices_cnt);
4466 if(ret < 0)
4467 return ret;
4468 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4469 dmar_remove_dev_scope(info, rmrr->segment,
4470 rmrru->devices, rmrru->devices_cnt);
4471 }
4472 }
4473
4474 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4475 if (atsru->include_all)
4476 continue;
4477
4478 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4479 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4480 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4481 (void *)atsr + atsr->header.length,
4482 atsr->segment, atsru->devices,
4483 atsru->devices_cnt);
4484 if (ret > 0)
4485 break;
4486 else if(ret < 0)
4487 return ret;
4488 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4489 if (dmar_remove_dev_scope(info, atsr->segment,
4490 atsru->devices, atsru->devices_cnt))
4491 break;
4492 }
4493 }
4494
4495 return 0;
4496 }
4497
4498 /*
4499 * Here we only respond to action of unbound device from driver.
4500 *
4501 * Added device is not attached to its DMAR domain here yet. That will happen
4502 * when mapping the device to iova.
4503 */
4504 static int device_notifier(struct notifier_block *nb,
4505 unsigned long action, void *data)
4506 {
4507 struct device *dev = data;
4508 struct dmar_domain *domain;
4509
4510 if (iommu_dummy(dev))
4511 return 0;
4512
4513 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4514 return 0;
4515
4516 domain = find_domain(dev);
4517 if (!domain)
4518 return 0;
4519
4520 dmar_remove_one_dev_info(domain, dev);
4521 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4522 domain_exit(domain);
4523
4524 return 0;
4525 }
4526
4527 static struct notifier_block device_nb = {
4528 .notifier_call = device_notifier,
4529 };
4530
4531 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4532 unsigned long val, void *v)
4533 {
4534 struct memory_notify *mhp = v;
4535 unsigned long long start, end;
4536 unsigned long start_vpfn, last_vpfn;
4537
4538 switch (val) {
4539 case MEM_GOING_ONLINE:
4540 start = mhp->start_pfn << PAGE_SHIFT;
4541 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4542 if (iommu_domain_identity_map(si_domain, start, end)) {
4543 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4544 start, end);
4545 return NOTIFY_BAD;
4546 }
4547 break;
4548
4549 case MEM_OFFLINE:
4550 case MEM_CANCEL_ONLINE:
4551 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4552 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4553 while (start_vpfn <= last_vpfn) {
4554 struct iova *iova;
4555 struct dmar_drhd_unit *drhd;
4556 struct intel_iommu *iommu;
4557 struct page *freelist;
4558
4559 iova = find_iova(&si_domain->iovad, start_vpfn);
4560 if (iova == NULL) {
4561 pr_debug("Failed get IOVA for PFN %lx\n",
4562 start_vpfn);
4563 break;
4564 }
4565
4566 iova = split_and_remove_iova(&si_domain->iovad, iova,
4567 start_vpfn, last_vpfn);
4568 if (iova == NULL) {
4569 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4570 start_vpfn, last_vpfn);
4571 return NOTIFY_BAD;
4572 }
4573
4574 freelist = domain_unmap(si_domain, iova->pfn_lo,
4575 iova->pfn_hi);
4576
4577 rcu_read_lock();
4578 for_each_active_iommu(iommu, drhd)
4579 iommu_flush_iotlb_psi(iommu, si_domain,
4580 iova->pfn_lo, iova_size(iova),
4581 !freelist, 0);
4582 rcu_read_unlock();
4583 dma_free_pagelist(freelist);
4584
4585 start_vpfn = iova->pfn_hi + 1;
4586 free_iova_mem(iova);
4587 }
4588 break;
4589 }
4590
4591 return NOTIFY_OK;
4592 }
4593
4594 static struct notifier_block intel_iommu_memory_nb = {
4595 .notifier_call = intel_iommu_memory_notifier,
4596 .priority = 0
4597 };
4598
4599 static void free_all_cpu_cached_iovas(unsigned int cpu)
4600 {
4601 int i;
4602
4603 for (i = 0; i < g_num_of_iommus; i++) {
4604 struct intel_iommu *iommu = g_iommus[i];
4605 struct dmar_domain *domain;
4606 int did;
4607
4608 if (!iommu)
4609 continue;
4610
4611 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4612 domain = get_iommu_domain(iommu, (u16)did);
4613
4614 if (!domain)
4615 continue;
4616 free_cpu_cached_iovas(cpu, &domain->iovad);
4617 }
4618 }
4619 }
4620
4621 static int intel_iommu_cpu_notifier(struct notifier_block *nfb,
4622 unsigned long action, void *v)
4623 {
4624 unsigned int cpu = (unsigned long)v;
4625
4626 switch (action) {
4627 case CPU_DEAD:
4628 case CPU_DEAD_FROZEN:
4629 free_all_cpu_cached_iovas(cpu);
4630 flush_unmaps_timeout(cpu);
4631 break;
4632 }
4633 return NOTIFY_OK;
4634 }
4635
4636 static struct notifier_block intel_iommu_cpu_nb = {
4637 .notifier_call = intel_iommu_cpu_notifier,
4638 };
4639
4640 static ssize_t intel_iommu_show_version(struct device *dev,
4641 struct device_attribute *attr,
4642 char *buf)
4643 {
4644 struct intel_iommu *iommu = dev_get_drvdata(dev);
4645 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4646 return sprintf(buf, "%d:%d\n",
4647 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4648 }
4649 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4650
4651 static ssize_t intel_iommu_show_address(struct device *dev,
4652 struct device_attribute *attr,
4653 char *buf)
4654 {
4655 struct intel_iommu *iommu = dev_get_drvdata(dev);
4656 return sprintf(buf, "%llx\n", iommu->reg_phys);
4657 }
4658 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4659
4660 static ssize_t intel_iommu_show_cap(struct device *dev,
4661 struct device_attribute *attr,
4662 char *buf)
4663 {
4664 struct intel_iommu *iommu = dev_get_drvdata(dev);
4665 return sprintf(buf, "%llx\n", iommu->cap);
4666 }
4667 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4668
4669 static ssize_t intel_iommu_show_ecap(struct device *dev,
4670 struct device_attribute *attr,
4671 char *buf)
4672 {
4673 struct intel_iommu *iommu = dev_get_drvdata(dev);
4674 return sprintf(buf, "%llx\n", iommu->ecap);
4675 }
4676 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4677
4678 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4679 struct device_attribute *attr,
4680 char *buf)
4681 {
4682 struct intel_iommu *iommu = dev_get_drvdata(dev);
4683 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4684 }
4685 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4686
4687 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4688 struct device_attribute *attr,
4689 char *buf)
4690 {
4691 struct intel_iommu *iommu = dev_get_drvdata(dev);
4692 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4693 cap_ndoms(iommu->cap)));
4694 }
4695 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4696
4697 static struct attribute *intel_iommu_attrs[] = {
4698 &dev_attr_version.attr,
4699 &dev_attr_address.attr,
4700 &dev_attr_cap.attr,
4701 &dev_attr_ecap.attr,
4702 &dev_attr_domains_supported.attr,
4703 &dev_attr_domains_used.attr,
4704 NULL,
4705 };
4706
4707 static struct attribute_group intel_iommu_group = {
4708 .name = "intel-iommu",
4709 .attrs = intel_iommu_attrs,
4710 };
4711
4712 const struct attribute_group *intel_iommu_groups[] = {
4713 &intel_iommu_group,
4714 NULL,
4715 };
4716
4717 int __init intel_iommu_init(void)
4718 {
4719 int ret = -ENODEV;
4720 struct dmar_drhd_unit *drhd;
4721 struct intel_iommu *iommu;
4722
4723 /* VT-d is required for a TXT/tboot launch, so enforce that */
4724 force_on = tboot_force_iommu();
4725
4726 if (iommu_init_mempool()) {
4727 if (force_on)
4728 panic("tboot: Failed to initialize iommu memory\n");
4729 return -ENOMEM;
4730 }
4731
4732 down_write(&dmar_global_lock);
4733 if (dmar_table_init()) {
4734 if (force_on)
4735 panic("tboot: Failed to initialize DMAR table\n");
4736 goto out_free_dmar;
4737 }
4738
4739 if (dmar_dev_scope_init() < 0) {
4740 if (force_on)
4741 panic("tboot: Failed to initialize DMAR device scope\n");
4742 goto out_free_dmar;
4743 }
4744
4745 if (no_iommu || dmar_disabled)
4746 goto out_free_dmar;
4747
4748 if (list_empty(&dmar_rmrr_units))
4749 pr_info("No RMRR found\n");
4750
4751 if (list_empty(&dmar_atsr_units))
4752 pr_info("No ATSR found\n");
4753
4754 if (dmar_init_reserved_ranges()) {
4755 if (force_on)
4756 panic("tboot: Failed to reserve iommu ranges\n");
4757 goto out_free_reserved_range;
4758 }
4759
4760 init_no_remapping_devices();
4761
4762 ret = init_dmars();
4763 if (ret) {
4764 if (force_on)
4765 panic("tboot: Failed to initialize DMARs\n");
4766 pr_err("Initialization failed\n");
4767 goto out_free_reserved_range;
4768 }
4769 up_write(&dmar_global_lock);
4770 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4771
4772 #ifdef CONFIG_SWIOTLB
4773 swiotlb = 0;
4774 #endif
4775 dma_ops = &intel_dma_ops;
4776
4777 init_iommu_pm_ops();
4778
4779 for_each_active_iommu(iommu, drhd)
4780 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4781 intel_iommu_groups,
4782 "%s", iommu->name);
4783
4784 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4785 bus_register_notifier(&pci_bus_type, &device_nb);
4786 if (si_domain && !hw_pass_through)
4787 register_memory_notifier(&intel_iommu_memory_nb);
4788 register_hotcpu_notifier(&intel_iommu_cpu_nb);
4789
4790 intel_iommu_enabled = 1;
4791
4792 return 0;
4793
4794 out_free_reserved_range:
4795 put_iova_domain(&reserved_iova_list);
4796 out_free_dmar:
4797 intel_iommu_free_dmars();
4798 up_write(&dmar_global_lock);
4799 iommu_exit_mempool();
4800 return ret;
4801 }
4802
4803 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4804 {
4805 struct intel_iommu *iommu = opaque;
4806
4807 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4808 return 0;
4809 }
4810
4811 /*
4812 * NB - intel-iommu lacks any sort of reference counting for the users of
4813 * dependent devices. If multiple endpoints have intersecting dependent
4814 * devices, unbinding the driver from any one of them will possibly leave
4815 * the others unable to operate.
4816 */
4817 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4818 {
4819 if (!iommu || !dev || !dev_is_pci(dev))
4820 return;
4821
4822 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4823 }
4824
4825 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4826 {
4827 struct intel_iommu *iommu;
4828 unsigned long flags;
4829
4830 assert_spin_locked(&device_domain_lock);
4831
4832 if (WARN_ON(!info))
4833 return;
4834
4835 iommu = info->iommu;
4836
4837 if (info->dev) {
4838 iommu_disable_dev_iotlb(info);
4839 domain_context_clear(iommu, info->dev);
4840 }
4841
4842 unlink_domain_info(info);
4843
4844 spin_lock_irqsave(&iommu->lock, flags);
4845 domain_detach_iommu(info->domain, iommu);
4846 spin_unlock_irqrestore(&iommu->lock, flags);
4847
4848 free_devinfo_mem(info);
4849 }
4850
4851 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4852 struct device *dev)
4853 {
4854 struct device_domain_info *info;
4855 unsigned long flags;
4856
4857 spin_lock_irqsave(&device_domain_lock, flags);
4858 info = dev->archdata.iommu;
4859 __dmar_remove_one_dev_info(info);
4860 spin_unlock_irqrestore(&device_domain_lock, flags);
4861 }
4862
4863 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4864 {
4865 int adjust_width;
4866
4867 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4868 DMA_32BIT_PFN);
4869 domain_reserve_special_ranges(domain);
4870
4871 /* calculate AGAW */
4872 domain->gaw = guest_width;
4873 adjust_width = guestwidth_to_adjustwidth(guest_width);
4874 domain->agaw = width_to_agaw(adjust_width);
4875
4876 domain->iommu_coherency = 0;
4877 domain->iommu_snooping = 0;
4878 domain->iommu_superpage = 0;
4879 domain->max_addr = 0;
4880
4881 /* always allocate the top pgd */
4882 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4883 if (!domain->pgd)
4884 return -ENOMEM;
4885 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4886 return 0;
4887 }
4888
4889 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4890 {
4891 struct dmar_domain *dmar_domain;
4892 struct iommu_domain *domain;
4893
4894 if (type != IOMMU_DOMAIN_UNMANAGED)
4895 return NULL;
4896
4897 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4898 if (!dmar_domain) {
4899 pr_err("Can't allocate dmar_domain\n");
4900 return NULL;
4901 }
4902 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4903 pr_err("Domain initialization failed\n");
4904 domain_exit(dmar_domain);
4905 return NULL;
4906 }
4907 domain_update_iommu_cap(dmar_domain);
4908
4909 domain = &dmar_domain->domain;
4910 domain->geometry.aperture_start = 0;
4911 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4912 domain->geometry.force_aperture = true;
4913
4914 return domain;
4915 }
4916
4917 static void intel_iommu_domain_free(struct iommu_domain *domain)
4918 {
4919 domain_exit(to_dmar_domain(domain));
4920 }
4921
4922 static int intel_iommu_attach_device(struct iommu_domain *domain,
4923 struct device *dev)
4924 {
4925 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4926 struct intel_iommu *iommu;
4927 int addr_width;
4928 u8 bus, devfn;
4929
4930 if (device_is_rmrr_locked(dev)) {
4931 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4932 return -EPERM;
4933 }
4934
4935 /* normally dev is not mapped */
4936 if (unlikely(domain_context_mapped(dev))) {
4937 struct dmar_domain *old_domain;
4938
4939 old_domain = find_domain(dev);
4940 if (old_domain) {
4941 rcu_read_lock();
4942 dmar_remove_one_dev_info(old_domain, dev);
4943 rcu_read_unlock();
4944
4945 if (!domain_type_is_vm_or_si(old_domain) &&
4946 list_empty(&old_domain->devices))
4947 domain_exit(old_domain);
4948 }
4949 }
4950
4951 iommu = device_to_iommu(dev, &bus, &devfn);
4952 if (!iommu)
4953 return -ENODEV;
4954
4955 /* check if this iommu agaw is sufficient for max mapped address */
4956 addr_width = agaw_to_width(iommu->agaw);
4957 if (addr_width > cap_mgaw(iommu->cap))
4958 addr_width = cap_mgaw(iommu->cap);
4959
4960 if (dmar_domain->max_addr > (1LL << addr_width)) {
4961 pr_err("%s: iommu width (%d) is not "
4962 "sufficient for the mapped address (%llx)\n",
4963 __func__, addr_width, dmar_domain->max_addr);
4964 return -EFAULT;
4965 }
4966 dmar_domain->gaw = addr_width;
4967
4968 /*
4969 * Knock out extra levels of page tables if necessary
4970 */
4971 while (iommu->agaw < dmar_domain->agaw) {
4972 struct dma_pte *pte;
4973
4974 pte = dmar_domain->pgd;
4975 if (dma_pte_present(pte)) {
4976 dmar_domain->pgd = (struct dma_pte *)
4977 phys_to_virt(dma_pte_addr(pte));
4978 free_pgtable_page(pte);
4979 }
4980 dmar_domain->agaw--;
4981 }
4982
4983 return domain_add_dev_info(dmar_domain, dev);
4984 }
4985
4986 static void intel_iommu_detach_device(struct iommu_domain *domain,
4987 struct device *dev)
4988 {
4989 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
4990 }
4991
4992 static int intel_iommu_map(struct iommu_domain *domain,
4993 unsigned long iova, phys_addr_t hpa,
4994 size_t size, int iommu_prot)
4995 {
4996 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4997 u64 max_addr;
4998 int prot = 0;
4999 int ret;
5000
5001 if (iommu_prot & IOMMU_READ)
5002 prot |= DMA_PTE_READ;
5003 if (iommu_prot & IOMMU_WRITE)
5004 prot |= DMA_PTE_WRITE;
5005 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5006 prot |= DMA_PTE_SNP;
5007
5008 max_addr = iova + size;
5009 if (dmar_domain->max_addr < max_addr) {
5010 u64 end;
5011
5012 /* check if minimum agaw is sufficient for mapped address */
5013 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5014 if (end < max_addr) {
5015 pr_err("%s: iommu width (%d) is not "
5016 "sufficient for the mapped address (%llx)\n",
5017 __func__, dmar_domain->gaw, max_addr);
5018 return -EFAULT;
5019 }
5020 dmar_domain->max_addr = max_addr;
5021 }
5022 /* Round up size to next multiple of PAGE_SIZE, if it and
5023 the low bits of hpa would take us onto the next page */
5024 size = aligned_nrpages(hpa, size);
5025 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5026 hpa >> VTD_PAGE_SHIFT, size, prot);
5027 return ret;
5028 }
5029
5030 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5031 unsigned long iova, size_t size)
5032 {
5033 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5034 struct page *freelist = NULL;
5035 struct intel_iommu *iommu;
5036 unsigned long start_pfn, last_pfn;
5037 unsigned int npages;
5038 int iommu_id, level = 0;
5039
5040 /* Cope with horrid API which requires us to unmap more than the
5041 size argument if it happens to be a large-page mapping. */
5042 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5043
5044 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5045 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5046
5047 start_pfn = iova >> VTD_PAGE_SHIFT;
5048 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5049
5050 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5051
5052 npages = last_pfn - start_pfn + 1;
5053
5054 for_each_domain_iommu(iommu_id, dmar_domain) {
5055 iommu = g_iommus[iommu_id];
5056
5057 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5058 start_pfn, npages, !freelist, 0);
5059 }
5060
5061 dma_free_pagelist(freelist);
5062
5063 if (dmar_domain->max_addr == iova + size)
5064 dmar_domain->max_addr = iova;
5065
5066 return size;
5067 }
5068
5069 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5070 dma_addr_t iova)
5071 {
5072 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5073 struct dma_pte *pte;
5074 int level = 0;
5075 u64 phys = 0;
5076
5077 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5078 if (pte)
5079 phys = dma_pte_addr(pte);
5080
5081 return phys;
5082 }
5083
5084 static bool intel_iommu_capable(enum iommu_cap cap)
5085 {
5086 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5087 return domain_update_iommu_snooping(NULL) == 1;
5088 if (cap == IOMMU_CAP_INTR_REMAP)
5089 return irq_remapping_enabled == 1;
5090
5091 return false;
5092 }
5093
5094 static int intel_iommu_add_device(struct device *dev)
5095 {
5096 struct intel_iommu *iommu;
5097 struct iommu_group *group;
5098 u8 bus, devfn;
5099
5100 iommu = device_to_iommu(dev, &bus, &devfn);
5101 if (!iommu)
5102 return -ENODEV;
5103
5104 iommu_device_link(iommu->iommu_dev, dev);
5105
5106 group = iommu_group_get_for_dev(dev);
5107
5108 if (IS_ERR(group))
5109 return PTR_ERR(group);
5110
5111 iommu_group_put(group);
5112 return 0;
5113 }
5114
5115 static void intel_iommu_remove_device(struct device *dev)
5116 {
5117 struct intel_iommu *iommu;
5118 u8 bus, devfn;
5119
5120 iommu = device_to_iommu(dev, &bus, &devfn);
5121 if (!iommu)
5122 return;
5123
5124 iommu_group_remove_device(dev);
5125
5126 iommu_device_unlink(iommu->iommu_dev, dev);
5127 }
5128
5129 #ifdef CONFIG_INTEL_IOMMU_SVM
5130 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5131 {
5132 struct device_domain_info *info;
5133 struct context_entry *context;
5134 struct dmar_domain *domain;
5135 unsigned long flags;
5136 u64 ctx_lo;
5137 int ret;
5138
5139 domain = get_valid_domain_for_dev(sdev->dev);
5140 if (!domain)
5141 return -EINVAL;
5142
5143 spin_lock_irqsave(&device_domain_lock, flags);
5144 spin_lock(&iommu->lock);
5145
5146 ret = -EINVAL;
5147 info = sdev->dev->archdata.iommu;
5148 if (!info || !info->pasid_supported)
5149 goto out;
5150
5151 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5152 if (WARN_ON(!context))
5153 goto out;
5154
5155 ctx_lo = context[0].lo;
5156
5157 sdev->did = domain->iommu_did[iommu->seq_id];
5158 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5159
5160 if (!(ctx_lo & CONTEXT_PASIDE)) {
5161 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5162 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) | ecap_pss(iommu->ecap);
5163 wmb();
5164 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5165 * extended to permit requests-with-PASID if the PASIDE bit
5166 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5167 * however, the PASIDE bit is ignored and requests-with-PASID
5168 * are unconditionally blocked. Which makes less sense.
5169 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5170 * "guest mode" translation types depending on whether ATS
5171 * is available or not. Annoyingly, we can't use the new
5172 * modes *unless* PASIDE is set. */
5173 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5174 ctx_lo &= ~CONTEXT_TT_MASK;
5175 if (info->ats_supported)
5176 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5177 else
5178 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5179 }
5180 ctx_lo |= CONTEXT_PASIDE;
5181 if (iommu->pasid_state_table)
5182 ctx_lo |= CONTEXT_DINVE;
5183 if (info->pri_supported)
5184 ctx_lo |= CONTEXT_PRS;
5185 context[0].lo = ctx_lo;
5186 wmb();
5187 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5188 DMA_CCMD_MASK_NOBIT,
5189 DMA_CCMD_DEVICE_INVL);
5190 }
5191
5192 /* Enable PASID support in the device, if it wasn't already */
5193 if (!info->pasid_enabled)
5194 iommu_enable_dev_iotlb(info);
5195
5196 if (info->ats_enabled) {
5197 sdev->dev_iotlb = 1;
5198 sdev->qdep = info->ats_qdep;
5199 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5200 sdev->qdep = 0;
5201 }
5202 ret = 0;
5203
5204 out:
5205 spin_unlock(&iommu->lock);
5206 spin_unlock_irqrestore(&device_domain_lock, flags);
5207
5208 return ret;
5209 }
5210
5211 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5212 {
5213 struct intel_iommu *iommu;
5214 u8 bus, devfn;
5215
5216 if (iommu_dummy(dev)) {
5217 dev_warn(dev,
5218 "No IOMMU translation for device; cannot enable SVM\n");
5219 return NULL;
5220 }
5221
5222 iommu = device_to_iommu(dev, &bus, &devfn);
5223 if ((!iommu)) {
5224 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5225 return NULL;
5226 }
5227
5228 if (!iommu->pasid_table) {
5229 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5230 return NULL;
5231 }
5232
5233 return iommu;
5234 }
5235 #endif /* CONFIG_INTEL_IOMMU_SVM */
5236
5237 static const struct iommu_ops intel_iommu_ops = {
5238 .capable = intel_iommu_capable,
5239 .domain_alloc = intel_iommu_domain_alloc,
5240 .domain_free = intel_iommu_domain_free,
5241 .attach_dev = intel_iommu_attach_device,
5242 .detach_dev = intel_iommu_detach_device,
5243 .map = intel_iommu_map,
5244 .unmap = intel_iommu_unmap,
5245 .map_sg = default_iommu_map_sg,
5246 .iova_to_phys = intel_iommu_iova_to_phys,
5247 .add_device = intel_iommu_add_device,
5248 .remove_device = intel_iommu_remove_device,
5249 .device_group = pci_device_group,
5250 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5251 };
5252
5253 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5254 {
5255 /* G4x/GM45 integrated gfx dmar support is totally busted. */
5256 pr_info("Disabling IOMMU for graphics on this chipset\n");
5257 dmar_map_gfx = 0;
5258 }
5259
5260 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5261 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5262 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5263 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5264 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5265 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5266 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5267
5268 static void quirk_iommu_rwbf(struct pci_dev *dev)
5269 {
5270 /*
5271 * Mobile 4 Series Chipset neglects to set RWBF capability,
5272 * but needs it. Same seems to hold for the desktop versions.
5273 */
5274 pr_info("Forcing write-buffer flush capability\n");
5275 rwbf_quirk = 1;
5276 }
5277
5278 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5279 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5280 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5281 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5282 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5283 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5284 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5285
5286 #define GGC 0x52
5287 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
5288 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5289 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
5290 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
5291 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5292 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5293 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5294 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5295
5296 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5297 {
5298 unsigned short ggc;
5299
5300 if (pci_read_config_word(dev, GGC, &ggc))
5301 return;
5302
5303 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5304 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5305 dmar_map_gfx = 0;
5306 } else if (dmar_map_gfx) {
5307 /* we have to ensure the gfx device is idle before we flush */
5308 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5309 intel_iommu_strict = 1;
5310 }
5311 }
5312 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5313 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5314 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5315 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5316
5317 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5318 ISOCH DMAR unit for the Azalia sound device, but not give it any
5319 TLB entries, which causes it to deadlock. Check for that. We do
5320 this in a function called from init_dmars(), instead of in a PCI
5321 quirk, because we don't want to print the obnoxious "BIOS broken"
5322 message if VT-d is actually disabled.
5323 */
5324 static void __init check_tylersburg_isoch(void)
5325 {
5326 struct pci_dev *pdev;
5327 uint32_t vtisochctrl;
5328
5329 /* If there's no Azalia in the system anyway, forget it. */
5330 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5331 if (!pdev)
5332 return;
5333 pci_dev_put(pdev);
5334
5335 /* System Management Registers. Might be hidden, in which case
5336 we can't do the sanity check. But that's OK, because the
5337 known-broken BIOSes _don't_ actually hide it, so far. */
5338 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5339 if (!pdev)
5340 return;
5341
5342 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5343 pci_dev_put(pdev);
5344 return;
5345 }
5346
5347 pci_dev_put(pdev);
5348
5349 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5350 if (vtisochctrl & 1)
5351 return;
5352
5353 /* Drop all bits other than the number of TLB entries */
5354 vtisochctrl &= 0x1c;
5355
5356 /* If we have the recommended number of TLB entries (16), fine. */
5357 if (vtisochctrl == 0x10)
5358 return;
5359
5360 /* Zero TLB entries? You get to ride the short bus to school. */
5361 if (!vtisochctrl) {
5362 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5363 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5364 dmi_get_system_info(DMI_BIOS_VENDOR),
5365 dmi_get_system_info(DMI_BIOS_VERSION),
5366 dmi_get_system_info(DMI_PRODUCT_VERSION));
5367 iommu_identity_mapping |= IDENTMAP_AZALIA;
5368 return;
5369 }
5370
5371 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5372 vtisochctrl);
5373 }
CRIS linux kernel tree